[hfst] 01/02: Imported Upstream version 3.12.2~r3258
Tino Didriksen
tinodidriksen-guest at moszumanska.debian.org
Thu Mar 30 08:05:14 UTC 2017
This is an automated email from the git hooks/post-receive script.
tinodidriksen-guest pushed a commit to branch master
in repository hfst.
commit 5e4e3141330070b9bacb844cd152410152954e3d
Author: Tino Didriksen <tino at didriksen.cc>
Date: Thu Mar 30 08:04:19 2017 +0000
Imported Upstream version 3.12.2~r3258
---
.gitignore | 71 +-
.travis.yml | 41 +
ChangeLog | 334 +-
ChangeLog.old | 357 +-
NEWS | 80 +-
README | 467 +-
README => README.rst | 40 +-
autogen.sh | 4 +-
back-ends/dlfcn/dlfcn.c | 2 +-
back-ends/dlfcn/dlfcn.h | 14 +-
back-ends/foma/CHANGELOG | 10 +-
back-ends/foma/Makefile.am | 4 +-
back-ends/foma/README.cmatrix | 2 +-
back-ends/foma/README.symbols | 10 +-
back-ends/foma/apply.c | 42 +-
back-ends/foma/coaccessible.c | 2 +-
back-ends/foma/constructions.c | 106 +-
back-ends/foma/define.c | 21 +-
back-ends/foma/determinize.c | 18 +-
back-ends/foma/dynarray.c | 20 +-
back-ends/foma/extract.c | 1 +
back-ends/foma/flags.c | 40 +-
back-ends/foma/foma.c | 10 +-
back-ends/foma/foma.h | 2 +-
back-ends/foma/fomalib.h | 19 +-
back-ends/foma/fomalibconf.h | 46 +-
back-ends/foma/iface.c | 73 +-
back-ends/foma/io.c | 37 +-
back-ends/foma/lex.cmatrix.c | 46 +-
back-ends/foma/lex.interface.c | 7986 ++++++++++----------
back-ends/foma/lex.lexc.c | 98 +-
back-ends/foma/lex.yy.c | 1943 ++---
back-ends/foma/lexc.l | 12 +-
back-ends/foma/lexcread.c | 44 +-
back-ends/foma/mem.c | 8 +-
back-ends/foma/minimize.c | 26 +-
back-ends/foma/regex.c | 330 +-
back-ends/foma/regex.l | 10 +-
back-ends/foma/regex.y | 10 +-
back-ends/foma/rewrite.c | 1251 ++-
back-ends/foma/sigma.c | 14 +-
back-ends/foma/spelling.c | 32 +-
back-ends/foma/stack.c | 6 +-
back-ends/foma/structures.c | 26 +-
back-ends/foma/topsort.c | 6 +-
back-ends/foma/trie.c | 2 +-
back-ends/foma/utf8.c | 16 +-
back-ends/openfst/NEWS | 24 +-
back-ends/openfst/README | 12 +-
back-ends/openfst/src/include/fst/config.h | 2 +-
back-ends/openfst/src/include/fst/config.h.in | 2 +-
back-ends/openfst/src/include/fst/encode.h | 6 +-
.../openfst/src/include/fst/generic-register.h | 4 +
back-ends/openfst/src/include/fst/interval-set.h | 2 +-
back-ends/openfst/src/include/fst/minimize.h | 2 +-
back-ends/openfst/src/include/fst/partition.h | 2 +-
back-ends/openfst/src/lib/compat.cc | 2 +-
back-ends/openfstwin/README | 2 +-
back-ends/openfstwin/src/include/fst/compat.h | 4 +-
back-ends/openfstwin/src/include/fst/config.h | 2 +-
back-ends/openfstwin/src/include/fst/flags.h | 6 +-
back-ends/openfstwin/src/include/fst/replace.h | 2 +-
back-ends/openfstwin/src/lib/compat.cc | 6 +-
back-ends/sfst/alphabet.cc | 12 +-
back-ends/sfst/alphabet.h | 10 +-
back-ends/sfst/compact.cc | 34 +-
back-ends/sfst/determinise.cc | 6 +-
back-ends/sfst/fst.cc | 42 +-
back-ends/sfst/fst.h | 20 +-
back-ends/sfst/generate.cc | 4 +-
back-ends/sfst/hopcroft.cc | 28 +-
back-ends/sfst/interface.h | 12 +-
back-ends/sfst/operators.cc | 42 +-
back-ends/sfst/sgi.h | 4 +-
configure.ac | 98 +-
doc/api_documentation.dox | 164 +-
doc/hfst-lexc.rst | 8 +-
doc/hfst3-metadata-header-registry.rst | 12 +-
hfst.m4 | 4 +-
lgpl-release/COPYING | 165 +
lgpl-release/Makefile.am | 28 +
lgpl-release/README-DEV | 13 +
lgpl-release/back-ends/Makefile.am | 14 +
configure.ac => lgpl-release/configure.ac | 127 +-
libhfst/Makefile.am | 16 +-
libhfst/hfst.pc.in | 2 +-
libhfst/src/FormatSpecifiers.h | 12 +-
libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc | 49 +-
libhfst/src/HarmonizeUnknownAndIdentitySymbols.h | 39 +-
libhfst/src/HfstApply.cc | 92 +-
libhfst/src/HfstDataTypes.cc | 67 +-
libhfst/src/HfstDataTypes.h | 65 +-
libhfst/src/HfstEpsilonHandler.cc | 22 +-
libhfst/src/HfstEpsilonHandler.h | 14 +-
libhfst/src/HfstExceptionDefs.cc | 36 +-
libhfst/src/HfstExceptionDefs.h | 89 +-
libhfst/src/HfstExtractStrings.h | 44 +-
libhfst/src/HfstFlagDiacritics.cc | 46 +-
libhfst/src/HfstFlagDiacritics.h | 76 +-
libhfst/src/HfstInputStream.cc | 130 +-
libhfst/src/HfstInputStream.h | 110 +-
libhfst/src/HfstLookupFlagDiacritics.cc | 42 +-
libhfst/src/HfstLookupFlagDiacritics.h | 27 +-
libhfst/src/HfstOutputStream.cc | 70 +-
libhfst/src/HfstOutputStream.h | 56 +-
libhfst/src/HfstPrintDot.cc | 80 +-
libhfst/src/HfstPrintDot.h | 18 +-
libhfst/src/HfstPrintPCKimmo.cc | 30 +-
libhfst/src/HfstPrintPCKimmo.h | 16 +-
libhfst/src/HfstRules.cc | 240 +-
libhfst/src/HfstStrings2FstTokenizer.cc | 54 +-
libhfst/src/HfstStrings2FstTokenizer.h | 22 +-
libhfst/src/HfstSymbolDefs.cc | 18 +-
libhfst/src/HfstSymbolDefs.h | 62 +-
libhfst/src/HfstTokenizer.cc | 67 +-
libhfst/src/HfstTokenizer.h | 38 +-
libhfst/src/HfstTransducer.cc | 1154 ++-
libhfst/src/HfstTransducer.h | 722 +-
libhfst/src/HfstXeroxRules.cc | 6287 +++++++--------
libhfst/src/HfstXeroxRules.h | 10 +-
libhfst/src/HfstXeroxRulesTest.cc | 20 +-
libhfst/src/Makefile.am | 27 +-
libhfst/src/hfst-string-conversions.cc | 12 +-
libhfst/src/hfst-string-conversions.h | 4 +-
libhfst/src/hfst.h | 12 +-
libhfst/src/hfst_apply_schemas.h | 20 +-
libhfst/src/hfstdll.h | 12 +-
.../src/implementations/ConvertFomaTransducer.cc | 93 +-
.../implementations/ConvertLogWeightTransducer.cc | 103 +-
libhfst/src/implementations/ConvertOlTransducer.cc | 80 +-
.../src/implementations/ConvertSfstTransducer.cc | 51 +-
.../src/implementations/ConvertTransducerFormat.cc | 116 +-
.../src/implementations/ConvertTransducerFormat.h | 93 +-
.../ConvertTropicalWeightTransducer.cc | 91 +-
.../src/implementations/ConvertXfsmTransducer.cc | 40 +-
libhfst/src/implementations/FomaTransducer.cc | 326 +-
libhfst/src/implementations/FomaTransducer.h | 40 +-
...fstTransitionGraph.h => HfstBasicTransducer.cc} | 2083 +++--
libhfst/src/implementations/HfstBasicTransducer.h | 1120 +++
libhfst/src/implementations/HfstBasicTransition.cc | 99 +
libhfst/src/implementations/HfstBasicTransition.h | 116 +
libhfst/src/implementations/HfstOlTransducer.cc | 72 +-
libhfst/src/implementations/HfstOlTransducer.h | 22 +-
libhfst/src/implementations/HfstTransition.h | 46 +-
libhfst/src/implementations/HfstTransitionGraph.cc | 18 +-
libhfst/src/implementations/HfstTransitionGraph.h | 916 ++-
.../HfstTropicalTransducerTransitionData.cc | 264 +-
.../HfstTropicalTransducerTransitionData.h | 290 +-
libhfst/src/implementations/LogWeightTransducer.cc | 436 +-
libhfst/src/implementations/LogWeightTransducer.h | 98 +-
libhfst/src/implementations/Makefile.am | 69 +-
.../MyTransducerLibraryTransducer.cc | 80 +-
.../MyTransducerLibraryTransducer.h | 118 +-
libhfst/src/implementations/SfstTransducer.cc | 238 +-
libhfst/src/implementations/SfstTransducer.h | 38 +-
.../implementations/TropicalWeightTransducer.cc | 586 +-
.../src/implementations/TropicalWeightTransducer.h | 122 +-
libhfst/src/implementations/XfsmTransducer.cc | 56 +-
libhfst/src/implementations/XfsmTransducer.h | 18 +-
.../compose_intersect/ComposeIntersectFst.cc | 82 +-
.../compose_intersect/ComposeIntersectFst.h | 24 +-
.../compose_intersect/ComposeIntersectLexicon.cc | 64 +-
.../compose_intersect/ComposeIntersectLexicon.h | 14 +-
.../compose_intersect/ComposeIntersectRule.cc | 22 +-
.../compose_intersect/ComposeIntersectRule.h | 12 +-
.../compose_intersect/ComposeIntersectRulePair.cc | 45 +-
.../compose_intersect/ComposeIntersectRulePair.h | 14 +-
.../compose_intersect/ComposeIntersectUtilities.cc | 16 +-
.../compose_intersect/ComposeIntersectUtilities.h | 20 +-
.../implementations/optimized-lookup/convert.cc | 42 +-
.../src/implementations/optimized-lookup/convert.h | 73 +-
.../optimized-lookup/find_epsilon_loops.cc | 16 +-
.../src/implementations/optimized-lookup/ospell.cc | 12 +-
.../src/implementations/optimized-lookup/pmatch.cc | 589 +-
.../src/implementations/optimized-lookup/pmatch.h | 116 +-
.../implementations/optimized-lookup/transducer.cc | 104 +-
.../implementations/optimized-lookup/transducer.h | 80 +-
.../src => libhfst/src/parsers}/HfstTwolcDefs.h | 8 +-
libhfst/src/parsers/LexcCompiler.cc | 368 +-
libhfst/src/parsers/LexcCompiler.h | 18 +-
libhfst/src/parsers/Makefile.am | 188 +-
libhfst/src/parsers/PmatchCompiler.cc | 21 +-
libhfst/src/parsers/PmatchCompiler.h | 18 +-
.../src/parsers/SfstAlphabet.cc | 70 +-
.../src/parsers/SfstAlphabet.h | 16 +-
.../src/parsers/SfstBasic.cc | 10 +-
.../HfstBasic.h => libhfst/src/parsers/SfstBasic.h | 10 +-
.../src/parsers/SfstCompiler.cc | 266 +-
.../src/parsers/SfstCompiler.h | 37 +-
.../HfstUtf8.cc => libhfst/src/parsers/SfstUtf8.cc | 7 +-
.../HfstUtf8.h => libhfst/src/parsers/SfstUtf8.h | 8 +-
libhfst/src/parsers/TwolcCompiler.cc | 116 +
.../src/parsers/TwolcCompiler.h | 30 +-
libhfst/src/parsers/XfstCompiler.cc | 799 +-
libhfst/src/parsers/XfstCompiler.h | 57 +-
libhfst/src/parsers/XreCompiler.cc | 146 +-
libhfst/src/parsers/XreCompiler.h | 27 +-
.../src/parsers}/alphabet_src/Alphabet.cc | 48 +-
.../src/parsers}/alphabet_src/Alphabet.h | 0
.../src/parsers}/commandline_src/CommandLine.cc | 100 +-
.../src/parsers}/commandline_src/CommandLine.h | 8 +-
.../src => libhfst/src/parsers}/common_globals.h | 0
.../src => libhfst/src/parsers}/grammar_defs.h | 8 +-
.../src/parsers/htwolcpre1-lexer.ll | 293 +-
.../src/parsers/htwolcpre1-parser.yy | 297 +-
libhfst/src/parsers/htwolcpre2-lexer.ll | 313 +
.../src/parsers/htwolcpre2-parser.yy | 186 +-
.../src/parsers/htwolcpre3-lexer.ll | 59 +-
.../src/parsers/htwolcpre3-parser.yy | 271 +-
.../src/parsers}/io_src/InputReader.cc | 40 +-
.../src/parsers}/io_src/InputReader.h | 16 +-
.../src/parsers}/io_src/input_defs.h | 2 +-
libhfst/src/parsers/lexc-lexer.ll | 22 +-
libhfst/src/parsers/lexc-parser.yy | 37 +-
libhfst/src/parsers/lexc-utils.cc | 44 +-
libhfst/src/parsers/lexc-utils.h | 16 +-
libhfst/src/parsers/pmatch_lex.ll | 68 +-
libhfst/src/parsers/pmatch_parse.yy | 249 +-
libhfst/src/parsers/pmatch_utils.cc | 903 ++-
libhfst/src/parsers/pmatch_utils.h | 180 +-
.../rule_src/ConflictResolvingLeftArrowRule.cc | 4 +-
.../rule_src/ConflictResolvingLeftArrowRule.h | 6 +-
.../rule_src/ConflictResolvingRightArrowRule.cc | 6 +-
.../rule_src/ConflictResolvingRightArrowRule.h | 0
.../src/parsers}/rule_src/LeftArrowRule.cc | 16 +-
.../src/parsers}/rule_src/LeftArrowRule.h | 0
.../parsers}/rule_src/LeftArrowRuleContainer.cc | 30 +-
.../src/parsers}/rule_src/LeftArrowRuleContainer.h | 2 +-
.../parsers}/rule_src/LeftRestrictionArrowRule.cc | 14 +-
.../parsers}/rule_src/LeftRestrictionArrowRule.h | 0
.../src/parsers}/rule_src/OtherSymbolTransducer.cc | 178 +-
.../src/parsers}/rule_src/OtherSymbolTransducer.h | 44 +-
.../src/parsers}/rule_src/RightArrowRule.cc | 14 +-
.../src/parsers}/rule_src/RightArrowRule.h | 0
.../parsers}/rule_src/RightArrowRuleContainer.cc | 18 +-
.../parsers}/rule_src/RightArrowRuleContainer.h | 2 +-
.../src => libhfst/src/parsers}/rule_src/Rule.cc | 20 +-
.../src => libhfst/src/parsers}/rule_src/Rule.h | 6 +-
.../src/parsers}/rule_src/RuleContainer.cc | 12 +-
.../src/parsers}/rule_src/RuleContainer.h | 0
.../src/parsers}/rule_src/TwolCGrammar.cc | 38 +-
.../src/parsers}/rule_src/TwolCGrammar.h | 18 +-
libhfst/src/parsers/sfst-compiler.yy | 225 +
libhfst/src/parsers/sfst-scanner.ll | 202 +
.../src/parsers}/string_src/string_manipulation.cc | 50 +-
.../src/parsers}/string_src/string_manipulation.h | 26 +-
.../parsers}/variable_src/ConstContainerIterator.h | 26 +-
.../variable_src/MatchedConstContainerIterator.h | 36 +-
.../variable_src/MixedConstContainerIterator.h | 30 +-
.../src/parsers}/variable_src/RuleSymbolVector.cc | 12 +-
.../src/parsers}/variable_src/RuleSymbolVector.h | 6 +-
.../src/parsers}/variable_src/RuleVariables.cc | 2 +-
.../src/parsers}/variable_src/RuleVariables.h | 0
.../variable_src/RuleVariablesConstIterator.cc | 6 +-
.../variable_src/RuleVariablesConstIterator.h | 4 +-
.../src/parsers}/variable_src/VariableBlock.h | 20 +-
.../parsers}/variable_src/VariableBlockContainer.h | 18 +-
.../src/parsers}/variable_src/VariableContainer.h | 6 +-
.../parsers}/variable_src/VariableContainerBase.h | 2 +-
.../src/parsers}/variable_src/VariableDefs.h | 0
.../parsers}/variable_src/VariableValueIterator.h | 4 +-
.../src/parsers}/variable_src/VariableValues.cc | 0
.../src/parsers}/variable_src/VariableValues.h | 0
libhfst/src/parsers/xfst-lexer.ll | 68 +-
libhfst/src/parsers/xfst-parser.yy | 156 +-
libhfst/src/parsers/xfst-utils.cc | 42 +-
libhfst/src/parsers/xfst-utils.h | 20 +-
libhfst/src/parsers/xfst_help_message.cc | 50 +-
libhfst/src/parsers/xfst_help_message.h | 30 +-
libhfst/src/parsers/xre_lex.ll | 86 +-
libhfst/src/parsers/xre_parse.yy | 292 +-
libhfst/src/parsers/xre_utils.cc | 224 +-
libhfst/src/parsers/xre_utils.h | 24 +-
m4/ax_check_compile_flag.m4 | 74 +
man/Makefile.am | 79 +-
man/hfst-affix-guessify.1 | 18 +-
man/hfst-apertium-proc.1 | 12 +-
man/hfst-calculate.1 | 22 +-
man/hfst-compare.1 | 16 +-
man/hfst-compose-intersect.1 | 18 +-
man/hfst-compose.1 | 22 +-
man/hfst-concatenate.1 | 18 +-
man/hfst-conjunct.1 | 18 +-
man/hfst-determinize.1 | 14 +-
man/hfst-disjunct.1 | 18 +-
man/hfst-edit-metadata.1 | 20 +-
man/hfst-expand-equivalences.1 | 20 +-
man/hfst-format.1 | 14 +-
man/hfst-fst2fst.1 | 16 +-
man/hfst-fst2strings.1 | 36 +-
man/hfst-fst2txt.1 | 16 +-
man/hfst-grep.1 | 48 +-
man/hfst-guess.1 | 14 +-
man/hfst-guessify.1 | 14 +-
man/hfst-head.1 | 16 +-
man/hfst-info.1 | 18 +-
man/hfst-invert.1 | 14 +-
man/hfst-lexc-compiler.1 | 57 -
man/hfst-lexc.1 | 22 +-
man/hfst-lookup.1 | 58 +-
man/hfst-minimize.1 | 14 +-
man/hfst-multiply.1 | 16 +-
man/hfst-name.1 | 18 +-
man/hfst-optimized-lookup.1 | 22 +-
man/hfst-pair-test.1 | 26 +-
man/hfst-pmatch.1 | 37 +-
man/hfst-pmatch2fst.1 | 16 +-
man/hfst-project.1 | 16 +-
man/hfst-prune-alphabet.1 | 14 +-
man/hfst-push-weights.1 | 16 +-
man/hfst-regexp2fst.1 | 29 +-
man/hfst-remove-epsilons.1 | 14 +-
man/hfst-repeat.1 | 18 +-
man/hfst-reverse.1 | 14 +-
man/hfst-reweight-tagger.1 | 14 +-
man/hfst-reweight.1 | 35 +-
man/hfst-shuffle.1 | 18 +-
man/hfst-split.1 | 16 +-
man/hfst-strings2fst.1 | 26 +-
man/hfst-substitute.1 | 28 +-
man/hfst-subtract.1 | 18 +-
man/hfst-summarize.1 | 16 +-
man/hfst-tag.1 | 14 +-
man/hfst-tail.1 | 16 +-
man/hfst-tokenize.1 | 52 +-
man/hfst-tool.1.template | 2 +-
man/hfst-train-tagger-system.1 | 1 -
man/hfst-traverse.1 | 14 +-
man/hfst-twolc-system.1 | 1 -
man/hfst-twolc.1 | 16 +-
man/hfst-txt2fst.1 | 18 +-
man/hfst-xfst.1 | 24 +-
python/Makefile.am | 20 +-
python/README | 121 +-
python/config.h | 0
python/doc/Doxyfile | 4 +-
python/doc/README | 17 +
python/doc/{libhfst.py => hfst/__init__.py} | 2259 +++---
python/doc/hfst/exceptions/__init__.py | 300 +
python/doc/hfst/sfst_rules/__init__.py | 242 +
python/doc/hfst/xerox_rules/__init__.py | 98 +
python/doc/xml2docstrings.sh | 35 +
python/docstrings.i | 3450 +++++++++
python/hfst/__init__.py | 1261 ++++
python/hfst/exceptions/__init__.py | 66 +
python/hfst/sfst_rules/__init__.py | 28 +
python/hfst/xerox_rules/__init__.py | 35 +
python/hfst_extensions.cc | 38 +
python/hfst_file_extensions.cc | 44 +
python/hfst_lexc_extensions.cc | 57 +
python/hfst_lookup_extensions.cc | 95 +
python/hfst_pmatch_extensions.cc | 93 +
python/hfst_prolog_extensions.cc | 28 +
python/hfst_regex_extensions.cc | 34 +
python/hfst_rules_extensions.cc | 174 +
python/hfst_sfst_extensions.cc | 45 +
python/hfst_xfst_extensions.cc | 68 +
python/libhfst.i | 2366 +++---
python/pypi/COPYING | 674 ++
python/pypi/MANIFEST.in | 22 +
python/pypi/README | 0
python/pypi/README.rst | 116 +
python/pypi/config.h | 0
python/pypi/copy-files.sh | 17 +
python/pypi/setup.cfg | 5 +
python/pypi/setup.py | 234 +
python/setup.py | 22 +-
python/test/Makefile.am | 10 +-
python/test/README | 18 +-
python/test/cat.prolog | 5 +
python/test/cats_and_dogs.prolog | 20 +
python/test/cats_and_dogs_fail.prolog | 31 +
python/test/examples.py | 388 -
python/test/foo_att_prolog | 3 -
python/test/foo_basic | 8 +-
python/test/foobar.hfst | Bin 768 -> 0 bytes
python/test/foobar2.hfst | Bin 512 -> 0 bytes
python/test/streets.txt | 6 +
python/test/test.bat | 17 +
python/test/test.py | 6 -
python/test/test.sfstpl | 1 +
python/test/test.sh | 105 +-
python/test/test_att_reader.py | 25 +
python/test/test_dir_hfst.py | 35 +
python/test/test_dir_hfst_exceptions.py | 18 +
python/test/test_dir_hfst_sfst_rules.py | 13 +
python/test/test_examples.py | 405 +
python/test/test_exceptions.py | 49 +
python/test/test_hfst.py | 353 +-
python/test/test_pmatch.py | 44 +
python/test/test_prolog.py | 65 +
python/test/test_prolog_reader.py | 25 +
python/test/test_read_att_transducer.py | 18 +
python/test/test_streams_1.py | 27 +-
python/test/test_streams_2.py | 25 +-
python/test/test_streams_3.py | 38 +-
python/test/test_tokenizer.py | 67 +
python/test/test_xerox_rules.py | 49 +
python/test/test_xre.py | 22 +
python/test/testfile.att | 12 +-
python/test/testfile1.hfst | Bin 1364 -> 0 bytes
python/test/testfile3.att | 11 -
python/test/testfile_fail.att | 15 +
scripts/README | 14 +-
scripts/copy-for-windows.sh | 236 +-
scripts/fetch-stats.sh | 34 +
scripts/generate-cc-files.sh | 12 +-
scripts/generate-static-binaries.sh | 4 +-
scripts/hfst-foma-wrapper.sh.in | 5 +-
scripts/hfst-foma.in | 3 +
scripts/make-libhfst-dist.sh | 43 +
scripts/make-python-package.sh | 17 +
scripts/readmes/README_eight_tools_mac.txt | 4 +-
scripts/readmes/README_eight_tools_win.txt | 6 +-
scripts/readmes/README_python_mac.txt | 27 +
scripts/readmes/README_python_win.txt | 27 +
scripts/readmes/README_xfst_mac.txt | 4 +-
scripts/readmes/README_xfst_win.txt | 6 +-
scripts/windows/make-hfst-lexc.bat | 3 +-
scripts/windows/make-hfst-proc.bat | 3 +-
scripts/windows/make-hfst-tool.bat | 41 +-
scripts/windows/make-hfst-xfst.bat | 3 +-
scripts/windows/make-htwolcpre1.bat | 3 +-
scripts/windows/make-htwolcpre2.bat | 3 +-
scripts/windows/make-htwolcpre3.bat | 3 +-
scripts/windows/make-python-bindings.bat | 8 +-
scripts/windows/make-python-package.bat | 18 +
test/Makefile.am | 7 +-
test/libhfst/Makefile.am | 2 +-
test/libhfst/auxiliary_functions.cc | 6 +-
test/libhfst/test_constructors.cc | 12 +-
test/libhfst/test_examples.cc | 34 +-
test/libhfst/test_flag_diacritics.cc | 14 +-
test/libhfst/test_hfst_basic_transducer.cc | 16 +-
test/libhfst/test_lexc.cc | 16 +-
test/libhfst/test_rules.cc | 46 +-
test/libhfst/test_streams.cc | 16 +-
test/libhfst/test_tokenizer.cc | 42 +-
test/libhfst/test_transducer_functions.cc | 116 +-
test/tools/Makefile.am | 169 +-
test/tools/basic.UTF-8.lexc.flag.result | Bin 1229 -> 0 bytes
test/tools/basic.UTF-8.lexc.flag.result.prolog | 26 +
test/tools/basic.UTF-8.lexc.result | 44 -
test/tools/basic.UTF-8.lexc.result.prolog | 22 +
.../basic.almost-reserved-words.lexc.flag.result | Bin 847 -> 0 bytes
...c.almost-reserved-words.lexc.flag.result.prolog | 49 +
test/tools/basic.almost-reserved-words.lexc.result | Bin 725 -> 0 bytes
.../basic.almost-reserved-words.lexc.result.prolog | 45 +
test/tools/basic.cat-dog-bird.lexc.flag.result | Bin 886 -> 0 bytes
.../basic.cat-dog-bird.lexc.flag.result.prolog | 16 +
test/tools/basic.cat-dog-bird.lexc.result | 27 -
test/tools/basic.cat-dog-bird.lexc.result.prolog | 12 +
test/tools/basic.colons.lexc.flag.result | Bin 1224 -> 0 bytes
test/tools/basic.colons.lexc.flag.result.prolog | 29 +
test/tools/basic.colons.lexc.result | 43 -
test/tools/basic.colons.lexc.result.prolog | 25 +
test/tools/basic.comments.lexc.flag.result | Bin 1453 -> 0 bytes
test/tools/basic.comments.lexc.flag.result.prolog | 31 +
test/tools/basic.comments.lexc.result | 41 -
test/tools/basic.comments.lexc.result.prolog | 23 +
test/tools/basic.empty-sides.lexc.flag.result | Bin 1225 -> 0 bytes
.../basic.empty-sides.lexc.flag.result.prolog | 25 +
test/tools/basic.empty-sides.lexc.result | 33 -
test/tools/basic.empty-sides.lexc.result.prolog | 17 +
test/tools/basic.escapes.lexc.flag.result | Bin 771 -> 0 bytes
test/tools/basic.escapes.lexc.flag.result.prolog | 13 +
test/tools/basic.escapes.lexc.result | 22 -
test/tools/basic.escapes.lexc.result.prolog | 9 +
test/tools/basic.infostrings.lexc.flag.result | Bin 1026 -> 0 bytes
.../basic.infostrings.lexc.flag.result.prolog | 20 +
test/tools/basic.infostrings.lexc.result | 28 -
test/tools/basic.infostrings.lexc.result.prolog | 14 +
.../basic.initial-lexicon-empty.lexc.flag.result | Bin 760 -> 0 bytes
...c.initial-lexicon-empty.lexc.flag.result.prolog | 11 +
test/tools/basic.initial-lexicon-empty.lexc.result | 14 -
.../basic.initial-lexicon-empty.lexc.result.prolog | 5 +
.../basic.lowercase-lexicon-end.lexc.flag.result | Bin 2266 -> 0 bytes
...c.lowercase-lexicon-end.lexc.flag.result.prolog | 46 +
test/tools/basic.lowercase-lexicon-end.lexc.result | 54 -
.../basic.lowercase-lexicon-end.lexc.result.prolog | 29 +
.../tools/basic.multi-entry-lines.lexc.flag.result | Bin 784 -> 0 bytes
...basic.multi-entry-lines.lexc.flag.result.prolog | 14 +
test/tools/basic.multi-entry-lines.lexc.result | 21 -
.../basic.multi-entry-lines.lexc.result.prolog | 10 +
.../basic.multichar-escaped-zero.lexc.flag.result | Bin 785 -> 0 bytes
....multichar-escaped-zero.lexc.flag.result.prolog | 13 +
.../tools/basic.multichar-escaped-zero.lexc.result | 16 -
...basic.multichar-escaped-zero.lexc.result.prolog | 7 +
.../tools/basic.multichar-symbols.lexc.flag.result | Bin 840 -> 0 bytes
...basic.multichar-symbols.lexc.flag.result.prolog | 14 +
test/tools/basic.multichar-symbols.lexc.result | 26 -
.../basic.multichar-symbols.lexc.result.prolog | 10 +
test/tools/basic.no-Root.lexc.flag.result | Bin 881 -> 0 bytes
test/tools/basic.no-Root.lexc.flag.result.prolog | 16 +
test/tools/basic.no-Root.lexc.result | 27 -
test/tools/basic.no-Root.lexc.result.prolog | 12 +
.../tools/basic.no-newline-at-end.lexc.flag.result | Bin 641 -> 0 bytes
...basic.no-newline-at-end.lexc.flag.result.prolog | 9 +
test/tools/basic.no-newline-at-end.lexc.result | 14 -
.../basic.no-newline-at-end.lexc.result.prolog | 5 +
test/tools/basic.punctuation.lexc.flag.result | Bin 2456 -> 0 bytes
.../basic.punctuation.lexc.flag.result.prolog | 73 +
test/tools/basic.punctuation.lexc.result | 132 -
test/tools/basic.punctuation.lexc.result.prolog | 69 +
test/tools/basic.regexps.lexc.flag.result | Bin 543 -> 0 bytes
test/tools/basic.regexps.lexc.flag.result.prolog | 14 +
test/tools/basic.regexps.lexc.result | Bin 315 -> 0 bytes
test/tools/basic.regexps.lexc.result.prolog | 6 +
test/tools/basic.root-loop.lexc.flag.result | Bin 785 -> 0 bytes
test/tools/basic.root-loop.lexc.flag.result.prolog | 13 +
test/tools/basic.root-loop.lexc.result | 22 -
test/tools/basic.root-loop.lexc.result.prolog | 9 +
test/tools/basic.spurious-lexicon.lexc.flag.result | Bin 844 -> 0 bytes
.../basic.spurious-lexicon.lexc.flag.result.prolog | 14 +
test/tools/basic.spurious-lexicon.lexc.result | 19 -
.../basic.spurious-lexicon.lexc.result.prolog | 8 +
test/tools/basic.string-pairs.lexc.flag.result | Bin 968 -> 0 bytes
.../basic.string-pairs.lexc.flag.result.prolog | 18 +
test/tools/basic.string-pairs.lexc.result | 31 -
test/tools/basic.string-pairs.lexc.result.prolog | 14 +
test/tools/basic.two-lexicons.lexc.flag.result | Bin 1132 -> 0 bytes
.../basic.two-lexicons.lexc.flag.result.prolog | 22 +
test/tools/basic.two-lexicons.lexc.result | 35 -
test/tools/basic.two-lexicons.lexc.result.prolog | 16 +
test/tools/basic.zeros-epsilons.lexc.flag.result | Bin 1733 -> 0 bytes
.../basic.zeros-epsilons.lexc.flag.result.prolog | 46 +
test/tools/basic.zeros-epsilons.lexc.result | 63 -
test/tools/basic.zeros-epsilons.lexc.result.prolog | 42 +
test/tools/cats_and_dogs_semicolon.xre | 2 +-
test/tools/compare-functionality.sh | 2 +
.../fsmbook-tests/hfst-scripts/Lingala.hfst.script | 2 +-
test/tools/fsmbook-tests/test.sh | 5 +
test/tools/hfst-check-environment.sh | 2 +-
test/tools/hfst-check-help.sh | 17 +-
test/tools/hfst-check-version.sh | 17 +-
test/tools/hfst.weights.lexc.flag.result | Bin 938 -> 0 bytes
test/tools/hfst.weights.lexc.flag.result.prolog | 16 +
test/tools/hfst.weights.lexc.result | Bin 610 -> 0 bytes
test/tools/hfst.weights.lexc.result.prolog | 10 +
test/tools/incompatible-formats.sh | 2 +-
test/tools/lexc-compiler-functionality.sh | 10 +-
test/tools/mismatching-input-streams.sh | 2 +-
.../no-newline-before-sublexicon.lexc.flag.result | 31 -
...wline-before-sublexicon.lexc.flag.result.prolog | 11 +
.../tools/no-newline-before-sublexicon.lexc.result | 19 -
...no-newline-before-sublexicon.lexc.result.prolog | 5 +
test/tools/pmatch-functionality.sh | 2 +-
test/tools/pmatch-tests.sh | 13 +
test/tools/proc-caps-gen.strings | 1 +
test/tools/proc-caps-out1.strings | 2 +-
test/tools/proc-caps-out2.strings | 1 +
test/tools/proc-caps.txt | 39 +-
test/tools/symbol-harmonization-functionality.sh | 6 +-
test/tools/tokenize-backtrack-functionality.sh | 56 +
...nize-backtrack-out-giella-cg-contiguous.strings | 9 +
...tokenize-backtrack-out-giella-cg-spaces.strings | 14 +
.../tools/tokenize-backtrack-out-giella-cg.strings | 21 +
test/tools/tokenize-backtrack.lexc | 65 +
test/tools/tokenize-backtrack.pmscript | 16 +
test/tools/tokenize-dog-in.lexc | 32 +
test/tools/tokenize-dog-out-cg.strings | 13 +
.../tokenize-dog-out-giella-cg-flushing.strings | 13 +
.../tokenize-dog-out-giella-cg-superblank.strings | 12 +
test/tools/tokenize-dog-out-giella-cg.strings | 15 +
test/tools/tokenize-dog-out-xerox.strings | 9 +
test/tools/tokenize-dog-out.strings | 4 +
test/tools/tokenize-dog.pmscript | 16 +
test/tools/tokenize-flushing-functionality.sh | 71 +
test/tools/tokenize-functionality.sh | 80 +
test/tools/warn.one-sided-flags.lexc.flag.result | Bin 461 -> 0 bytes
.../warn.one-sided-flags.lexc.flag.result.prolog | 11 +
test/tools/warn.one-sided-flags.lexc.result | Bin 347 -> 0 bytes
test/tools/warn.one-sided-flags.lexc.result.prolog | 7 +
...icon-mentioned-but-not-defined.lexc.flag.result | 19 -
...ntioned-but-not-defined.lexc.flag.result.prolog | 6 +
...ublexicon-mentioned-but-not-defined.lexc.result | 11 -
...on-mentioned-but-not-defined.lexc.result.prolog | 2 +
test/tools/xre.any-variations.lexc.flag.result | Bin 1971 -> 0 bytes
.../xre.any-variations.lexc.flag.result.prolog | 87 +
test/tools/xre.any-variations.lexc.result | Bin 1705 -> 0 bytes
test/tools/xre.any-variations.lexc.result.prolog | 83 +
...re.automatic-multichar-symbols.lexc.flag.result | Bin 1631 -> 0 bytes
...matic-multichar-symbols.lexc.flag.result.prolog | 41 +
.../xre.automatic-multichar-symbols.lexc.result | 62 -
....automatic-multichar-symbols.lexc.result.prolog | 33 +
test/tools/xre.basic.lexc.flag.result | Bin 1073 -> 0 bytes
test/tools/xre.basic.lexc.flag.result.prolog | 22 +
test/tools/xre.basic.lexc.result | 38 -
test/tools/xre.basic.lexc.result.prolog | 18 +
test/tools/xre.definitions.lexc.flag.result | Bin 1915 -> 0 bytes
test/tools/xre.definitions.lexc.flag.result.prolog | 67 +
test/tools/xre.definitions.lexc.result | 98 -
test/tools/xre.definitions.lexc.result.prolog | 63 +
test/tools/xre.less-than-twice.lexc | 1 +
.../xre.less-than-twice.lexc.flag.result.prolog | 9 +
test/tools/xre.less-than-twice.lexc.result.prolog | 5 +
test/tools/xre.months.lexc.flag.result | Bin 1998 -> 0 bytes
test/tools/xre.months.lexc.flag.result.prolog | 52 +
test/tools/xre.months.lexc.result | 80 -
test/tools/xre.months.lexc.result.prolog | 48 +
test/tools/xre.more-than-twice.lexc | 1 +
.../xre.more-than-twice.lexc.flag.result.prolog | 14 +
test/tools/xre.more-than-twice.lexc.result.prolog | 10 +
test/tools/xre.nested-definitions.lexc.flag.result | Bin 1248 -> 0 bytes
.../xre.nested-definitions.lexc.flag.result.prolog | 39 +
test/tools/xre.nested-definitions.lexc.result | 53 -
.../xre.nested-definitions.lexc.result.prolog | 35 +
test/tools/xre.numeric-star.lexc.flag.result | Bin 801 -> 0 bytes
.../tools/xre.numeric-star.lexc.flag.result.prolog | 16 +
test/tools/xre.numeric-star.lexc.result | 28 -
test/tools/xre.numeric-star.lexc.result.prolog | 12 +
test/tools/xre.quotations.lexc.flag.result | Bin 908 -> 0 bytes
test/tools/xre.quotations.lexc.flag.result.prolog | 15 +
test/tools/xre.quotations.lexc.result | 27 -
test/tools/xre.quotations.lexc.result.prolog | 11 +
test/tools/xre.sharp.lexc.flag.result | Bin 1360 -> 0 bytes
test/tools/xre.sharp.lexc.flag.result.prolog | 35 +
test/tools/xre.sharp.lexc.result | 67 -
test/tools/xre.sharp.lexc.result.prolog | 31 +
test/tools/xre.star-plus-optional.lexc.flag.result | Bin 1064 -> 0 bytes
.../xre.star-plus-optional.lexc.flag.result.prolog | 23 +
test/tools/xre.star-plus-optional.lexc.result | 36 -
.../xre.star-plus-optional.lexc.result.prolog | 19 +
tools/src/FunctionalTransducer.cc | 10 +-
tools/src/FunctionalTransducer.h | 2 +-
tools/src/Makefile.am | 29 +-
tools/src/generate_model_forms.cc | 28 +-
tools/src/guessify_fst.cc | 16 +-
tools/src/guessify_fst.h | 2 +-
tools/src/hfst-affix-guessify.cc | 12 +-
tools/src/hfst-binary-tool.cc | 8 +-
tools/src/hfst-commandline.cc | 25 +-
tools/src/hfst-commandline.h | 24 +-
tools/src/hfst-compare.cc | 18 +-
tools/src/hfst-compiler.yy | 494 --
tools/src/hfst-compose-intersect.cc | 48 +-
tools/src/hfst-compose.cc | 30 +-
tools/src/hfst-concatenate.cc | 20 +-
tools/src/hfst-conjunct.cc | 20 +-
tools/src/hfst-determinize.cc | 12 +-
tools/src/hfst-disjunct.cc | 14 +-
tools/src/hfst-edit-metadata.cc | 16 +-
tools/src/hfst-expand-equivalences.cc | 14 +-
tools/src/hfst-file-to-mem.cc | 12 +-
tools/src/hfst-flookup.cc | 86 +-
tools/src/hfst-format.cc | 12 +-
tools/src/hfst-fst2fst.cc | 62 +-
tools/src/hfst-fst2strings.cc | 68 +-
tools/src/hfst-fst2txt.cc | 40 +-
tools/src/hfst-getopt.cc | 2 +-
tools/src/hfst-getopt.h | 2 +-
tools/src/hfst-grep.cc | 24 +-
tools/src/hfst-guess.cc | 50 +-
tools/src/hfst-guessify.cc | 34 +-
tools/src/hfst-head.cc | 12 +-
tools/src/hfst-info.cc | 20 +-
tools/src/hfst-invert.cc | 12 +-
tools/src/hfst-lexc-compiler.cc | 4 +-
tools/src/hfst-lexc-wrapper.cc | 10 +-
tools/src/hfst-lookup.cc | 540 +-
tools/src/hfst-minimize.cc | 12 +-
tools/src/hfst-multiply.cc | 12 +-
tools/src/hfst-name.cc | 12 +-
tools/src/hfst-optimized-lookup.cc | 60 +-
tools/src/hfst-optimized-lookup.h | 26 +-
tools/src/hfst-pair-test-commandline.h | 22 +-
tools/src/hfst-pair-test-transducers.h | 70 +-
tools/src/hfst-pair-test.cc | 88 +-
tools/src/hfst-pmatch.cc | 136 +-
tools/src/hfst-pmatch2fst.cc | 83 +-
.../hfst-preprocess-for-optimized-lookup-format.cc | 18 +-
tools/src/hfst-proc/Makefile.am | 9 -
tools/src/hfst-proc/alphabet.cc | 20 +-
tools/src/hfst-proc/alphabet.h | 24 +-
tools/src/hfst-proc/applicators.cc | 137 +-
tools/src/hfst-proc/applicators.h | 10 +-
tools/src/hfst-proc/buffer.h | 10 +-
tools/src/hfst-proc/formatter.cc | 72 +
tools/src/hfst-proc/formatter.h | 22 +-
tools/src/hfst-proc/hfst-proc.cc | 21 +-
tools/src/hfst-proc/lookup-path.h | 4 +-
tools/src/hfst-proc/lookup-state.cc | 8 +-
tools/src/hfst-proc/lookup-state.h | 2 +-
tools/src/hfst-proc/tokenizer.cc | 46 +-
tools/src/hfst-proc/tokenizer.h | 71 +-
tools/src/hfst-program-options.cc | 14 +-
tools/src/hfst-program-options.h | 2 +-
tools/src/hfst-project.cc | 10 +-
tools/src/hfst-prune-alphabet.cc | 8 +-
tools/src/hfst-push-weights.cc | 10 +-
tools/src/hfst-regexp2fst.cc | 61 +-
tools/src/hfst-remove-epsilons.cc | 12 +-
tools/src/hfst-repeat.cc | 24 +-
tools/src/hfst-reverse.cc | 12 +-
tools/src/hfst-reweight.cc | 22 +-
tools/src/hfst-scanner.ll | 194 -
tools/src/hfst-shuffle.cc | 14 +-
tools/src/hfst-split.cc | 14 +-
tools/src/hfst-strings2fst.cc | 36 +-
tools/src/hfst-strip-header.cc | 6 +-
tools/src/hfst-substitute.cc | 74 +-
tools/src/hfst-subtract.cc | 20 +-
tools/src/hfst-summarize.cc | 46 +-
tools/src/hfst-tagger/src/Makefile.am | 57 +-
.../hfst-tagger/src/build_model_src/FstBuilder.cc | 38 +-
.../hfst-tagger/src/build_model_src/FstBuilder.h | 2 +-
.../src/build_model_src/LexicalModelBuilder.cc | 2 +-
tools/src/hfst-tagger/src/build_model_src/Makefile | 2 +-
.../src/build_model_src/ModelBuilder.cc | 22 +-
.../src/build_model_src/SequenceModelBuilder.cc | 6 +-
.../src/build_model_src/TaggerBuilder.cc | 4 +-
.../src/build_model_src/TaggerBuilder.h | 2 +-
.../WeightedStringVectorCollection.cc | 26 +-
.../WeightedStringVectorCollection.h | 10 +-
.../src/build_model_src/string_handling.cc | 163 +-
.../src/build_model_src/string_handling.h | 6 +-
.../hfst-tagger/src/build_model_src/tagger_aux.py | 20 +-
tools/src/hfst-tagger/src/hfst-build-tagger.cc | 8 +-
.../src/hfst-open-input-file-for-tagger.cc | 12 +-
tools/src/hfst-tagger/src/hfst-reweight-tagger.cc | 6 +-
tools/src/hfst-tagger/src/hfst-tag.cc | 6 +-
.../hfst-tagger/src/hfst-train-tagger-system.cc | 10 +-
.../src/hfst_tagger_compute_data_statistics.py | 16 +-
.../src/use_model_src/AcyclicAutomaton.cc | 6 +-
.../src/use_model_src/AcyclicAutomaton.h | 2 +-
.../use_model_src/DelayedSequenceModelComponent.cc | 10 +-
.../use_model_src/DelayedSequenceModelComponent.h | 2 +-
.../src/use_model_src/NewLexicalModel.cc | 87 +-
.../src/use_model_src/NewLexicalModel.h | 14 +-
.../src/use_model_src/SentenceTagger.cc | 22 +-
.../src/use_model_src/SentenceTransducer.cc | 28 +-
.../src/use_model_src/SequenceModelComponent.cc | 60 +-
.../src/use_model_src/SequenceModelComponent.h | 2 +-
.../use_model_src/SequenceModelComponentPair.cc | 22 +-
.../src/use_model_src/SequenceModelComponentPair.h | 2 +-
.../src/use_model_src/SequenceTagger.cc | 28 +-
.../hfst-tagger/src/use_model_src/SequenceTagger.h | 4 +-
tools/src/hfst-tail.cc | 8 +-
tools/src/hfst-tokenize.cc | 830 +-
tools/src/hfst-tool-metadata.cc | 21 +-
tools/src/hfst-tool-metadata.h | 4 +-
tools/src/hfst-traverse.cc | 22 +-
tools/src/hfst-twolc/src/Makefile.am | 231 +-
.../src/doxygen_src/hfst_twolc_doxygen_header.h | 32 +-
tools/src/hfst-twolc/src/hfst-twolc-system.cc | 8 +-
tools/src/hfst-twolc/src/hfst-twolc.cc | 164 +
tools/src/hfst-twolc/src/htwolcpre1-main.cc | 70 +
tools/src/hfst-twolc/src/htwolcpre2-main.cc | 49 +
tools/src/hfst-twolc/src/htwolcpre3-main.cc | 102 +
tools/src/hfst-twolc/src/scanner2.ll | 294 -
tools/src/hfst-twolc/test/test | 114 +-
tools/src/hfst-txt2fst.cc | 22 +-
tools/src/hfst-unary-tool.cc | 12 +-
tools/src/inc/check-params-unary.h | 10 +-
tools/src/inc/getopt-cases-common.h | 4 +-
tools/src/inc/getopt-cases-unary.h | 4 +-
tools/src/inc/globals-common.h | 4 +-
tools/src/lexc-readline-ui.cc | 2 +-
tools/src/parsers/Makefile.am | 32 +-
tools/src/parsers/abbrcmd.h | 5 +-
tools/src/parsers/cmd.h | 5 +-
tools/src/parsers/foma_commands | 16 +-
tools/src/parsers/foma_commands.help | 2 +-
tools/src/parsers/hfst-xfst.cc | 29 +-
tools/src/parsers/name2cmd.h | 2 +-
tools/src/parsers/test/Makefile.am | 9 +-
tools/src/parsers/test/eliminate_flag.output | 5 +
tools/src/parsers/test/eliminate_flag.xfst | 45 +
tools/src/parsers/test/test.sh | 43 +-
tools/src/parsers/xfst_commands.h | 2 +-
tools/src/sfst-main.cc | 248 +
tools/src/test-memory-leaks.sh | 4 +-
tools/src/test.cc | 30 +-
773 files changed, 41119 insertions(+), 26918 deletions(-)
diff --git a/.gitignore b/.gitignore
index 4033fe6..197eb1b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,17 +29,44 @@
*.sfst
*.foma
*.hfstol
+*.genhfstol
# Generated Python files and folders
/python/__pycache__/
/check_installation/__pycache__/
/python/_libhfst.i
/python/build/
+/python/dist/
/python/doc/html/
/python/libhfst.py
/python/libhfst_wrap.cpp
+/python/hfst.egg-info/
+/python/hfstpy.egg-info/
+/python/libhfst_swig.egg-info/
*.pyc
+# Generated and copied python pypi files and folders
+/python/pypi/back-ends/
+/python/pypi/build/
+/python/pypi/dist/
+/python/pypi/docstrings.i
+/python/pypi/hfst.egg-info/
+/python/pypi/hfst/
+/python/pypi/hfst_extensions.cc
+/python/pypi/hfst_file_extensions.cc
+/python/pypi/hfst_lexc_extensions.cc
+/python/pypi/hfst_lookup_extensions.cc
+/python/pypi/hfst_pmatch_extensions.cc
+/python/pypi/hfst_prolog_extensions.cc
+/python/pypi/hfst_regex_extensions.cc
+/python/pypi/hfst_rules_extensions.cc
+/python/pypi/hfst_xfst_extensions.cc
+/python/pypi/hfst_sfst_extensions.cc
+/python/pypi/libhfst.i
+/python/pypi/libhfst.py
+/python/pypi/libhfst/
+/python/pypi/libhfst_wrap.cpp
+
# Back-up files
*~
@@ -75,6 +102,7 @@ hfst-*.tar.gz
/libhfst/src/implementations/FomaTransducer
/libhfst/src/implementations/HfstOlTransducer
/libhfst/src/implementations/HfstTransitionGraph
+/libhfst/src/implementations/HfstBasicTransducer
/libhfst/src/implementations/LogWeightTransducer
/libhfst/src/implementations/SfstTransducer
/libhfst/src/implementations/TropicalWeightTransducer
@@ -87,6 +115,20 @@ hfst-*.tar.gz
/libhfst/src/implementations/optimized-lookup/transducer
/libhfst/src/parsers/LexcCompiler
/libhfst/src/parsers/XreCompiler
+/libhfst/src/parsers/Alphabet
+/libhfst/src/parsers/ConflictResolvingLeftArrowRule
+/libhfst/src/parsers/ConflictResolvingRightArrowRule
+/libhfst/src/parsers/InputReader
+/libhfst/src/parsers/LeftArrowRule
+/libhfst/src/parsers/LeftArrowRuleContainer
+/libhfst/src/parsers/LeftRestrictionArrowRule
+/libhfst/src/parsers/OtherSymbolTransducer
+/libhfst/src/parsers/RightArrowRule
+/libhfst/src/parsers/RightArrowRuleContainer
+/libhfst/src/parsers/Rule
+/libhfst/src/parsers/RuleContainer
+/libhfst/src/parsers/TwolCGrammar
+/libhfst/src/parsers/string_manipulation
/tools/src/hfst-tagger/src/AcyclicAutomaton
/tools/src/hfst-tagger/src/DelayedSequenceModelComponent
/tools/src/hfst-tagger/src/FstBuilder
@@ -159,14 +201,16 @@ stamp-h?
/libhfst/src/parsers/pmatch_parse.hh
/libhfst/src/parsers/lexc-parser.hh
/libhfst/src/parsers/xfst-parser.hh
-
-/tools/src/hfst-compiler.cc
-/tools/src/hfst-scanner.cc
-/tools/src/hfst-compiler.hh
-
-/tools/src/hfst-twolc/src/htwolcpre?.cc
-/tools/src/hfst-twolc/src/scanner?.cc
-/tools/src/hfst-twolc/src/htwolcpre?.hh
+/libhfst/src/parsers/htwolcpre1-lexer.cc
+/libhfst/src/parsers/htwolcpre1-parser.cc
+/libhfst/src/parsers/htwolcpre1-parser.hh
+/libhfst/src/parsers/htwolcpre2-lexer.cc
+/libhfst/src/parsers/htwolcpre2-parser.cc
+/libhfst/src/parsers/htwolcpre2-parser.hh
+/libhfst/src/parsers/htwolcpre3-lexer.cc
+/libhfst/src/parsers/htwolcpre3-parser.cc
+/libhfst/src/parsers/htwolcpre3-parser.hh
+
# Otherwise generated code
/libhfst/hfst.pc
@@ -238,14 +282,20 @@ stamp-h?
/tools/src/hfst-tokenize
/tools/src/hfst-traverse
/tools/src/hfst-twolc/src/hfst-twolc
-/tools/src/hfst-twolc/src/hfst-twolc-loc
-/tools/src/hfst-twolc/src/hfst-twolc-system
+/tools/src/hfst-twolc/src/hfst-twolc-script
+/tools/src/hfst-twolc/src/hfst-twolc-local-script
/tools/src/hfst-twolc/src/htwolcpre1
/tools/src/hfst-twolc/src/htwolcpre2
/tools/src/hfst-twolc/src/htwolcpre3
+/tools/src/hfst-twolc/src/htwolc
/tools/src/hfst-txt2fst
/tools/src/parsers/hfst-xfst
+# Files generated by make check
+/test/tools/pmatch_endtag.pmatch
+/libhfst/src/parsers/LexcCompiler_test.lexc
+/libhfst/src/parsers/LexcCompiler_test2.lexc
+
# Aggressive ignores
Makefile
/m4/*
@@ -253,3 +303,4 @@ Makefile
# Add back files ignored by aggressive ignores
!/tools/src/hfst-tagger/src/build_model_src/Makefile
!/m4/README
+sfst-*
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..ecd6d73
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,41 @@
+language: cpp
+
+# let linux be Ubuntu 14.04 Trusty
+sudo: required
+dist: trusty
+
+cache: ccache
+
+before_install:
+ - echo "$TRAVIS_OS_NAME"
+ - if [[ "$TRAVIS_OS_NAME" = "linux" ]];then wget http://apertium.projectjj.com/apt/install-nightly.sh -O - | sudo bash; fi
+ - if [[ "$TRAVIS_OS_NAME" = "linux" ]];then sudo apt-get install -y autopoint dh-autoreconf libpython3-all-dev libreadline-dev libreadline6-dev python3-all python3-all-dev swig swig2.0; fi
+ - if [[ "$TRAVIS_OS_NAME" = "osx" ]];then wget http://apertium.projectjj.com/osx/install-nightly.sh -O - | sudo bash; fi
+ - if [[ "$TRAVIS_OS_NAME" = "osx" ]];then sudo brew update; fi
+ - if [[ "$TRAVIS_OS_NAME" = "osx" ]];then sudo brew install bison flex; fi
+ - if [[ "$TRAVIS_OS_NAME" = "osx" ]];then export "PATH=/usr/local/opt/bison/bin:$PATH"; fi
+
+# TODO: "apt-get build-dep hfst" fails with "Unable to find a source package for hfst";
+# is there a source package in the nightly repo?
+
+script:
+ - ./autogen.sh
+ - export CCACHE_ARGS=()
+ - if command -v ccache >/dev/null 2>/dev/null; then CCACHE_ARGS+=( CC="ccache gcc" ); CCACHE_ARGS+=( CXX="ccache g++" ); fi
+ - echo "${CCACHE_ARGS[@]:-}"
+ - ./configure --enable-xfst --enable-proc --without-foma --enable-lexc --enable-all-tools --enable-silent-rules "${CCACHE_ARGS[@]:-}"
+ - if [[ $CXX = g++ ]]; then make -j2; else make -j; fi
+ - make check
+
+# g++ with -j runs out of memory on travis, thus -j2
+
+notifications:
+ irc:
+ channels:
+ - secure: "FWjUrhqWeVjjJnLQb7CwxxztJgE6OgL5h39MRzF9LKjGML1kd2GEBrJBITzdWhLvwJhkhY1so4zZ73MuxUJGSNOKagvt2qamliGIx/IbgAwlTlwQtR8ya0qdV6ITWMXa5pm86RoFrpnD+45Bitu7R0sPgHNNFImBbWmTKntiIw6xvvHklIsIvxWvMNny/FNEl95ZCTz8SkPeFYyRbdM2krRsKoPefI8kbCAaBdbXgo6zSboGkZinkDCXDJbn9cOeb06nBSeMd9SWzZSyx2DP1L8KhGvCnegwahN6bVYAcWZ5cCYDkB35pW0Of81siJzYMjr7yLggL27eROG3CgCeK8x9jg5rJW2IhZkxECYzAKftQPPALXkBOiEtFABhAdtBa316W5uz+XasIG9COvUSzh2f7s7M1cqkhmLTi3PLP8JcjViyV9PgH54sP+pjGilaXEGkAH++Rtj/0Jc3edTap [...]
+ on_failure: always
+ on_success: never
+# The irc channel is encrypted for hfst/hfst, so build notifications from forks won't show up on the IRC channel
+# Encrypt with:
+# $ gem install --user-install travis
+# $ ~/.gem/ruby/*/bin/travis encrypt -r hfst/hfst 'chat.freenode.net#hfst'
diff --git a/ChangeLog b/ChangeLog
index 7044589..0247c9f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,336 @@
-97c1285 (HEAD -> master, origin/master, origin/HEAD) Swap implementations of commands 'apply up' and 'apply down' in hfst-xfst, making them work as they do in foma and xfst.
+77c845d (HEAD -> master, origin/master, origin/HEAD) Fix flag elimination bug (reported in issue #342).
+c827bb6 Do not call fsm_destroy until issue with 'double free or corruption' has been fixed.
+3b0e58a Provide a get_current_dir_name() for platforms without one
+b0fb7a6 Merge branch 'master' of https://github.com/hfst/hfst
+65a4d79 Search for included files under scriptdir, not working dir. If an included filename is non-absolute, prefix it with the directory the script was in when trying to read it. This only happens in non-Windows environments for now.
+f5e94d7 Free allocated memory.
+6144e75 tokenise -g: check for eof
+cf46e69 Apply more changes from latest release of foma.
+a9f8397 Apply some changes from newest release of foma.
+cc8c96b Make a separate directory python/pypi for creating pypi packages.
+479789e Rename the pypi package to 'hfst' and use a four-number versioning scheme. Also define '__version__'. Add an empty config file as hfst c++ headers include it.
+eb304f0 tests: trying sed instead of awk for exit on <STREAMCMD:FLUSH>
+798383d tokenize: handle Apertium-style superblanks in --giella-cg format
+7922399 Provide a way to scale the vector similarity calculations via variable
+166d418 Merge branch 'master' of https://github.com/hfst/hfst
+9bc5315 parse word2vec -style text format and assume first line is info
+5d3f6b8 Do not allow unescaped dots in regular expressions.
+ce5e2a5 Merge branch 'master' of https://github.com/hfst/hfst
+a8f21f0 Observe HAVE_SFST and HAVE_FOMA when including back-end header files.
+8a9c0de nwords catenate-style argument for Like() function
+1433e45 Merge branch 'master' of https://github.com/hfst/hfst
+1ec8e03 Add experimental two-vector model for word sense
+72d4f72 (tag: v3.12.0) Ready for release 3.12.0.
+da86647 Read vector models and do simple ranking
+42df833 Small fixes to memory leaks and efficiency.
+e2abd89 Fix some memory leaks.
+da54acc Fix leaks reported by valgrind.
+92cf4dc Fix more memory leaks.
+9bd9749 Fix some lost memory issues reported by valgrind.
+6e129a4 Remove unnecessary variable. Delete transducer definitions and latest regexp when destructor is called.
+d700309 flush given outstream, not necessarily cout
+8d4f824 Add a destructor to XfstCompiler.
+6b10aab Update README.
+863e94a Add missing files to dist and remove generated ones.
+e4c1304 Rename hfst.rules into hfst.sfst_rules. Fix hfst.types into hfst.ImplementationType in tests.
+80f18a8 Use single quotes instead of double quotes in docstrings.
+52a9d75 Add test for hfst.xerox_rules.
+00b020c Rename xerox_replace_left and xerox_restriction back to replace_left and restriction in package hfst.xerox_rules.
+d1556dd Fix documentation of parameter 'output' of function 'extract_paths'.
+3df4a48 Sentence separation for finnpos-mode
+d3058c7 Issue a warning if automake is too old, which can make build fail if pre-generated yacc files are modified.
+9166cdc Move ImplementationTypes and ReplaceTypes to their own classes and get rid of package hfst.types.
+6d6c631 Fix datatypes used in for loops.
+c6aa5e1 Fix some typos in documentation.
+8073675 Use renamed versions of xerox rule functions.
+ae70056 Tentatively add HfstXeroxRules to python API in package hfst.xerox_rules.
+dee0176 Fix some more warnings.
+37edbbb Fix minor warnings generated by clang.
+faa7647 Fix typos in ifdef guards.
+41c5a32 Add a simple script for fetching statistics about hfst downloads.
+d9d8077 Update pypi setup.
+822f4bb doh, forgot to add test expected output
+cccba22 hfst-tokenise giella-cg: fix indentation issue with backtracking
+5a26269 Fix still some warnings.
+32c82b3 giella-cg backtracking: ensure ignored spaces are still output
+03d6a23 Do not warn about pragma push/pop issues or unsafe cstring functions on windows
+9c12094 Use hfst::hfst_fopen instead of fopen. Fix some warnings related to datatype conversions.
+d64cb49 Add function hfst::hfst_fopen that uses fopen_s when compiling with msvc.
+5ed27ea Fix some warnings caused by conversions (possible loss of data).
+b93a144 Do not use extern templates when compiling with msvc.
+73f7e18 Remove files generated by the tests.
+61ac4f8 Add a batch file version of test.sh for testing the python bindings on windows.
+b53a128 Fix warnings.
+7cb8740 Update pypi setup script.
+cc9cca4 Remove unnecessary back-end include directories as well as commented code from Makefiles.
+fd5f007 Fix some minor warnings.
+ec114ea --giella-cg support for @PMATCH_BACKTRACK@ symbol (#337)
+15fbb1f Update windows script.
+097826c Remove extra comma from initialization.
+d6b001d Add missing header file.
+9b3c122 Fix warnings generated when compiling with -Wall.
+a0dee64 Fix most of the warnings generated when compiling with -Wall.
+89b2061 rename --gtd to --giella-cg, and more descriptive help text
+335df14 Add support for compiling on windows.
+85585c4 Fix a typo.
+e628f55 Add a manifest file for pypi packaging.
+48c34ef Tentatively add a setup file for creating pypi packages.
+6abde49 update to proc to make it possible to weight surface forms
+5e379fc Update windows compilation script.
+1f55a51 Define int64 for msvc.
+21f52d0 Implement snprintf for msvc compilation.
+c05469e Replace definition _Bool with Boolean, as _Bool conflicts with typedefs in msvc header files.
+00bbb1d Get rid of unnecessary typenames and typos noticed during compiling HFST on windows.
+b160c07 Forward declare back-end implementation datatypes and wrap their destructors.
+067c3c9 Add missing <iterator> header.
+2f0c8df Add missing std:: to vector.
+34e99ad Issue an error message if glib unicode handler is requested but not found.
+d28075c Declare all functions that are needed in start_xfst().
+ec34dea Add math.h
+4a3be6b Make preparations for supporting OpenFst version 1.5.4.
+c9940f6 Include HfstBasicTransducer.h instead of HfstTransitionGraph.h.
+8560638 Add extern templates.
+dc96953 Remove unnecessary OpenFst headers and use forward declarations instead.
+699c098 Wrap StdVectorFst and LogFst destructors as both classes are forward declared.
+7f81f41 Forward declare as much as possible when using OpenFst library. Also remove unnecessary include directives and add missing std namespaces.
+a0b2fb3 Prepend missing 'std::'.
+c8137fa Make HfstBasicTransducer and HfstBasicTransition fully implemented classes instead of generated template specializations.
+eb5b877 Improve the way error messages are handled.
+b8278ec Change 'import libhfst' to 'import hfst'.
+2b9e44b Move implementations of HfstTropicalTransducerTransitionData away from header file.
+ce05c67 Merge branch 'master' of https://github.com/hfst/hfst
+3ec0d09 Alter expected result to conform with newly expected behaviour
+759605c Do not allow newline characters inside quoted literals.
+f296b6b Merge branch 'master' of https://github.com/hfst/hfst
+63d7544 Allocate enough memory for XRE error message.
+f968ebf Trying out facility for communicating sentence boundaries to plain tokenization mode
+6a78c4c Merge branch 'master' of https://github.com/hfst/hfst
+bf2ded3 Don't print size info when unverbose
+5a33841 Don't separate tokens with empty lines in plain tokenization mode
+d0a1b55 Warn about undefined multichar symbols when compiling lexc in verbose mode.
+5d2c67d Fix a typo in option handling.
+e2fee81 Tentatively add an option --do-not-minimize to hfst-regexp2fst which performs determinization instead of minimization. Also make hfst-xfst use determinization if variable 'minimal' is set to 'OFF'. Implement this by adding function hfst::set_minimization(bool) to HFST API and a function HfstTransducer::optimize() which performs minimization, or determinization if hfst::set_minimization(false) has been called. Replace calls to minimize() with optimize() in parts of HFST code relev [...]
+82aa26e Get rid of CR-LF newlines.
+44f2e21 Remove commented code.
+94b7607 Print an error message if output type is defined several times.
+966a895 Fix mismatched free/delete reported by valgrind.
+6dc1d8f Do not use strdup with -std=c99.
+407a517 Use delete_foma instead of delete/free when deallocating memory used by a foma transducer.
+07508b5 Limit to C++11 for now
+d15b4be Use free with malloc, not delete.
+b693b3a Add a function 'void LexcCompiler::reset()' and use it in XfstCompiler when reading lexc input again.
+bc5ac27 Remove revision info from output of hfst-info as it doesn't work correctly in git.
+ace4801 Pass cascade argument by reference in lookup_cascading functions.
+e2ea4cb Fix a typo in function 'strip_percents' which resulted in incorrectly escaped zeros.
+92a7152 delete more lines
+9f940a3 remove osx lines from yml
+f7bd085 Unobservable change - nothing happened here
+d1bf79b close #336: off-by-one segfaults in OlLetterTrie
+2574e27 Avoid using projections in casing functions (which cause minimization issues with large weighted transducers).
+73a3ff0 Parametrize optionality in casing functions
+0adc1f9 Make sure there's no case fallthrough
+58eb35d Fix other casing functions too
+c366c4c Another fix to Cap()
+704c1c7 Merge branch 'master' of https://github.com/hfst/hfst
+5a5df2d Change operation of Cap() and make timing per-object
+44f0ae7 Fix missing empty lines in output when cascading composition is used.
+284f9d0 Add an option --max-number=N for OL transducers which limits the number of results. If not defined, defaults to 5 for infinitely ambiguous input and to infinity for finitely ambiguous input.
+5686864 There are some problems in using the bindings with python version 2, so refer only to python3 in README.
+5a17fb0 Add a missing 'break' to switch block, making variable infinite_cutoff work correctly.
+4c2f26f If no results are found, print it only once.
+d703776 Tentatively add support for --xfst=print-pairs for OL transducers.
+b7fcab9 Remember to flush the streams also when no results are found.
+7ba32cf Merge branch 'master' of https://github.com/hfst/hfst
+bc7a80a Implement a pair-lookup for optimized-lookup transducers.
+ce3133f Add a fourth argument of type hfst::ImplementationType to the constructor and a function get_type() to ImplementationTypeNotAvailableException.
+4ffe105 Print to console on windows if print-pairs is specified.
+587f3c0 Do not print extra newlines in cascading composition.
+4353894 Fix a bug/typo in string comparison.
+72de782 Improve documentation in HfstTransducer.lookup.
+0c888e9 Rename HfstBasicTransducer.lookup_fd to HfstBasicTransducer.lookup and add an argument 'obey_flags'.
+d5493ac Rename HfstTransitionGraph's lookup_fd to lookup and change some parameter names to be more descriptive.
+56a8d2f Add instructions for linking hfst c++ library statically in comments.
+67e25ff Add missing file to windows scripts.
+8db14ce (tag: v3.11.0) Ready for release 3.11.0.
+fe37aa0 Update morphology checks.
+8d3f162 Update EXTRA_DIST.
+6dac246 Do not use binary transducers in tests.
+ee62c44 Add missing doc file.
+a367645 Mention that python tests are no longer available.
+da2678d Skip Python tests as they can now be performed in the folder python/test.
+ea47f0b One more stray memory leaking cache
+95e3480 Memory conservation: only cache named objects and always minimize them.
+6b113b2 Merge branch 'master' of https://github.com/hfst/hfst
+974d240 Turn off automatic delimiter-contexts ("set need-separators off") in the presence of user-defined contexts
+01fccfd Use docstrings.i instead of swig's autodoc. Document the contents of hfst and hfst.exceptions modules.
+4322ca7 Add a docstrings file created semiautomatically from doxygen documentation.
+816ce6c Add at least a simple docstring for all functions.
+82c8f6a Merge branch 'master' of https://github.com/hfst/hfst
+1b11fbf Obey -i option for giving the pmatch archive filename
+4620b1b std=c++ works on OS X but not Linux, so disable Linux+Clang
+d245a74 Test if gnu++ appeases Travis' clang
+3d9c66b Include needed m4
+5e73db3 Copy hfst-ospell's C++11+ detection
+9af66d0 short int -> short; long int -> long
+daa457e Yet more documentation..
+e1b01ff More docstrings.
+711a089 Add docstrings.
+acf5696 tokenise: don't skip output-empties
+b5b4434 tokenise: don't let empty analyses affect weight classes
+18f5a2b travis: fix indentation
+62d216a getopt_long() returns an int, not char; this is important because char is unsigned on many platforms which means char(-1) is 255
+39d28d1 Do not check if lookup is infinitely ambiguous if --time-cutoff is defined.
+c42af37 Remove redundant flag diacritic check. It was once removed, but got pushed there again.
+b7b5882 Add support for --cascade also for optimized-lookup transducers.
+9169e38 Support combination --cascade=composition --xfst=print-pairs.
+51a0b61 Merge branch 'master' of https://github.com/hfst/hfst
+ef6ed81 Fix the way cycle limit argument is passed.
+9ad0950 Merge branch 'master' of https://github.com/hfst/hfst
+6d94c90 Add a preliminary omorfi-friendly conll-u mode
+45a0bf2 Remove redundant filtering of paths containing flag diacritics. Flags are handled already in function HfstBasicTransducer::lookup_fd.
+5c7135a Implement option --cascade.
+18d5e66 Tentatively add option --cascade to hfst-lookup.
+1dcad09 Print error message if --xfst=obey-flags is used with optimized-lookup transducers. They always obey flag diacritics.
+670ee39 Merge branch 'master' of https://github.com/hfst/hfst
+f903d9c When doing apply up or down in hfst-xfst's, check if 'obey-flags' is ON or OFF.
+8e9c6e6 travis: --enable-silent-rules
+a253b8d Add an argument obey_flags=false to is_lookup_infinitely_ambiguous functions.
+14bdd9c Make is_lookup_infinitely_ambiguous aware of flags. Use typedef hfst::StringVector in HfstLookupFlagDiacritics to prevent conflicting definitions in hfst-twolc which defines a class StringVector.
+349fb73 Add option obey_flags=false to lookup_fd.
+fdf912a Add tentative script for converting from doxygen documentation to docstrings.
+7618907 Modify function visibility.
+bc01c0b Control visibility of functions in HfstTransducer.
+b535aa6 Follow foma and xfst convention for operator ignore '/', i.e. expand any identities and unknowns before inserting the strings to be ignored.
+c1b4341 Add an argument 'force' to HfstTransducer::harmonize which makes the function harmonize also FOMA_TYPE transducers.
+724b4e9 Obey variable 'print-sigma' when print_net is called.
+6bec507 Add brief descriptions for hfst packages.
+ca892d1 Document the implementation types.
+98764da Fix bugs in options --xfst=show-flags and --xfst=obey-flags when non-optimized lookup is performed.
+93b9fe2 Close the outstream when done. Not doing this was causing a bug where some buffered data never got written to the output file.
+7d97106 Obey variable 'print-pairs' when performing apply down in non-optimized format.
+ac2fc3a Add parentheses in conditionals of function 'flag_build' to ensure correct interpretation. Also remove redundant code.
+28867c1 Allow punctuation as context in the "need-separators on" mode
+0cd05ee Add "need-separators no" to test script to reflect changed default in pmatch
+2597799 exclude gcc from osx builds
+7dfa8ec Track used symbol names
+81128c2 Allow lowercase define()
+52a4ca4 Merge branch 'master' of https://github.com/hfst/hfst
+af71662 Add side arguments to casing functions
+7c1b388 Revert back to the original extra_link_args in python bindings.
+2d5c670 Rename examples.py to test_examples.py. Perform tests in a more logical order.
+4c8a1e3 Update documentation about linking to the HFST C++ library.
+9f8ca0f If --enable-no-tools is used, skip tests in test/tools/.
+41dc9a2 Mention module hfst.types.
+a8c3e56 Remember to apply modifications to weight pushing to tests and hfst package.
+835f0ef Get rid of TO_FINAL_STATE and TO_INITIAL_STATE and replace function 'push_weights' with 'push_weights_to_start' and 'push_weights_to_end'.
+d05b9db Wrap HfstTransducer implementation types in package 'hfst.types'.
+5e1a245 Add missing files to dist.
+58d4602 Update README. Handle linking to extension library better.
+de888b6 Update README. Handle cases where an implementation type is not available. Add missing files to dist.
+1a68126 Fix a typo in URL.
+7a2296b Update link in README.
+861baf6 tokeniser: --weight-classes=N opt, as in hfst-proc
+151f1fa Add functions lookup_optimize, remove_optimization and copy to HfstTransducer.
+4efc9af Make HfstTransducer() create the empty transducer.
+0f237c1 Comment out hfst_ol::set_extract_tags_mode(bool). Improve error messages in att and prolog readers.
+57426ca Remove unnecessary commented file.
+b7139cd Fix HfstException documentation.
+3b13bf5 Merge branch 'master' of https://github.com/hfst/hfst
+7e6d27c Fix a typo.
+9f4ceb7 Merge branch 'master' of https://github.com/hfst/hfst
+829c9e2 Implement remaining variables in the library and support them in hfst-pmatch, with the exception of need-separators which has to be specified during rule compilation, at least for now.
+457c5de Document AttReader and PrologReader.
+8695c47 Add HfstException::what() to python api.
+2cffffa Add an alias what() for operator() to HfstException.
+5fdf61a Add class PrologReader and tests for it.
+77eda55 Add a new class AttReader and tests for it.
+18edac5 Add package hfst to setup. The package is essentially a wrapper around libhfst module which is still available as before.
+e41f131 travis: re-encrypt channel to #hfst
+6329bb8 Use all implementation formats in tests.
+5a5018c Do not use default_fst_type as pmatch transducers are always in ol format.
+b38f87b Add stream tests.
+ccaea0e Represent empty transducers as empty lines so that read_att_transducer will work correctly.
+fe66b18 Remove unnecessary files.
+bf5511d Forgot to do git add before committing...
+3710f91 Update test examples.py and add it to test.sh. Remove extra files.
+7d5064d Add directory tree for package 'hfst' and its subpackages 'exceptions' and 'rules'. This package encapsulates module libhfst.
+7818c1e Delete unnecessary file.
+91b86a0 Rewrite part of tests. Document the interface file.
+657013d Add new tests.
+3b4c186 Merge pull request #323 from unhammer/tokenise--gtd-tags-are-multichars
+67125ab rm unneeded import
+d927d79 pmatch: store input/output_symbol_strings, _parts refer to them
+c665f8a Add missing endverbatim tag.
+884d1dc Add missing file.
+bc87fc5 Get rid of HfstFile wrapper.
+30cde02 Try to preserve the names of transducers when converting.
+90b3328 Expand prolog tests. Fix a small bug in naming transducers.
+8e61537 Forgot to update testfile.
+63a0456 Add a new function for reading prolog input and improve the way att input is handled.
+9ef38f7 Split part of code related to reading att input into a separate function. Change the way transducer name is set when reading prolog input.
+6d12e0d erge branch 'master' of https://github.com/hfst/hfst
+22f1c47 Modify documentation, update tests, add new functions.
+d14bc2f link to travis; bug tracker link sf→github
+e8c7fd6 README is now symlink to README.rst; for more readable github
+2258354 travis: no irc notes on success
+1d8a1f4 travis: encrypt IRC channel for hfst/hfst
+38c9544 Add HfstTokenizer examples as tests.
+c635cd4 More documentation to HfstTokenizer.
+c1ca3b6 Add more examples for HfstTokenizer.
+0dc0cf4 Reorganize doxygen documentation to represent the new module structure of hfst python api.
+95d658c Remove hfst.py, a directory structure is needed instead for doxygen.
+5025aab Add new documentation for hfst python package.
+4749745 Add more tests.
+8f8b345 Split python tests to several files.
+45aeb38 Skip one more failing pmatch test.
+83dee76 Add missing files to dist.
+a44b99e erge branch 'master' of https://github.com/hfst/hfst
+b5ca1ec Add more documentation and modify functions.
+579ee71 Merge branch 'master' of https://github.com/hfst/hfst
+8050575 A PmatchContainer constructor taking a std::vector<HfstTransducer>. This should leave the passed vector untouched.
+88aa682 Add more documentation.
+ab82047 Add a simple test for XreCompiler.
+a25eb2b A new test.
+caf1ca8 XreCompiler::setOutputToConsole is needed, uncomment it.
+f6acf2a Split c++ code to multiple files from libhfst.i. Update URL in setup.py.
+780374a Merge branch 'master' of https://github.com/hfst/hfst
+4c9ca19 Settle on a way to handle variable names with dashes (which look like minus signs)
+967cc78 actually run the tokenize tests
+5df87b6 Add more failing tests.
+775a499 Fix some typos.
+2e77e2d Add missing files to dist.
+11ebca0 Merge branch 'master' of https://github.com/hfst/hfst
+067e637 Merging master
+4c9d43a Variable setting and reading
+881b81c Use -Wno-deprecated also for tests.
+85cb277 Modify messages that get printed during make check.
+0761730 Skip tests that do not exist. Make prints more informative.
+573324e Skip failing pmatch tests.
+164e621 Add missing tests for ConvertTransducerFormat. Compile them with -Wno-deprecated.
+f945e17 travis: irc notifications on build failure
+0229baa Merge branch 'master' of https://github.com/hfst/hfst
+93ed0db Remove const declarations from openfst back-end that cause compiler warnings.
+5df2a2b Run g++ with -j2 (not -j) on travis; o/w runs out of memory
+54b8f45 Comment out a function that is not defined.
+48c91b8 Specify -Wno-deprecated also for tests in folders hfst-tagger and hfst-twolc.
+810c330 https://travis-ci.org test rules
+708171e doh, define multichar-symbols (for 2eb6888)
+2eb6888 test for #320 / bf26db9
+3aecdbc Merge branch 'master' of github.com:hfst/hfst
+bf26db9 only change case if sf and lm case differ (#320)
+b3782de Remove a rule that cannot be matched since a rule has already been defined for "DownCase(" earlier.
+5ad3487 Add the keyword const to arrays used to store xfst commands.
+79c7dfe Make compiler happy by using strdup instead of plain const char *.
+f6d9953 Handle return values of fread in foma and sfst back-ends to prevent warnings during compilation.
+214b7b6 Fix a typo.
+862e858 Replace foma binary files that are used for testing results from lexc with equivalent files in prolog format. Then the tests will not depend on the foma back-end.
+680389b tokenize tests
+2b7cf7e tokenize: new --gtd Giellatekno/Divvun mode
+adb37d4 tokenise: fix #318, header parsing for TOP
+76e9069 Fix a bug in flag elimination involving multiple flags. Fixes bug #315.
+f653780 Evaluate transducers (as opposed to argumentless functions) where declared
+e6c6e82 (tag: v3.10.0) Ready for release 3.10.0.
+97c1285 Swap implementations of commands 'apply up' and 'apply down' in hfst-xfst, making them work as they do in foma and xfst.
85ef34c Forgot to add before latest commit...
13280ea Edit warning about converting native foma transducers.
b31a6c1 Fix typo: swap left and right.
diff --git a/ChangeLog.old b/ChangeLog.old
index eef6319..0ac39cf 100644
--- a/ChangeLog.old
+++ b/ChangeLog.old
@@ -1,4 +1,359 @@
-dd7d0be (HEAD -> master, origin/master, origin/HEAD) Merge pull request #309 from unhammer/pmatch-input-marks
+42df833 (HEAD -> master, origin/master, origin/HEAD) Small fixes to memory leaks and efficiency.
+e2abd89 Fix some memory leaks.
+da54acc Fix leaks reported by valgrind.
+92cf4dc Fix more memory leaks.
+9bd9749 Fix some lost memory issues reported by valgrind.
+6e129a4 Remove unnecessary variable. Delete transducer definitions and latest regexp when destructor is called.
+d700309 flush given outstream, not necessarily cout
+8d4f824 Add a destructor to XfstCompiler.
+6b10aab Update README.
+863e94a Add missing files to dist and remove generated ones.
+e4c1304 Rename hfst.rules into hfst.sfst_rules. Fix hfst.types into hfst.ImplementationType in tests.
+80f18a8 Use single quotes instead of double quotes in docstrings.
+52a9d75 Add test for hfst.xerox_rules.
+00b020c Rename xerox_replace_left and xerox_restriction back to replace_left and restriction in package hfst.xerox_rules.
+d1556dd Fix documentation of parameter 'output' of function 'extract_paths'.
+d3058c7 Issue a warning if automake is too old, which can make build fail if pre-generated yacc files are modified.
+9166cdc Move ImplementationTypes and ReplaceTypes to their own classes and get rid of package hfst.types.
+6d6c631 Fix datatypes used in for loops.
+c6aa5e1 Fix some typos in documentation.
+8073675 Use renamed versions of xerox rule functions.
+ae70056 Tentatively add HfstXeroxRules to python API in package hfst.xerox_rules.
+dee0176 Fix some more warnings.
+37edbbb Fix minor warnings generated by clang.
+faa7647 Fix typos in ifdef guards.
+41c5a32 Add a simple script for fetching statistics about hfst downloads.
+d9d8077 Update pypi setup.
+822f4bb doh, forgot to add test expected output
+cccba22 hfst-tokenise giella-cg: fix indentation issue with backtracking
+5a26269 Fix still some warnings.
+32c82b3 giella-cg backtracking: ensure ignored spaces are still output
+03d6a23 Do not warn about pragma push/pop issues or unsafe cstring functions on windows
+9c12094 Use hfst::hfst_fopen instead of fopen. Fix some warnings related to datatype conversions.
+d64cb49 Add function hfst::hfst_fopen that uses fopen_s when compiling with msvc.
+5ed27ea Fix some warnings caused by conversions (possible loss of data).
+b93a144 Do not use extern templates when compiling with msvc.
+73f7e18 Remove files generated by the tests.
+61ac4f8 Add a batch file version of test.sh for testing the python bindings on windows.
+b53a128 Fix warnings.
+7cb8740 Update pypi setup script.
+cc9cca4 Remove unnecessary back-end include directories as well as commented code from Makefiles.
+fd5f007 Fix some minor warnings.
+ec114ea --giella-cg support for @PMATCH_BACKTRACK@ symbol (#337)
+15fbb1f Update windows script.
+097826c Remove extra comma from initialization.
+d6b001d Add missing header file.
+9b3c122 Fix warnings generated when compiling with -Wall.
+a0dee64 Fix most of the warnings generated when compiling with -Wall.
+89b2061 rename --gtd to --giella-cg, and more descriptive help text
+335df14 Add support for compiling on windows.
+85585c4 Fix a typo.
+e628f55 Add a manifest file for pypi packaging.
+48c34ef Tentatively add a setup file for creating pypi packages.
+6abde49 update to proc to make it possible to weight surface forms
+5e379fc Update windows compilation script.
+1f55a51 Define int64 for msvc.
+21f52d0 Implement snprintf for msvc compilation.
+c05469e Replace definition _Bool with Boolean, as _Bool conflicts with typedefs in msvc header files.
+00bbb1d Get rid of unnecessary typenames and typos noticed during compiling HFST on windows.
+b160c07 Forward declare back-end implementation datatypes and wrap their destructors.
+067c3c9 Add missing <iterator> header.
+2f0c8df Add missing std:: to vector.
+34e99ad Issue an error message if glib unicode handler is requested but not found.
+d28075c Declare all functions that are needed in start_xfst().
+ec34dea Add math.h
+4a3be6b Make preparations for supporting OpenFst version 1.5.4.
+c9940f6 Include HfstBasicTransducer.h instead of HfstTransitionGraph.h.
+8560638 Add extern templates.
+dc96953 Remove unnecessary OpenFst headers and use forward declarations instead.
+699c098 Wrap StdVectorFst and LogFst destructors as both classes are forward declared.
+7f81f41 Forward declare as much as possible when using OpenFst library. Also remove unnecessary include directives and add missing std namespaces.
+a0b2fb3 Prepend missing 'std::'.
+c8137fa Make HfstBasicTransducer and HfstBasicTransition fully implemented classes instead of generated template specializations.
+eb5b877 Improve the way error messages are handled.
+b8278ec Change 'import libhfst' to 'import hfst'.
+2b9e44b Move implementations of HfstTropicalTransducerTransitionData away from header file.
+ce05c67 Merge branch 'master' of https://github.com/hfst/hfst
+3ec0d09 Alter expected result to conform with newly expected behaviour
+759605c Do not allow newline characters inside quoted literals.
+f296b6b Merge branch 'master' of https://github.com/hfst/hfst
+63d7544 Allocate enough memory for XRE error message.
+f968ebf Trying out facility for communicating sentence boundaries to plain tokenization mode
+6a78c4c Merge branch 'master' of https://github.com/hfst/hfst
+bf2ded3 Don't print size info when unverbose
+5a33841 Don't separate tokens with empty lines in plain tokenization mode
+d0a1b55 Warn about undefined multichar symbols when compiling lexc in verbose mode.
+5d2c67d Fix a typo in option handling.
+e2fee81 Tentatively add an option --do-not-minimize to hfst-regexp2fst which performs determinization instead of minimization. Also make hfst-xfst use determinization if variable 'minimal' is set to 'OFF'. Implement this by adding function hfst::set_minimization(bool) to HFST API and a function HfstTransducer::optimize() which performs minimization, or determinization if hfst::set_minimization(false) has been called. Replace calls to minimize() with optimize() in parts of HFST code relev [...]
+82aa26e Get rid of CR-LF newlines.
+44f2e21 Remove commented code.
+94b7607 Print an error message if output type is defined several times.
+966a895 Fix mismatched free/delete reported by valgrind.
+6dc1d8f Do not use strdup with -std=c99.
+407a517 Use delete_foma instead of delete/free when deallocating memory used by a foma transducer.
+07508b5 Limit to C++11 for now
+d15b4be Use free with malloc, not delete.
+b693b3a Add a function 'void LexcCompiler::reset()' and use it in XfstCompiler when reading lexc input again.
+bc5ac27 Remove revision info from output of hfst-info as it doesn't work correctly in git.
+ace4801 Pass cascade argument by reference in lookup_cascading functions.
+e2ea4cb Fix a typo in function 'strip_percents' which resulted in incorrectly escaped zeros.
+92a7152 delete more lines
+9f940a3 remove osx lines from yml
+f7bd085 Unobservable change - nothing happened here
+d1bf79b close #336: off-by-one segfaults in OlLetterTrie
+2574e27 Avoid using projections in casing functions (which cause minimization issues with large weighted transducers).
+73a3ff0 Parametrize optionality in casing functions
+0adc1f9 Make sure there's no case fallthrough
+58eb35d Fix other casing functions too
+c366c4c Another fix to Cap()
+704c1c7 Merge branch 'master' of https://github.com/hfst/hfst
+5a5df2d Change operation of Cap() and make timing per-object
+44f0ae7 Fix missing empty lines in output when cascading composition is used.
+284f9d0 Add an option --max-number=N for OL transducers which limits the number of results. If not defined, defaults to 5 for infinitely ambiguous input and to infinity for finitely ambiguous input.
+5686864 There are some problems in using the bindings with python version 2, so refer only to python3 in README.
+5a17fb0 Add a missing 'break' to switch block, making variable infinite_cutoff work correctly.
+4c2f26f If no results are found, print it only once.
+d703776 Tentatively add support for --xfst=print-pairs for OL transducers.
+b7fcab9 Remember to flush the streams also when no results are found.
+7ba32cf Merge branch 'master' of https://github.com/hfst/hfst
+bc7a80a Implement a pair-lookup for optimized-lookup transducers.
+ce3133f Add a fourth argument of type hfst::ImplementationType to the constructor and a function get_type() to ImplementationTypeNotAvailableException.
+4ffe105 Print to console on windows if print-pairs is specified.
+587f3c0 Do not print extra newlines in cascading composition.
+4353894 Fix a bug/typo in string comparison.
+72de782 Improve documentation in HfstTransducer.lookup.
+0c888e9 Rename HfstBasicTransducer.lookup_fd to HfstBasicTransducer.lookup and add an argument 'obey_flags'.
+d5493ac Rename HfstTransitionGraph's lookup_fd to lookup and change some parameter names to be more descriptive.
+56a8d2f Add instructions for linking hfst c++ library statically in comments.
+67e25ff Add missing file to windows scripts.
+8db14ce (tag: v3.11.0) Ready for release 3.11.0.
+fe37aa0 Update morphology checks.
+8d3f162 Update EXTRA_DIST.
+6dac246 Do not use binary transducers in tests.
+ee62c44 Add missing doc file.
+a367645 Mention that python tests are no longer available.
+da2678d Skip Python tests as they can now be performed in the folder python/test.
+ea47f0b One more stray memory leaking cache
+95e3480 Memory conservation: only cache named objects and always minimize them.
+6b113b2 Merge branch 'master' of https://github.com/hfst/hfst
+974d240 Turn off automatic delimiter-contexts ("set need-separators off") in the presence of user-defined contexts
+01fccfd Use docstrings.i instead of swig's autodoc. Document the contents of hfst and hfst.exceptions modules.
+4322ca7 Add a docstrings file created semiautomatically from doxygen documentation.
+816ce6c Add at least a simple docstring for all functions.
+82c8f6a Merge branch 'master' of https://github.com/hfst/hfst
+1b11fbf Obey -i option for giving the pmatch archive filename
+4620b1b std=c++ works on OS X but not Linux, so disable Linux+Clang
+d245a74 Test if gnu++ appeases Travis' clang
+3d9c66b Include needed m4
+5e73db3 Copy hfst-ospell's C++11+ detection
+9af66d0 short int -> short; long int -> long
+daa457e Yet more documentation..
+e1b01ff More docstrings.
+711a089 Add docstrings.
+acf5696 tokenise: don't skip output-empties
+b5b4434 tokenise: don't let empty analyses affect weight classes
+18f5a2b travis: fix indentation
+62d216a getopt_long() returns an int, not char; this is important because char is unsigned on many platforms which means char(-1) is 255
+39d28d1 Do not check if lookup is infinitely ambiguous if --time-cutoff is defined.
+c42af37 Remove redundant flag diacritic check. It was once removed, but got pushed there again.
+b7b5882 Add support for --cascade also for optimized-lookup transducers.
+9169e38 Support combination --cascade=composition --xfst=print-pairs.
+51a0b61 Merge branch 'master' of https://github.com/hfst/hfst
+ef6ed81 Fix the way cycle limit argument is passed.
+9ad0950 Merge branch 'master' of https://github.com/hfst/hfst
+6d94c90 Add a preliminary omorfi-friendly conll-u mode
+45a0bf2 Remove redundant filtering of paths containing flag diacritics. Flags are handled already in function HfstBasicTransducer::lookup_fd.
+5c7135a Implement option --cascade.
+18d5e66 Tentatively add option --cascade to hfst-lookup.
+1dcad09 Print error message if --xfst=obey-flags is used with optimized-lookup transducers. They always obey flag diacritics.
+670ee39 Merge branch 'master' of https://github.com/hfst/hfst
+f903d9c When doing apply up or down in hfst-xfst's, check if 'obey-flags' is ON or OFF.
+8e9c6e6 travis: --enable-silent-rules
+a253b8d Add an argument obey_flags=false to is_lookup_infinitely_ambiguous functions.
+14bdd9c Make is_lookup_infinitely_ambiguous aware of flags. Use typedef hfst::StringVector in HfstLookupFlagDiacritics to prevent conflicting definitions in hfst-twolc which defines a class StringVector.
+349fb73 Add option obey_flags=false to lookup_fd.
+fdf912a Add tentative script for converting from doxygen documentation to docstrings.
+7618907 Modify function visibility.
+bc01c0b Control visibility of functions in HfstTransducer.
+b535aa6 Follow foma and xfst convention for operator ignore '/', i.e. expand any identities and unknowns before inserting the strings to be ignored.
+c1b4341 Add an argument 'force' to HfstTransducer::harmonize which makes the function harmonize also FOMA_TYPE transducers.
+724b4e9 Obey variable 'print-sigma' when print_net is called.
+6bec507 Add brief descriptions for hfst packages.
+ca892d1 Document the implementation types.
+98764da Fix bugs in options --xfst=show-flags and --xfst=obey-flags when non-optimized lookup is performed.
+93b9fe2 Close the outstream when done. Not doing this was causing a bug where some buffered data never got written to the output file.
+7d97106 Obey variable 'print-pairs' when performing apply down in non-optimized format.
+ac2fc3a Add parentheses in conditionals of function 'flag_build' to ensure correct interpretation. Also remove redundant code.
+28867c1 Allow punctuation as context in the "need-separators on" mode
+0cd05ee Add "need-separators no" to test script to reflect changed default in pmatch
+2597799 exclude gcc from osx builds
+7dfa8ec Track used symbol names
+81128c2 Allow lowercase define()
+52a4ca4 Merge branch 'master' of https://github.com/hfst/hfst
+af71662 Add side arguments to casing functions
+7c1b388 Revert back to the original extra_link_args in python bindings.
+2d5c670 Rename examples.py to test_examples.py. Perform tests in a more logical order.
+4c8a1e3 Update documentation about linking to the HFST C++ library.
+9f8ca0f If --enable-no-tools is used, skip tests in test/tools/.
+41dc9a2 Mention module hfst.types.
+a8c3e56 Remember to apply modifications to weight pushing to tests and hfst package.
+835f0ef Get rid of TO_FINAL_STATE and TO_INITIAL_STATE and replace function 'push_weights' with 'push_weights_to_start' and 'push_weights_to_end'.
+d05b9db Wrap HfstTransducer implementation types in package 'hfst.types'.
+5e1a245 Add missing files to dist.
+58d4602 Update README. Handle linking to extension library better.
+de888b6 Update README. Handle cases where an implementation type is not available. Add missing files to dist.
+1a68126 Fix a typo in URL.
+7a2296b Update link in README.
+861baf6 tokeniser: --weight-classes=N opt, as in hfst-proc
+151f1fa Add functions lookup_optimize, remove_optimization and copy to HfstTransducer.
+4efc9af Make HfstTransducer() create the empty transducer.
+0f237c1 Comment out hfst_ol::set_extract_tags_mode(bool). Improve error messages in att and prolog readers.
+57426ca Remove unnecessary commented file.
+b7139cd Fix HfstException documentation.
+3b13bf5 Merge branch 'master' of https://github.com/hfst/hfst
+7e6d27c Fix a typo.
+9f4ceb7 Merge branch 'master' of https://github.com/hfst/hfst
+829c9e2 Implement remaining variables in the library and support them in hfst-pmatch, with the exception of need-separators which has to be specified during rule compilation, at least for now.
+457c5de Document AttReader and PrologReader.
+8695c47 Add HfstException::what() to python api.
+2cffffa Add an alias what() for operator() to HfstException.
+5fdf61a Add class PrologReader and tests for it.
+77eda55 Add a new class AttReader and tests for it.
+18edac5 Add package hfst to setup. The package is essentially a wrapper around libhfst module which is still available as before.
+e41f131 travis: re-encrypt channel to #hfst
+6329bb8 Use all implementation formats in tests.
+5a5018c Do not use default_fst_type as pmatch transducers are always in ol format.
+b38f87b Add stream tests.
+ccaea0e Represent empty transducers as empty lines so that read_att_transducer will work correctly.
+fe66b18 Remove unnecessary files.
+bf5511d Forgot to do git add before committing...
+3710f91 Update test examples.py and add it to test.sh. Remove extra files.
+7d5064d Add directory tree for package 'hfst' and its subpackages 'exceptions' and 'rules'. This package encapsulates module libhfst.
+7818c1e Delete unnecessary file.
+91b86a0 Rewrite part of tests. Document the interface file.
+657013d Add new tests.
+3b4c186 Merge pull request #323 from unhammer/tokenise--gtd-tags-are-multichars
+67125ab rm unneeded import
+d927d79 pmatch: store input/output_symbol_strings, _parts refer to them
+c665f8a Add missing endverbatim tag.
+884d1dc Add missing file.
+bc87fc5 Get rid of HfstFile wrapper.
+30cde02 Try to preserve the names of transducers when converting.
+90b3328 Expand prolog tests. Fix a small bug in naming transducers.
+8e61537 Forgot to update testfile.
+63a0456 Add a new function for reading prolog input and improve the way att input is handled.
+9ef38f7 Split part of code related to reading att input into a separate function. Change the way transducer name is set when reading prolog input.
+6d12e0d erge branch 'master' of https://github.com/hfst/hfst
+22f1c47 Modify documentation, update tests, add new functions.
+d14bc2f link to travis; bug tracker link sf→github
+e8c7fd6 README is now symlink to README.rst; for more readable github
+2258354 travis: no irc notes on success
+1d8a1f4 travis: encrypt IRC channel for hfst/hfst
+38c9544 Add HfstTokenizer examples as tests.
+c635cd4 More documentation to HfstTokenizer.
+c1ca3b6 Add more examples for HfstTokenizer.
+0dc0cf4 Reorganize doxygen documentation to represent the new module structure of hfst python api.
+95d658c Remove hfst.py, a directory structure is needed instead for doxygen.
+5025aab Add new documentation for hfst python package.
+4749745 Add more tests.
+8f8b345 Split python tests to several files.
+45aeb38 Skip one more failing pmatch test.
+83dee76 Add missing files to dist.
+a44b99e erge branch 'master' of https://github.com/hfst/hfst
+b5ca1ec Add more documentation and modify functions.
+579ee71 Merge branch 'master' of https://github.com/hfst/hfst
+8050575 A PmatchContainer constructor taking a std::vector<HfstTransducer>. This should leave the passed vector untouched.
+88aa682 Add more documentation.
+ab82047 Add a simple test for XreCompiler.
+a25eb2b A new test.
+caf1ca8 XreCompiler::setOutputToConsole is needed, uncomment it.
+f6acf2a Split c++ code to multiple files from libhfst.i. Update URL in setup.py.
+780374a Merge branch 'master' of https://github.com/hfst/hfst
+4c9ca19 Settle on a way to handle variable names with dashes (which look like minus signs)
+967cc78 actually run the tokenize tests
+5df87b6 Add more failing tests.
+775a499 Fix some typos.
+2e77e2d Add missing files to dist.
+11ebca0 Merge branch 'master' of https://github.com/hfst/hfst
+067e637 Merging master
+4c9d43a Variable setting and reading
+881b81c Use -Wno-deprecated also for tests.
+85cb277 Modify messages that get printed during make check.
+0761730 Skip tests that do not exist. Make prints more informative.
+573324e Skip failing pmatch tests.
+164e621 Add missing tests for ConvertTransducerFormat. Compile them with -Wno-deprecated.
+f945e17 travis: irc notifications on build failure
+0229baa Merge branch 'master' of https://github.com/hfst/hfst
+93ed0db Remove const declarations from openfst back-end that cause compiler warnings.
+5df2a2b Run g++ with -j2 (not -j) on travis; o/w runs out of memory
+54b8f45 Comment out a function that is not defined.
+48c91b8 Specify -Wno-deprecated also for tests in folders hfst-tagger and hfst-twolc.
+810c330 https://travis-ci.org test rules
+708171e doh, define multichar-symbols (for 2eb6888)
+2eb6888 test for #320 / bf26db9
+3aecdbc Merge branch 'master' of github.com:hfst/hfst
+bf26db9 only change case if sf and lm case differ (#320)
+b3782de Remove a rule that cannot be matched since a rule has already been defined for "DownCase(" earlier.
+5ad3487 Add the keyword const to arrays used to store xfst commands.
+79c7dfe Make compiler happy by using strdup instead of plain const char *.
+f6d9953 Handle return values of fread in foma and sfst back-ends to prevent warnings during compilation.
+214b7b6 Fix a typo.
+862e858 Replace foma binary files that are used for testing results from lexc with equivalent files in prolog format. Then the tests will not depend on the foma back-end.
+680389b tokenize tests
+2b7cf7e tokenize: new --gtd Giellatekno/Divvun mode
+adb37d4 tokenise: fix #318, header parsing for TOP
+76e9069 Fix a bug in flag elimination involving multiple flags. Fixes bug #315.
+f653780 Evaluate transducers (as opposed to argumentless functions) where declared
+e6c6e82 (tag: v3.10.0) Ready for release 3.10.0.
+97c1285 Swap implementations of commands 'apply up' and 'apply down' in hfst-xfst, making them work as they do in foma and xfst.
+85ef34c Forgot to add before latest commit...
+13280ea Edit warning about converting native foma transducers.
+b31a6c1 Fix typo: swap left and right.
+808d984 Add hfst-flookup to windows scripts.
+66db285 Update bug tracker url and copyright year.
+b127ab3 Git ignore hfst-flookup.
+1b40de6 Add tests for hfst-flookup.
+12e4cca Trandsducer operations no longer return a reference to the transducer, fix this in tests and interface operations.
+a128c7f Add a tool hfst-flookup which does lookup from right to left, i.e. in the same way as foma's flookup and xfst's lookup. hfst-lookup and hfst-optimized-lookup stay as they are, but mention in their help message that they do lookup in the opposite direction.
+cea21b4 Add a comment about the way gzipped input is handled.
+da7ebcc Make HfstInputStream constructor throw a FileIsInGZFormatException if input is in gzipped format. In hfst-fst2fst, issue an error message that asks user to unzip the transducer.
+5c2755b Merge commit frankier:add-gitignore manually and also add more files to be ignored by git.
+99b470d Merge pull request #313 from frankier/escape-dot-labels
+f6905a1 Improve handling of argumentless functions and funcalls
+d804d3d Remove some memory leaks in compilation
+ef740e0 Fix nested funcall issues
+61e1e0a Add optional versions of cap functions and AnyCase().
+68e078b Corrections and additions to casing functions, and add Implode() and Explode()
+2484633 Add a facility for detecting when hfst-tokenize is given a "bare" dictionary and generating a somewhat intelligent tokenizer when that happens. Due to awkward architectural reasons this involves changes and additions in many parts of the library, but they are not major.
+089f3ca Add .gitignore
+d1d81db Merge branch 'master' of https://github.com/hfst/hfst
+485f575 Fix bug that caused exclusionary lists to be missed much of the time.
+03ff992 Escape " in dot labels.
+9984096 Factor out replace_all into string-utils.cc.
+ca10d77 Revert swapping of hfst-xfst's 'apply up' and 'apply down' commands.
+7352cf7 Swap implementations of 'apply up' and 'apply down' in hfst-xfst, so that these commands work in the same way as in foma and xfst.
+db2d4dc Add an example for HfstTransducer constructor.
+f37b8d1 Tentatively implement lookup also for transducers that are not in optimized-lookup format using composition and path extraction.
+1cb2992 Add an HfstTransducer constructor that takes a StringVector as argument.
+f32c049 Support python 2 unicode strings in lookup.
+8d2c6b8 Fix a typo
+f442b91 Make HfstInputStream iterable by implementing functions __iter__, __next__ (for Python 3) and next (for Python 2).
+2c660ff Add functions in namespace hfst::rules to python API.
+947bb62 Add some information about Python interface in README and advise the users to see file python/README.
+067b0c2 Remove slashes from headers used when processing the file.
+c4b5ce8 Update documentation.
+0e4e8a7 Redefine functions which return a reference to themselves as void to avoid problems on the python side.
+5f97eac Tentatively add a C++ function 'std::vector<hfst::HfstTransducer> compile_pmatch_expression(const std::string & pmatch)', mostly copied from hfst-pmatch2fst.cc.
+21da500 Add a const declaration to string& argument of PmatchContainer::process and PmatchContainer::match.
+cd7f2f8 Tentatively add some pmatch functions. Add a missing typedef for HfstTransducerVector.
+a9fbd44 Add a function to HfstInputStream for finding out whether hfst headers are included in transducers. When reading native foma transducers in hfst-fst2fst, warn that inversion may be needed for lookup to work as expected.
+fa4565e (tag: v3.9.2) Release 3.9.2.
+3f7a57e Update hfst binary file used for tests.
+5cd8a71 Add option --pythonpath to python tests. Document the test tools better.
+927da9c Make fst2txt functionality test silent.
+ad8c82c Update installation check scripts.
+ddf749d Remove test.py from EXTRA_DIST, the tests have been rewritten.
+dd7d0be Merge pull request #309 from unhammer/pmatch-input-marks
97b6e3f Merge pull request #308 from unhammer/tokenise-minor-cg-fix
3781806 Make small modifications to documentation and typedefs used in function declarations/definitions, so that doxygen can generate documentation right.
00d89ce Update Python and C++ API documentation files.
diff --git a/NEWS b/NEWS
index 8ddd163..f00cdf4 100644
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,58 @@
This file contains all noteworthy changes in HFST development between releases.
For full listing of changes see ChangeLog.
+Noteworthy changes in 3.12.1
+----------------------------
+
+* Fix flag elimination bug (reported in issue #342)
+
+* Do not allow unescaped dots in regular expressions
+
+* Improvements to pmatch and tokenization tools:
+
+ * Search for included files under scriptdir, not working dir
+
+ * Add experimental two-vector model for word sense
+
+ * Handle Apertium-style superblanks in --giella-cg format
+
+* Rename the PyPI package to 'hfst' (currently available at testpypi.python.org)
+
+* Update foma back-end
+
+
+Noteworthy changes in 3.12.0
+----------------------------
+
+* fixes to memory leaks and efficiency
+
+* fixes to numerous warnings
+
+* changes in Python bindings:
+
+ * rename hfst.rules into hfst.sfst_rules
+
+ * get rid of hfst.types and offer implementation types in class ImplementationType
+
+ * add Xerox-type rules in module hfst.xerox_rules
+
+ * improve documentation
+
+ * tentatively add partial support for pypi installation
+
+* improvements to pmatch tools and hfst-proc
+
+
+Noteworthy changes in 3.11.0
+----------------------------
+
+* Add docstrings to Python API
+
+* Changes and improvements to pmatch tools, hfst-tokenize, hfst-optimized-lookup, hfst-lookup and hfst-xfst
+
+* Fix bugs in flag elimination
+
+
Noteworthy changes in 3.10.0
----------------------------
@@ -118,10 +170,10 @@ Noteworthy changes in 3.8.1
* now hfst-lexc has options --withFlags, --minimizeFlags and --renameFlags to control
hyperminimization
- * hfst-xfst has corresponding variables --lexc-with-flags, --lexc-minimize-flags and
+ * hfst-xfst has corresponding variables --lexc-with-flags, --lexc-minimize-flags and
--lexc-rename-flags
- * hfst-xfst has variable xerox-composition (default ON) and hfst-lexc option
+ * hfst-xfst has variable xerox-composition (default ON) and hfst-lexc option
--xerox-composition (default ON) to control if flags are matched with unknowns and
identities in composition (as Xerox does)
@@ -148,7 +200,7 @@ Noteworthy changes in 3.8.1
Noteworthy changes in 3.7.1
---------------------------
-* same mappings with different weights are now handled right in
+* same mappings with different weights are now handled right in
parallel replace rules.
* improvements to optimized lookup format
@@ -165,7 +217,7 @@ Noteworthy changes in 3.7.0
* bugfixes to handling of optimized lookup transducers
-* support for weighted containment operator
+* support for weighted containment operator
* tentatively added option and variable 'encode-weights' to regexp
and xfst parsers to control determinization of weighted transducers,
@@ -233,7 +285,7 @@ Noteworthy changes in 3.5.0
* changes to other tools
- * added support for weights in prolog format, also added an option
+ * added support for weights in prolog format, also added an option
--prolog to hfst-fst2txt
* added option --eliminate-flags to hfst-compare
@@ -315,7 +367,7 @@ Noteworthy changes in 3.4.2
* A bug in including zlib in foma fixed.
-* Now ncurses library is used instead if termcap is not found when
+* Now ncurses library is used instead if termcap is not found when
building hfst-xfst.
@@ -348,13 +400,13 @@ Noteworthy changes in 3.4.0
Noteworthy changes in 3.3.15
----------------------------
-* Small bugs fixed in function insert_freely, flag diacritic handling
+* Small bugs fixed in function insert_freely, flag diacritic handling
as well as lexc and pmatch tools
* GNU and bash dependencies removed from scripts to make them more portable
* The epsilon symbol argument in HfstTransducer(FILE*, ...) is made obligatory,
- also a new obligatory argument added that keeps track of the line number
+ also a new obligatory argument added that keeps track of the line number
in the AT&T file
@@ -403,7 +455,7 @@ Noteworthy changes in 3.3.11
* Containment and term negation added to regexp parser
-* Composing intersection supports identity symbols and
+* Composing intersection supports identity symbols and
inverting the composition
* Introduction of semifunctional pmatch2fst parser
@@ -423,7 +475,7 @@ Noteworthy changes in 3.3.10
* hfst-summarize prints at the end how many transducers it has
read in total, making it easier to work with transducer archives
-* hfst-fst2strings bug fixed, now it prints weights with
+* hfst-fst2strings bug fixed, now it prints weights with
options --random -w
Noteworthy changes in 3.3.9
@@ -433,7 +485,7 @@ Noteworthy changes in 3.3.9
* Xfst rules tentatively supported in hfst-regexp2fst
-* hfst-fst2strings does not produce duplicate paths any more
+* hfst-fst2strings does not produce duplicate paths any more
when using foma as backend
* hfst-lexc implementation format defaults to openfst-tropical
@@ -504,7 +556,7 @@ Noteworthy changes in 3.3.1
* More examples in fsmbook-examples included in tests.
-* Extra files created during tests as well as generated autotools files
+* Extra files created during tests as well as generated autotools files
are now ignored by svn.
Noteworthy changes in 3.3.0
@@ -598,7 +650,7 @@ Noteworthy changes in 3.0
* optimized-lookup backend supported in library
* dropped special tools for optimised handling since optimised lookup is
- included in library: hfst-optimised-lookup -> hfst-lookup;
+ included in library: hfst-optimised-lookup -> hfst-lookup;
hfst-lookup-optimize -> hfst-fst2fst
* hfst-proc tool to emulate lt-proc, cg-proc
@@ -665,7 +717,7 @@ Noteworthy changes in release 2.3
Noteworthy changes in release 2.2
---------------------------------
-* Bug fixes in intersecting composition, is_subset, negation,
+* Bug fixes in intersecting composition, is_subset, negation,
* Handling of flag diacritics included in lookup
diff --git a/README b/README
deleted file mode 100644
index 0e4d847..0000000
--- a/README
+++ /dev/null
@@ -1,466 +0,0 @@
-.. -*- mode: rst -*-
-
-==================================================================
- Helsinki Finite-State Technology (library and application suite)
-==================================================================
-
-This package contains a bridging library for multiple FST libraries and toolkits
-and set of tools for processing of finite-state automate especially for
-linguistic systems. HFST library and tools are licenced under GNU GPL licence
-version 3, you may read the full licence in the file named COPYING. The
-authors specified in AUTHORS file may be contacted about licencing issues.
-
-There's an online wiki-based information source about HFST in
-`Kitwiki HFST pages <https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstHome>`_.
-
-Installation
-============
-
-Installation method depends on operating system and the version you want to
-install. For stable versions there exists packages for some of the better
-operating system and package manager combinations. For bleeding edge newest
-versions, development and non-supported operating systems and versions you
-will have to perform `installation from the sources`_.
-
-Installation packages for Debian and Ubuntu
--------------------------------------------
-
-Debian packages for HFST are distributed via `Apertium project
-<http://apertium.projectjj.com/apt/nightly/pool/main/h/hfst/>`_. This folder
-contains debian packages for HFST API library and tool packages.
-The debian packages are experimental; the requirements of debian or ubuntu
-installations are same as main packages. SFST is excluded from the packages
-as it has portability issues with hash_maps and hash_sets. The python packages
-contain binaries for swig-generated Python bindings for HFST.
-
-Binaries for Windows
---------------------
-
-Binaries for Windows are distributed via `Apertium project
-<http://apertium.projectjj.com/win32/nightly/>`_. This folder contains
-ready-compiled HFST library and command line tools. Performing
-`installation from the sources`_ is also possible on Windows with MinGW
-and Cygwin. Python bindings for Windows are currently available only for 64-bit
-Python versions 3.3 and 3.4 on our `download page
-<https://hfst.github.io/downloads/index.html>`_.
-
-Installation for MacPorts
--------------------------
-
-The port tree for HFST and related software can be downloaded from `HFST
-download directory <https://sourceforge.net/projects/hfst/files/hfst/>`_.
-To add this port tree to your macports installation, unpack it to wherever
-and insert that directory to your ``sources.conf``, e.g. in the default
-installation to ``/opt/local/etc/macports/sources.conf``::
-
- [...]
- # If an rsync URL points to a .tar file, a signed .rmd160 must exist next to
- # it on the server and will be used to verify its integrity.
-
- file:///Users/flammie/Sources/hfst-macport/
- rsync://rsync.macports.org/release/tarballs/ports.tar [default]
-
-Now you can install hfst and related software using port commands::
-
- sudo port install hfst
-
-Installation for Gentoo Linux
------------------------------
-
-HFST software is available in *science* overlay. For portage, use::
-
- layman -a science
-
-To add the repository. Then::
-
- emerge -av hfst
-
-to install. Parts of the spell-checking tools are in *Finnish* overlay. If you
-use Paludis, just try to ``cave resolve sci-misc/hfst`` to get the current
-instructions.
-
-Installation for other systems
-------------------------------
-
-For rest of the systems, HFST needs to be installed from the source, the
-traditional GNU/linux way. To begin your installation, you need to start by
-gathering the dependencies, as adviced in the following chapter.
-
-Dependencies
-============
-
-Please note that the dependencies for the library can be set during the compile
-time. Settings that determine dependencies relate to features the resulting
-library will have; e.g. if you *disable* openfst you cannot use weighted
-finite-state automata. If you *enable* foma, you will be able to read foma
-format automata as HFST files and use foma's algorithms to process automata.
-The command ``./configure --help`` lists all features that can be controlled
-with configure switches and whether they are enabled or disabled by default.
-
-Compilation requirements
-------------------------
-
-- To use the OpenFST_ backend (default):
-
- - source code of OpenFST version 1.2.10 is bundled with HFST and included
- by default when building HFST
-
- - compiling against OpenFST library and linking may require recent
- GCC version and pthread and m libraries
-
- - to disable OpenFST support, configure switch ``--without-openfst`` may
- be used (however, this seriously limits the use of HFST)
-
-- To use the SFST_ backend (default):
-
- - the SFST library version 1.4.6g is bundled with HFST and included
- by default when building HFST
-
- - SFST requires readline and ncurses
-
- - The SFST *frontend* a.k.a. the SFST-PL parser a.k.a. ``hfst-sfstpl2fst``
- does **not** require the SFST library to be installed; the library is only
- used for library-stuff like reading SFST format automata.
-
- - to disable SFST backend, configure switch ``--without-sfst`` must be used
-
-- To use the foma_ backend (default):
-
- - the foma library version 0.9.18alpha is bundled with HFST and included
- be default when building HFST
-
- - foma requires editline, termcap (or ncurses) and zlib libraries to compile
-
- - to disable foma backend, configure switch ``--without-foma`` may be used
-
- - the *hfst-xfst* frontend does not require foma binary or foma library
-
-- For proper Unicode support in corpus tools:
-
- - Glib version 2.16 or newer; configure ``--with-unicode-handler=glib``
-
- - Pkg-config 0.14 or newer
-
- - Without unicode libraries the corpus tools may be unable to handle
- operations such as *case-folding* properly (default)
-
- - To compile corpus processing tool ``hfst-proc``, you need to use the
- ``configure`` switch ``--enable-proc`` (or ``--enable-all-tools``)
-
-- To use the Python interface:
-
- - Compiling the bindings requires swig (tested with versions 2.0.4 and 3.0.0)
-
- - *NOTE:* the Python API is not under autotools, you must compile it yourself;
- for more information, see file ``python/README``
-
-Note that if you did install dependent libraries, such as libxml or glib
-to your home directory instead of using your system's package manager
-(or supported default location):
-
-- If you only have a local version of a library, you can use it with
- appropriate LDFLAGS and CXXFLAGS, eg.
- ``./configure LDFLAGS=-L/path/to/local/lib
- --prefix=/path/to/local/installation`` and
- ``make CXXFLAGS=-I/path/to/local/headers``
-
-If you are building a development version you *loaded from the version control
-system*, you must have new brand of GNU development tools installed:
-
- - autoconf >=2.62
-
- - automake >=1.11.1
-
- - libtool >=2.2
-
- - >=gettext-0.17
-
- - GNU tool-chain is also needed with distributed packages if the user wishes to
- modify Makefile.am or configure.ac files.
-
- - Mac OS X users are advised to use MacPorts; Mac OS X 10.6 with
- XCode 2.3.2 at least is not sufficient
-
- - A package loaded from hfst web site does *not* have these requirements
-
-The source codes *loaded from the version controls system* will also require
-parse generator system:
-
-- GNU flex 2.5.35 or compatible and
-
-- GNU bison 1.31 (2.4 suggested) or compatible
-
-- flex 2.5.4-2.5.33 will choke on perfectly valid rules used in hfst
-
-- bison older than 1.31 do not support name-prefix needed for having
- multiple parsers in one library
-
-- A package loaded from hfst web site does *not* have these requirements
-
-- source code loaded from version control system requires them only to
- bootstrap; if you use systems with archaic versions of flex or bison and
- cannot install updates, you might be able to get the needed files from
- somewhere
-
-When running HFST software or using HFST libraries from HFST-enabled software:
-
-- If the executable is dynamically linked (almost always), the operating system
- *must* be able to find hfst libraries
-
-- If you install the libraries in non-standard paths, you need to ensure
- that operating system is aware of this; In linux this may happen by setting
- ``LD_LIBRARY_PATH``, on Mac ``DYLD_LIBRARY_PATH``
-
-- the *hfst-xfst* frontend needs GNU compatible getopt, or basic getopts
- *without* GNU-incompatible getopt installed
-
-- the *hfst-sfstpl2fst* frontend does **not** require SFST libraries or
- binaries, you do **not** need to enable SFST libraries (via the switch
- --with-sfst) in order to compile SFST-PL scripts to HFST automata.
-
-Installation from the sources
------------------------------
-
-INSTALL describes the GNU build system in detail, but for most users the usual::
-
- ./configure
- make
- (as root) make install
-
-should result in a local installation and::
-
- (as root) make uninstall
-
-in its uninstallation.
-
-If you aren't going to be linking to the library after
-building it and don't need to be able to send debugging information, you can
-save a considerable amount of space and memory by doing::
-
- make install-strip
-
-instead of make install. This strips all the symbols from the binaries,
-reducing sizes by a factor of 5-10.
-
-If you would rather install in e.g. your home directory
-(or aren't the system administrator), you can tell ./configure::
-
- ./configure --prefix=$HOME
-
-The HFST library may link to numerous FST handling backends with varying
-licences. If you are going to redistribute the HFST library you compiled, make
-sure there are no clashes in the licences of the linked libraries.
-
-If you are checking out the development versions from SVN you must first create
-and install the necessary autotools files from the host system:
-
- autoreconf -i
-
-It is common practice to keep `generated files out of version control
-<http://www.gnu.org/software/automake/manual/automake.html#CVS>`_.
-
-For further installation instruction refer to file ``INSTALL``, which contains
-the standard installation instructions for GNU autoconf based software.
-
-If you are compiling HFST from source on Windows with *MinGW*, use the switch
-``--enable-mingw`` when running ``./configure``.
-Warning: Using this option with *Cygwin* will cause compilation errors.
-
-Troubleshooting
-===============
-
-In this section we list the errors that pop up commonly on `our support channel
-<irc://FreeNode/#hfst>`_ or in `our issue tracker
-<https://github.com/hfst/hfst/issues>`_,
-
-\::malloc has not been declared
--------------------------------
-
-**During the compilation** errors of form::
-
- /usr/include/c++/4.3/cstdlib:124: error: '::malloc' has not been declared
-
-or::
-
- your configure failed to find malloc, check README for further instructions
-
-are mosts often caused by broken library installation. The simplest solution
-in Linux-based platforms is ldconfig::
-
- ldconfig -v
-
-This is actually told and performed by the autotools libtool library
-installation, but it's easy to miss. It looks like this::
-
- Libraries have been installed in:
- /usr/local/lib
-
- If you ever happen to want to link against installed libraries
- in a given directory, LIBDIR, you must either use libtool, and
- specify the full pathname of the library, or use the '-LLIBDIR'
- flag during linking and do at least one of the following:
- - add LIBDIR to the 'LD_LIBRARY_PATH' environment variable
- during execution
- - add LIBDIR to the 'LD_RUN_PATH' environment variable
- during linking
- - use the '-Wl,-rpath -Wl,LIBDIR' linker flag
- - have your system administrator add LIBDIR to '/etc/ld.so.conf'
-
- See any operating system documentation about shared libraries for
- more information, such as the ld(1) and ld.so(8) manual pages.
-
-If you installed a library on non-standard path, or installed it to the
-default ``/usr/local/lib``, but your variant of Linux doesn't support libraries
-there, you may need to set it up and/or ldconfig the directory explicitly::
-
- export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
- ldconfig -v -n /usr/local/lib
-
-If all else fails, try installing the library to wherever your blend of Linux
-installs all its libraries, such as /usr/lib.
-
-See also:
-<http://nerdland.net/unstumping-the-internet/malloc-has-not-been-declared/> for
-the gory details.
-
-Error while loading shared libraries: libhfst.so.0: cannot open shared object file: No such file or directory
--------------------------------------------------------------------------------------------------------------
-
-**After installing HFST and running programs**, the installed programs should
-on most systems be able to find and use the shared libraries that just got
-installed alongside the programs that were installed on the same go, but this
-is not always the case. Typically on first installation of the HFST library
-or after a major version update of HFST library, the system may not know about
-it. To fix this, you must run ``ldconfig`` on GNU systems. To ensure proper
-linking, use ``ldconfig -v`` to get a print out of known libraries, the
-listing should include libhfst.so indicating the current version.
-
-If this is the first time you install a library on your system by hand, it may
-happen on some systems that the library configuration does not include that
-directory at all. Notably on ubuntu it seems that /usr/local/lib is not one of
-library directories, and that is the default target for manually installed
-libraries. Either fix this by doing ``./configure --prefix=/usr`` or check
-your distributions manuals on how to set up new library directories. Same
-applies for libraries installed to e.g. home directory.
-
-See also the previous error description.
-
-chmod: cannot access 'scripts/hfst-foma-wrapper.sh': No such file or directory
-------------------------------------------------------------------------------
-
-**During the configuration phase.** This is a case of autoconf being silly; the
-commands for creating a script and making it executable from autoconf are ran
-in parallel but must be ran once per file and autoconf just doesn't keep track
-of the files it creates. The script for doing this is copied to configure.ac
-from autoconf manual so it supposably will always work correctly...
-
-fatal error: htwolcpre1.h: No such file or directory
-----------------------------------------------------
-
-**During the compilation phase**, an error message including things like::
-
- scanner1.ll:22:27: fatal error: htwolcpre1.h: No such file or directory
- compilation terminated.
- Makefile:1029: recipe for target `scanner1.o' failed
-
-indicates that the header files generated by flex/yacc have the extension `.hh`
-instead of `.h`. This error has been encountered at least on some versions of
-Cygwin. Currently, the best solution is to manually rename each
-flex/yacc-generated header file of the form `foo.hh` as `foo.h`. You probably
-have to do this iteratively after each error message of this type.
-
-....libs/lt-hfst-strings2fst: Incorrect utf-8 coding
-----------------------------------------------------
-
-**During the make check phase**. This indicates that a test that tests for
-expected failure fails expectedly, which is also indicated by a green word
-*PASS* or *XFAIL* on the next line. This is expected behaviour and not an
-indication of a bug. If there is a bug effecting that or other tests in the
-test suite, it will be indicated by a line starting with word *FAIL* or *XPASS*
-in red colour.
-
-Same applies for other messages printed during make check phase. The cases where
-something actually fails will be clearly stated in the end of the test suite by
-a message such as::
-
- ======================================
- 2 of 36 tests failed
- Please report to hfst-bugs at helsinki.fi
- ======================================
-
-These errors can be reported either to the stated mail address or the `HFST
-bug tracker in sf.net infrastructure
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>`_.
-
-cat: hvVqf:o:l:u:: No such file or directory
---------------------------------------------
-**During use of bash-based scripts**, an error message including things like::
-
-
- cat: -l: No such file or directory
- cat: version,quiet,format:,output:,latin1::,utf8::: No such file or directory
- cat: -n: No such file or directory
- cat: hfst-lexc: No such file or directory
- cat: --: No such file or directory
-
-indicate that script is trying to use Mac OS X's getopt as if it was GNU getopt.
-However default getopt in Mac OS X does not work at all like GNU getopt.
-Easiest solution is to install working getopt, e.g. by using MacPorts::
-
- sudo port install getopt
-
-The newer versions of bash scripts detect Mac OS X's getopt and fallback to
-using getopts. Note that getopts does not support long options and filenames
-must be last parameters on commandline with it, so its use is strongly
-discouraged.
-
-libc++-abi.dylib: terminate called throwing an exception
---------------------------------------------------------
-
-**During program execution** *(Mac OS X only)*, errors of form::
-
- terminate called throwing an exception
- Abort trap: 6
-
-Can be caused by, just about any exceptional situation that does not have
-specific handler. On Linux it will read::
-
- terminate called after throwing an instance of 'ImplementationTypeNotAvailableException'
-
-And then you'll know that this specific exception is about backend that was disabled during `configure` phase. Or it might read::
-
- terminate called after throwing an instance of 'UndefinedSymbolPairsFound'
-
-And you'd know it's something with the alphabet. But OS X won't tell us this. So
-it is an unexpected error situation. Usual suspects are still: missing library
-in configure, empty file, reading error, writing error...
-
-
-Further information
-===================
-
-The `Hfst wiki site
-<https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstHome>`_ contains further
-details of the HFST system.
-
-Reporting bugs
-==============
-
-Bugs can be reported via email to `HFST team bug mail address
-<hfst-bugs at helsinki.fi>`_, or preferably to `HFST's bug tracking system
-at Github
-<https://github.com/hfst/hfst/issues>`_
-When reporting, please include at least following:
-
-* version of software used, if command-line tool (hfst-toolname --version)
-
-* version of hfst-library, if possible
-
-* steps to reproduce, attach or all related files if possible
-
-* information about platform used (e.g. uname -a)
-
-.. _libxml2: http://www.xmlsoft.org/
-.. _libreadline: http://www.gnu.org/software/readline/
-
-.. vim: set ft=rst:
diff --git a/README b/README
new file mode 120000
index 0000000..92cacd2
--- /dev/null
+++ b/README
@@ -0,0 +1 @@
+README.rst
\ No newline at end of file
diff --git a/README b/README.rst
similarity index 96%
copy from README
copy to README.rst
index 0e4d847..1de4599 100644
--- a/README
+++ b/README.rst
@@ -10,8 +10,10 @@ linguistic systems. HFST library and tools are licenced under GNU GPL licence
version 3, you may read the full licence in the file named COPYING. The
authors specified in AUTHORS file may be contacted about licencing issues.
-There's an online wiki-based information source about HFST in
-`Kitwiki HFST pages <https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstHome>`_.
+For additional information, see the project page `<https://hfst.github.io>`_.
+
+.. image:: https://travis-ci.org/hfst/hfst.svg
+ :target: https://travis-ci.org/hfst/hfst
Installation
============
@@ -22,24 +24,24 @@ operating system and package manager combinations. For bleeding edge newest
versions, development and non-supported operating systems and versions you
will have to perform `installation from the sources`_.
-Installation packages for Debian and Ubuntu
+Installation packages for Debian and Ubuntu
-------------------------------------------
-Debian packages for HFST are distributed via `Apertium project
-<http://apertium.projectjj.com/apt/nightly/pool/main/h/hfst/>`_. This folder
+Debian packages for HFST are distributed via `Apertium project
+<http://apertium.projectjj.com/apt/nightly/pool/main/h/hfst/>`_. This folder
contains debian packages for HFST API library and tool packages.
The debian packages are experimental; the requirements of debian or ubuntu
installations are same as main packages. SFST is excluded from the packages
as it has portability issues with hash_maps and hash_sets. The python packages
-contain binaries for swig-generated Python bindings for HFST.
+contain binaries for swig-generated Python bindings for HFST.
Binaries for Windows
--------------------
-Binaries for Windows are distributed via `Apertium project
+Binaries for Windows are distributed via `Apertium project
<http://apertium.projectjj.com/win32/nightly/>`_. This folder contains
-ready-compiled HFST library and command line tools. Performing
-`installation from the sources`_ is also possible on Windows with MinGW
+ready-compiled HFST library and command line tools. Performing
+`installation from the sources`_ is also possible on Windows with MinGW
and Cygwin. Python bindings for Windows are currently available only for 64-bit
Python versions 3.3 and 3.4 on our `download page
<https://hfst.github.io/downloads/index.html>`_.
@@ -93,7 +95,7 @@ Please note that the dependencies for the library can be set during the compile
time. Settings that determine dependencies relate to features the resulting
library will have; e.g. if you *disable* openfst you cannot use weighted
finite-state automata. If you *enable* foma, you will be able to read foma
-format automata as HFST files and use foma's algorithms to process automata.
+format automata as HFST files and use foma's algorithms to process automata.
The command ``./configure --help`` lists all features that can be controlled
with configure switches and whether they are enabled or disabled by default.
@@ -155,7 +157,7 @@ Compilation requirements
for more information, see file ``python/README``
Note that if you did install dependent libraries, such as libxml or glib
-to your home directory instead of using your system's package manager
+to your home directory instead of using your system's package manager
(or supported default location):
- If you only have a local version of a library, you can use it with
@@ -215,7 +217,7 @@ When running HFST software or using HFST libraries from HFST-enabled software:
*without* GNU-incompatible getopt installed
- the *hfst-sfstpl2fst* frontend does **not** require SFST libraries or
- binaries, you do **not** need to enable SFST libraries (via the switch
+ binaries, you do **not** need to enable SFST libraries (via the switch
--with-sfst) in order to compile SFST-PL scripts to HFST automata.
Installation from the sources
@@ -262,7 +264,7 @@ It is common practice to keep `generated files out of version control
For further installation instruction refer to file ``INSTALL``, which contains
the standard installation instructions for GNU autoconf based software.
-If you are compiling HFST from source on Windows with *MinGW*, use the switch
+If you are compiling HFST from source on Windows with *MinGW*, use the switch
``--enable-mingw`` when running ``./configure``.
Warning: Using this option with *Cygwin* will cause compilation errors.
@@ -319,7 +321,7 @@ there, you may need to set it up and/or ldconfig the directory explicitly::
If all else fails, try installing the library to wherever your blend of Linux
installs all its libraries, such as /usr/lib.
-See also:
+See also:
<http://nerdland.net/unstumping-the-internet/malloc-has-not-been-declared/> for
the gory details.
@@ -363,8 +365,8 @@ fatal error: htwolcpre1.h: No such file or directory
compilation terminated.
Makefile:1029: recipe for target `scanner1.o' failed
-indicates that the header files generated by flex/yacc have the extension `.hh`
-instead of `.h`. This error has been encountered at least on some versions of
+indicates that the header files generated by flex/yacc have the extension `.hh`
+instead of `.h`. This error has been encountered at least on some versions of
Cygwin. Currently, the best solution is to manually rename each
flex/yacc-generated header file of the form `foo.hh` as `foo.h`. You probably
have to do this iteratively after each error message of this type.
@@ -388,9 +390,9 @@ a message such as::
Please report to hfst-bugs at helsinki.fi
======================================
-These errors can be reported either to the stated mail address or the `HFST
-bug tracker in sf.net infrastructure
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>`_.
+These errors can be reported either to the stated mail address or the `HFST bug
+tracker on Github
+<https://github.com/hfst/hfst/issues>`_.
cat: hvVqf:o:l:u:: No such file or directory
--------------------------------------------
diff --git a/autogen.sh b/autogen.sh
index 13652a3..ab2b47d 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -584,7 +584,7 @@ else
_version="0.0.0"
fi
$ECHO "Found GNU Automake version $_version"
- version_check "$AUTOMAKE_VERSION" "$_version"
+ version_check "$AUTOMAKE_VERSION" "$_version"
if [ $? -ne 0 ] ; then
_report_error=yes
fi
@@ -686,7 +686,7 @@ else
_version="0.0.0"
fi
$ECHO "Found GNU Libtool version $_version"
- version_check "$LIBTOOL_VERSION" "$_version"
+ version_check "$LIBTOOL_VERSION" "$_version"
if [ $? -ne 0 ] ; then
_report_error=yes
fi
diff --git a/back-ends/dlfcn/dlfcn.c b/back-ends/dlfcn/dlfcn.c
index 43cc5d1..335f87b 100644
--- a/back-ends/dlfcn/dlfcn.c
+++ b/back-ends/dlfcn/dlfcn.c
@@ -214,7 +214,7 @@ void *dlopen( const char *file, int mode )
* to UNIX's search paths (start with system folders instead of current
* folder).
*/
- hModule = LoadLibraryEx( (LPSTR) lpFileName, NULL,
+ hModule = LoadLibraryEx( (LPSTR) lpFileName, NULL,
LOAD_WITH_ALTERED_SEARCH_PATH );
/* If the object was loaded with RTLD_GLOBAL, add it to list of global
diff --git a/back-ends/dlfcn/dlfcn.h b/back-ends/dlfcn/dlfcn.h
index ba3225f..8d8c97d 100644
--- a/back-ends/dlfcn/dlfcn.h
+++ b/back-ends/dlfcn/dlfcn.h
@@ -21,16 +21,16 @@
* This is a modified copy of Ramiro Polla's original library.
* Modifications were made so that this library can be used as a part
* of the HFST library.
- * The modifications are indicated in comments.
+ * The modifications are indicated in comments.
* ****************
*/
#ifndef DLFCN_H
-#define DLFCN_H
+#define DLFCN_H
/* HFST addition : The following three lines. */
-#ifdef __cplusplus
-extern "C" {
+#ifdef __cplusplus
+extern "C" {
#endif
/* POSIX says these are implementation-defined.
@@ -57,8 +57,8 @@ char *dlerror( void );
/* HFST addition: The following three lines. */
-#ifdef __cplusplus
-}
-#endif
+#ifdef __cplusplus
+}
+#endif
#endif /* DLFCN_H */
diff --git a/back-ends/foma/CHANGELOG b/back-ends/foma/CHANGELOG
index 2468a36..b75a4d8 100644
--- a/back-ends/foma/CHANGELOG
+++ b/back-ends/foma/CHANGELOG
@@ -30,7 +30,7 @@
- flookup now applies in chain (a virtual compose) if there are several transducers in a file, except if the [-a] flag is given, in which case flookup looks for an output in each transducer until it finds one (simulating priority union .P.)
- changes and bug fixes in application code/flag diacritic handling
- lexc now accepts forced alignments with zeroes and info strings
-- \r (carriage return) awarness added
+- \r (carriage return) awarness added
- adds support for the construct regex @re"regexfile.txt" (assumes file has one regular expression on one line)
- fixes argument order of interface commands compose net and concatenate net
- adds functions in API to read in multiple nets from one file through an iterator fsm_read_binary_file_multiple_init(infilename)/fsm_read_binary_file_multiple(rh))
@@ -49,7 +49,7 @@
0.9.13alpha (20101025)
- Various memory-management improvements throughout
-- Adds the commands "read text" and "read spaced-text" for building automata/transducers from word lists. The functionality can be embedded in regexes with @txt"filename" and @stxt"filename". From the API, the functions fsm_read_text_file() and fsm_read_spaced_text_file(), reads files and builds trie FSMs from the lists
+- Adds the commands "read text" and "read spaced-text" for building automata/transducers from word lists. The functionality can be embedded in regexes with @txt"filename" and @stxt"filename". From the API, the functions fsm_read_text_file() and fsm_read_spaced_text_file(), reads files and builds trie FSMs from the lists
- Adds the external utility flookup, which reads words from stdin and applies them to a transducer given in a file, and prints output to stdout
- Adds the command "substitute defined", and the corresponding fsm_substitute_label() in the API.
- Improved prolog file reading and writing
@@ -95,7 +95,7 @@
- Minor bugfix in "test unambiguous"/_isunambiguous()
0.9.8alpha (20090604)
-- Added option to load confusion matrices (load cmatrix filename) that specify costs for minimum edit distance matching and attaches to a network. The command "print cmatrix" prints the matrix associated with the network on top of the stack. These are used whenever "apply med" is called. If no matrix is specified, the default distance is Levenshtein.
+- Added option to load confusion matrices (load cmatrix filename) that specify costs for minimum edit distance matching and attaches to a network. The command "print cmatrix" prints the matrix associated with the network on top of the stack. These are used whenever "apply med" is called. If no matrix is specified, the default distance is Levenshtein.
- Added the global variables med-cutoff and med-limit to control the med search
- Minor bugfixes in fsm_compose()
- Changes in dot file output (print dot, print dot > filename) and "view net"
@@ -131,13 +131,13 @@
--
0.9.2alpha (20090111)
-- Determinization now uses much less temporary memory without sacrificing efficiency.
+- Determinization now uses much less temporary memory without sacrificing efficiency.
The memory use caused severe problems for compiling very large lexicons or determinizing
large automata with limited memory.
- Added a "-r" command line option for starting foma without readline (mainly for Win32 version)
-- Fixed a serious bug in the ternary `[A,B,C] substitution operator and added the interface command
+- Fixed a serious bug in the ternary `[A,B,C] substitution operator and added the interface command
"substitute symbol B for A"
- Added a slight optimization in directed replacements
diff --git a/back-ends/foma/Makefile.am b/back-ends/foma/Makefile.am
index 90a9d4d..8ec4ce6 100644
--- a/back-ends/foma/Makefile.am
+++ b/back-ends/foma/Makefile.am
@@ -28,7 +28,7 @@ hfst_foma_LDFLAGS += -ltermcap
else
if HAVE_NCURSES
hfst_foma_LDFLAGS += -lncurses
-else
+else
if HAVE_CURSES
hfst_foma_LDFLAGS += -lcurses
endif
@@ -41,4 +41,4 @@ hfst_foma_SOURCES=foma.c stack.c iface.c lex.interface.c lex.cmatrix.c
AM_CPPFLAGS += -DZLIB
endif
-EXTRA_DIST=foma.h fomalibconf.h fomalib.h lexc.h regex.h
+EXTRA_DIST=foma.h fomalibconf.h fomalib.h lexc.h regex.h
diff --git a/back-ends/foma/README.cmatrix b/back-ends/foma/README.cmatrix
index d67ea27..f829bb6 100644
--- a/back-ends/foma/README.cmatrix
+++ b/back-ends/foma/README.cmatrix
@@ -1,4 +1,4 @@
-Since version 0.9.8alpha, foma allows attaching a confusion matrix specification to a network. The command "read cmatrix <filename>" read a confusion matrix and attaches is to the top network on the stack. Subsequent "apply med" commands will use this matrix in determining the minimum cost approximate match to a word. If no confusion matrix is specified, Levenshtein distance is used (insert = substitute = delete = 1).
+Since version 0.9.8alpha, foma allows attaching a confusion matrix specification to a network. The command "read cmatrix <filename>" read a confusion matrix and attaches is to the top network on the stack. Subsequent "apply med" commands will use this matrix in determining the minimum cost approximate match to a word. If no confusion matrix is specified, Levenshtein distance is used (insert = substitute = delete = 1).
The command "print cmatrix" also prints out the confusion matrix attached to the top network in tabular format.
diff --git a/back-ends/foma/README.symbols b/back-ends/foma/README.symbols
index c65176c..3976f32 100644
--- a/back-ends/foma/README.symbols
+++ b/back-ends/foma/README.symbols
@@ -8,12 +8,12 @@ In addition the following Unicode symbols are reserved and need to be escaped as
× 303 227 MULTIPLICATION SIGN U+00D7 .x. :
Σ 316 243 GREEK CAPITAL LETTER SIGMA U+03A3 ?
ε 316 265 GREEK SMALL LETTER EPSILON U+03B5 [] 0
-→ 342 206 222 RIGHTWARDS ARROW U+2192
-↔ 342 206 224 LEFT RIGHT ARROW U+2194
-∀ 342 210 200 FOR ALL U+2200
-∃ 342 210 203 THERE EXISTS U+2203
+→ 342 206 222 RIGHTWARDS ARROW U+2192
+↔ 342 206 224 LEFT RIGHT ARROW U+2194
+∀ 342 210 200 FOR ALL U+2200
+∃ 342 210 203 THERE EXISTS U+2203
∅ 342 210 205 EMPTY SET U+2205 \?
-∈ 342 210 210 ELEMENT OF U+2208
+∈ 342 210 210 ELEMENT OF U+2208
∘ 342 210 230 RING OPERATOR U+2218 .o.
∥ 342 210 245 PARALLEL TO U+2225 <>
∧ 342 210 247 LOGICAL AND U+2227 &
diff --git a/back-ends/foma/apply.c b/back-ends/foma/apply.c
index bfa9452..4512152 100644
--- a/back-ends/foma/apply.c
+++ b/back-ends/foma/apply.c
@@ -79,22 +79,22 @@ void apply_set_show_flags(struct apply_handle *h, int value) {
void apply_set_print_space(struct apply_handle *h, int value) {
h->print_space = value;
- h->space_symbol = strdup(" ");
+ h->space_symbol = xxstrdup(" ");
}
void apply_set_separator(struct apply_handle *h, char *symbol) {
- h->separator = strdup(symbol);
+ h->separator = xxstrdup(symbol);
}
void apply_set_epsilon(struct apply_handle *h, char *symbol) {
xxfree(h->epsilon_symbol);
- h->epsilon_symbol = strdup(symbol);
+ h->epsilon_symbol = xxstrdup(symbol);
(h->sigs+EPSILON)->symbol = h->epsilon_symbol;
(h->sigs+EPSILON)->length = strlen(h->epsilon_symbol);
}
void apply_set_space_symbol(struct apply_handle *h, char *space) {
- h->space_symbol = strdup(space);
+ h->space_symbol = xxstrdup(space);
h->print_space = 1;
}
@@ -160,7 +160,7 @@ char *apply_random_words(struct apply_handle *h) {
char *apply_random_lower(struct apply_handle *h) {
apply_clear_flags(h);
- h->mode = DOWN + ENUMERATE + LOWER + RANDOM;
+ h->mode = DOWN + ENUMERATE + LOWER + RANDOM;
return(apply_enumerate(h));
}
@@ -211,7 +211,7 @@ void apply_clear(struct apply_handle *h) {
if (h->flagstates != NULL) {
xxfree(h->flagstates);
h->flagstates = NULL;
- }
+ }
apply_clear_index(h);
h->last_net = NULL;
h->iterator = 0;
@@ -247,7 +247,7 @@ char *apply_updown(struct apply_handle *h, char *word) {
char *apply_down(struct apply_handle *h, char *word) {
h->mode = DOWN;
- if (h->index_in) {
+ if (h->index_in) {
h->indexed = 1;
} else {
h->indexed = 0;
@@ -284,8 +284,8 @@ struct apply_handle *apply_init(struct fsm *net) {
h->show_flags = 0;
h->print_space = 0;
h->print_pairs = 0;
- h->separator = strdup(":");
- h->epsilon_symbol = strdup("0");
+ h->separator = xxstrdup(":");
+ h->epsilon_symbol = xxstrdup("0");
h->last_net = net;
h->outstring = xxmalloc(sizeof(char)*DEFAULT_OUTSTRING_SIZE);
h->outstringtop = DEFAULT_OUTSTRING_SIZE;
@@ -515,7 +515,7 @@ void apply_index(struct apply_handle *h, int inout, int densitycutoff, int mem_l
if (j == EPSILON)
(*(indexptr + tp->state_no) + j)->next = NULL;
else
- (*(indexptr + tp->state_no) + j)->next = (*(indexptr + tp->state_no)); /* all tails point to epsilon */
+ (*(indexptr + tp->state_no) + j)->next = (*(indexptr + tp->state_no)); /* all tails point to epsilon */
}
}
}
@@ -704,10 +704,10 @@ int apply_follow_next_arc(struct apply_handle *h) {
}
}
if ((h->gstates+h->curr_ptr)->state_no == (h->gstates+h->curr_ptr+1)->state_no) {
- h->curr_ptr++;
+ h->curr_ptr++;
h->ptr = h->curr_ptr;
- if ((h->gstates+h->curr_ptr)-> target == -1) {
- return 0;
+ if ((h->gstates+h->curr_ptr)-> target == -1) {
+ return 0;
}
continue;
}
@@ -717,7 +717,7 @@ int apply_follow_next_arc(struct apply_handle *h) {
} else {
for (h->curr_ptr = h->ptr; (h->gstates+h->curr_ptr)->state_no == (h->gstates+h->ptr)->state_no && (h->gstates+h->curr_ptr)-> in != -1; (h->curr_ptr)++) {
- /* Select one random arc to follow out of all outgoing arcs */
+ /* Select one random arc to follow out of all outgoing arcs */
if ((h->mode & RANDOM) == RANDOM) {
vcount = 0;
for (h->curr_ptr = h->ptr; (h->gstates+h->curr_ptr)->state_no == (h->gstates+h->ptr)->state_no && (h->gstates+h->curr_ptr)-> in != -1; (h->curr_ptr)++) {
@@ -1007,7 +1007,7 @@ int apply_append(struct apply_handle *h, int cptr, int sym) {
bstring = ""; blen = 0;
}
if (((h->mode) & (UPPER|LOWER)) == UPPER) {
- pstring = astring;
+ pstring = astring;
len = alen;
} else {
pstring = bstring;
@@ -1177,7 +1177,7 @@ void apply_add_sigma_trie(struct apply_handle *h, int number, char *symbol, int
st->signum = number;
} else {
if (st->next == NULL) {
- st->next = xxcalloc(256,sizeof(struct sigma_trie));
+ st->next = xxcalloc(256,sizeof(struct sigma_trie));
st = st->next;
/* store these arrays to free them later */
sta = xxmalloc(sizeof(struct sigma_trie_arrays));
@@ -1207,7 +1207,7 @@ void apply_mark_flagstates(struct apply_handle *h) {
h->flagstates = xxcalloc(BITNSLOTS(h->last_net->statecount), sizeof(uint8_t));
fsm = h->last_net->states;
for (i=0; (fsm+i)->state_no != -1; i++) {
- if ((fsm+i)->target == -1) {
+ if ((fsm+i)->target == -1) {
continue;
}
if ((h->flag_lookup+(fsm+i)->in)->type) {
@@ -1275,7 +1275,7 @@ void apply_create_sigarray(struct apply_handle *h, struct fsm *net) {
if (flag_check(sig->symbol)) {
(h->flag_lookup+sig->number)->type = flag_get_type(sig->symbol);
(h->flag_lookup+sig->number)->name = flag_get_name(sig->symbol);
- (h->flag_lookup+sig->number)->value = flag_get_value(sig->symbol);
+ (h->flag_lookup+sig->number)->value = flag_get_value(sig->symbol);
}
}
apply_mark_flagstates(h);
@@ -1402,12 +1402,12 @@ int apply_check_flag(struct apply_handle *h, int type, char *name, char *value)
return SUCCEED;
}
else if (strcmp(value, flist->value) == 0 && flist->neg == 0) {
- return SUCCEED;
+ return SUCCEED;
} else if (strcmp(value, flist->value) != 0 && flist->neg == 1) {
flist->value = xxstrdup(value);
flist->neg = 0;
return SUCCEED;
- }
+ }
return FAIL;
}
@@ -1491,7 +1491,7 @@ int apply_check_flag(struct apply_handle *h, int type, char *name, char *value)
} else if (strcmp(flist2->value, flist->value) == 0 && flist->neg == flist2->neg) {
return SUCCEED;
}
- return FAIL;
+ return FAIL;
}
fprintf(stderr,"***Don't know what do with flag [%i][%s][%s]\n", type, name, value);
return FAIL;
diff --git a/back-ends/foma/coaccessible.c b/back-ends/foma/coaccessible.c
index 67748e5..f67d54d 100644
--- a/back-ends/foma/coaccessible.c
+++ b/back-ends/foma/coaccessible.c
@@ -129,7 +129,7 @@ struct fsm *fsm_coaccessible(struct fsm *net) {
*(added+(fsm+i)->state_no) = 1;
if ((fsm+i)->target != -1) {
new_arccount++;
- }
+ }
}
}
diff --git a/back-ends/foma/constructions.c b/back-ends/foma/constructions.c
index 198dae2..3d2ab28 100644
--- a/back-ends/foma/constructions.c
+++ b/back-ends/foma/constructions.c
@@ -95,7 +95,7 @@ struct fsm *fsm_letter_machine(struct fsm *net) {
if (innum <= IDENTITY || inlen < 1) {
if (inlen < 1)
currin = "@_EPSILON_SYMBOL_@";
- else
+ else
currin = in;
} else {
strncpy(tmpin, in, utf8skip(in)+1);
@@ -116,7 +116,7 @@ struct fsm *fsm_letter_machine(struct fsm *net) {
out = out+utf8skip(out)+1;
outlen--;
}
- if (i == 0 && steps > 1) {
+ if (i == 0 && steps > 1) {
target = addstate;
addstate++;
}
@@ -268,7 +268,7 @@ struct state_arr *init_state_pointers(struct fsm_state *fsm_state) {
(state_arr+((fsm_state+i)->state_no))->transitions = (fsm_state+i);
sold = (fsm_state+i)->state_no;
}
- }
+ }
return(state_arr);
}
@@ -312,7 +312,7 @@ void triplet_hash_free(struct triplethash *th) {
xxfree(th->triplets);
}
xxfree(th);
- }
+ }
}
void triplet_hash_rehash(struct triplethash *th);
@@ -462,7 +462,7 @@ struct fsm *fsm_intersect(struct fsm *net1, struct fsm *net2) {
if (machine_b->in < 0) continue;
bptr = array+(machine_b->in*sigma2size)+machine_b->out;
bptr->mainloop = mainloop;
- bptr->target = machine_b->target;
+ bptr->target = machine_b->target;
}
/* The main loop where we run the machines in parallel */
@@ -478,7 +478,7 @@ struct fsm *fsm_intersect(struct fsm *net1, struct fsm *net2) {
if ((target_number = triplet_hash_find(th, machine_a->target, bptr->target, 0)) == -1) {
STACK_2_PUSH(bptr->target, machine_a->target);
- target_number = triplet_hash_insert(th, machine_a->target, bptr->target, 0);
+ target_number = triplet_hash_insert(th, machine_a->target, bptr->target, 0);
}
fsm_state_add_arc(current_state, machine_a->in, machine_a->out, target_number, current_final, current_start);
@@ -549,7 +549,7 @@ struct fsm *fsm_compose(struct fsm *net1, struct fsm *net2) {
struct state_arr *point_a, *point_b;
struct triplethash *th;
int mode;
- _Bool *is_flag = NULL;
+ Boolean *is_flag = NULL;
net1 = fsm_minimize(net1);
@@ -576,7 +576,7 @@ struct fsm *fsm_compose(struct fsm *net1, struct fsm *net2) {
for (sig1 = net1->sigma; sig1 != NULL; sig1 = sig1->next) {
if (flag_check(sig1->symbol)) {
flags1 = 1;
- if (sigma_find(sig1->symbol, sig2) == -1) {
+ if (sigma_find(sig1->symbol, sig2) == -1) {
sigma_add(sig1->symbol, sig2);
}
}
@@ -593,7 +593,7 @@ struct fsm *fsm_compose(struct fsm *net1, struct fsm *net2) {
}
}
}
- sigma_sort(net2);
+ sigma_sort(net2);
sigma_sort(net1);
if (flags1 && flags2) {
printf("***Warning: flag-is-epsilon is ON and both networks contain flags in composition. This may yield incorrect results. Set flag-is-epsilon to OFF.\n");
@@ -605,7 +605,7 @@ struct fsm *fsm_compose(struct fsm *net1, struct fsm *net2) {
if (g_flag_is_epsilon) {
/* Create lookup table for quickly checking if a symbol is a flag */
struct sigma *sig1;
- is_flag = xxmalloc(sizeof(_Bool)*(sigma_max(net1->sigma)+1));
+ is_flag = xxmalloc(sizeof(Boolean)*(sigma_max(net1->sigma)+1));
for (sig1 = net1->sigma; sig1 != NULL; sig1=sig1->next) {
if (flag_check(sig1->symbol)) {
*(is_flag+(sig1->number)) = 1;
@@ -618,7 +618,7 @@ struct fsm *fsm_compose(struct fsm *net1, struct fsm *net2) {
fsm_update_flags(net1, YES, NO, UNK, YES, UNK, UNK);
machine_a = net1->states;
- machine_b = net2->states;
+ machine_b = net2->states;
max2sigma = sigma_max(net2->sigma);
@@ -739,7 +739,7 @@ struct fsm *fsm_compose(struct fsm *net1, struct fsm *net2) {
}
/* Treat epsilon outputs on machine a (may include flags) */
- for (machine_a = (point_a+a)->transitions ; machine_a->state_no == a ; machine_a++) {
+ for (machine_a = (point_a+a)->transitions ; machine_a->state_no == a ; machine_a++) {
aout = machine_a->out;
if (aout != EPSILON && g_flag_is_epsilon == 0)
continue;
@@ -756,10 +756,10 @@ struct fsm *fsm_compose(struct fsm *net1, struct fsm *net2) {
if (!g_compose_tristate) {
/* Check A:0 arcs on upper side */
if (aout == EPSILON && mode == 0) {
- /* mode -> 0 */
+ /* mode -> 0 */
if ((target_number = triplet_hash_find(th, machine_a->target, b, 0)) == -1) {
STACK_3_PUSH(0, b, machine_a->target);
- target_number = triplet_hash_insert(th, machine_a->target, b, 0);
+ target_number = triplet_hash_insert(th, machine_a->target, b, 0);
}
fsm_state_add_arc(current_state, ain, EPSILON, target_number, current_final, current_start);
@@ -771,7 +771,7 @@ struct fsm *fsm_compose(struct fsm *net1, struct fsm *net2) {
/* mode -> 1 */
if ((target_number = triplet_hash_find(th, machine_a->target, b, 1)) == -1) {
STACK_3_PUSH(1, b, machine_a->target);
- target_number = triplet_hash_insert(th, machine_a->target, b, 1);
+ target_number = triplet_hash_insert(th, machine_a->target, b, 1);
}
fsm_state_add_arc(current_state, ain, EPSILON, target_number, current_final, current_start);
@@ -840,7 +840,7 @@ struct fsm *fsm_compose(struct fsm *net1, struct fsm *net2) {
return(fsm_coaccessible(net1));
}
-struct mergesigma *add_to_mergesigma(struct mergesigma *msigma, struct sigma *sigma, short presence) {
+struct mergesigma *add_to_mergesigma(struct mergesigma *msigma, struct sigma *sigma, short int presence) {
int number = 0;
if (msigma->number == -1) {
@@ -1004,7 +1004,7 @@ void fsm_merge_sigma(struct fsm *net1, struct fsm *net2) {
equal = 0;
}
continue;
- }
+ }
}
/* Go over both net1 and net2 and replace arc numbers with new mappings */
@@ -1058,7 +1058,7 @@ void fsm_merge_sigma(struct fsm *net1, struct fsm *net2) {
}
new_1_state = xxmalloc(sizeof(struct fsm_state)*(net_adds+net_lines+1));
- for(i=0,j=0; (fsm_state+i)->state_no != -1; i++) {
+ for(i=0,j=0; (fsm_state+i)->state_no != -1; i++) {
if ((fsm_state+i)->in == IDENTITY) {
add_fsm_arc(new_1_state, j, (fsm_state+i)->state_no, (fsm_state+i)->in, (fsm_state+i)->out, (fsm_state+i)->target, (fsm_state+i)->final_state, (fsm_state+i)->start_state);
@@ -1147,7 +1147,7 @@ void fsm_merge_sigma(struct fsm *net1, struct fsm *net2) {
/* We need net_add new lines in fsm_state */
new_2_state = xxmalloc(sizeof(struct fsm_state)*(net_adds+net_lines+1));
- for(i=0,j=0; (fsm_state+i)->state_no != -1; i++) {
+ for(i=0,j=0; (fsm_state+i)->state_no != -1; i++) {
if ((fsm_state+i)->in == IDENTITY) {
add_fsm_arc(new_2_state, j, (fsm_state+i)->state_no, (fsm_state+i)->in, (fsm_state+i)->out, (fsm_state+i)->target, (fsm_state+i)->final_state, (fsm_state+i)->start_state);
@@ -1246,7 +1246,7 @@ void fsm_count(struct fsm *net) {
fsm = net->states;
for (i=0; (fsm+i)->state_no != -1; i++) {
- if ((fsm+i)->state_no > maxstate)
+ if ((fsm+i)->state_no > maxstate)
maxstate = (fsm+i)->state_no;
linecount++;
@@ -1260,7 +1260,7 @@ void fsm_count(struct fsm *net) {
finalcount++;
}
oldstate = (fsm+i)->state_no;
- }
+ }
}
linecount++;
@@ -1403,7 +1403,7 @@ struct fsm *fsm_union(struct fsm *net1, struct fsm *net2) {
struct fsm *fsm_completes(struct fsm *net, int operation) {
struct fsm_state *fsm, *new_fsm;
int i, j, offset, statecount, sigsize, *state_table, sink_state, target, last_sigma = 0, arccount = 0, incomplete;
- short *starts, *finals, *sinks;
+ short int *starts, *finals, *sinks;
/* TODO: this currently relies on that the sigma is gap-free in its numbering */
/* which can't always be counted on, especially when reading external machines */
@@ -1448,7 +1448,7 @@ struct fsm *fsm_completes(struct fsm *net, int operation) {
(fsm+i)->final_state = 0;
} else if ((fsm+i)->final_state == 0) {
(fsm+i)->final_state = 1;
- }
+ }
}
if ((fsm+i)->target != -1)
arccount++;
@@ -1529,7 +1529,7 @@ struct fsm *fsm_completes(struct fsm *net, int operation) {
}
}
/* Add looping arcs from and to sink state */
- for (j=2; j<=last_sigma; j++)
+ for (j=2; j<=last_sigma; j++)
*(state_table+(sink_state*sigsize+j)) = sink_state;
/* Add missing arcs to sink state from all states */
for (i=0; i<statecount; i++) {
@@ -1585,7 +1585,7 @@ struct fsm *fsm_kleene_closure(struct fsm *net, int operation) {
}
net = fsm_minimize(net);
- fsm_count(net);
+ fsm_count(net);
fsm = net->states;
@@ -1594,7 +1594,7 @@ struct fsm *fsm_kleene_closure(struct fsm *net, int operation) {
j = 0;
if (operation == KLEENE_STAR)
add_fsm_arc(new_fsm, j++, 0, EPSILON, EPSILON, 1, 1, 1);
- if (operation == KLEENE_PLUS)
+ if (operation == KLEENE_PLUS)
add_fsm_arc(new_fsm, j++, 0, EPSILON, EPSILON, 1, 0, 1);
laststate = 0;
arccount = 1;
@@ -1636,7 +1636,7 @@ char *fsm_network_to_char(struct fsm *net) {
for (; sigma != NULL && sigma->number != -1 ; sigma = sigma->next) {
sigprev = sigma;
}
- return(strdup(sigprev->symbol));
+ return(xxstrdup(sigprev->symbol));
}
struct fsm *fsm_substitute_label(struct fsm *net, char *original, struct fsm *substitute) {
@@ -1689,7 +1689,7 @@ struct fsm *fsm_substitute_label(struct fsm *net, char *original, struct fsm *su
} else {
subnet2 = fsm_minimize(fsm_cross_product(fsm_symbol(fsm_get_arc_in(inh)),fsm_copy(substitute)));
}
- fsm_construct_add_arc_nums(outh, source, addstate1, EPSILON, EPSILON);
+ fsm_construct_add_arc_nums(outh, source, addstate1, EPSILON, EPSILON);
subh2 = fsm_read_init(subnet2);
while (fsm_get_next_arc(subh2)) {
source = fsm_get_arc_source(subh2);
@@ -1884,7 +1884,7 @@ struct fsm *fsm_cross_product(struct fsm *net1, struct fsm *net2) {
if (((fsm+i)->in == EPSILON) || ((fsm+i)->out == EPSILON))
epsilon = 1;
if (((fsm+i)->in == UNKNOWN) || ((fsm+i)->out == UNKNOWN))
- unknown = 1;
+ unknown = 1;
}
if (epsilon == 1) {
if (sigma_find_number(EPSILON, net1->sigma) == -1) {
@@ -1921,7 +1921,7 @@ struct fsm *fsm_unflatten(struct fsm *net, char *epsilon_sym, char *repeat_sym)
fsm_count(net);
epsilon = sigma_find(epsilon_sym, net->sigma);
- repeat = sigma_find(repeat_sym, net->sigma);
+ repeat = sigma_find(repeat_sym, net->sigma);
even_state = net->states;
@@ -2111,7 +2111,7 @@ int fsm_equivalent(struct fsm *net1, struct fsm *net2) {
a = int_stack_pop();
b = int_stack_pop();
- if ((point_a+a)->final != (point_b+b)->final) {
+ if ((point_a+a)->final != (point_b+b)->final) {
goto not_equivalent;
}
/* Check that all arcs in A have matching arc in B, push new state pair on stack */
@@ -2207,12 +2207,12 @@ struct fsm *fsm_minus(struct fsm *net1, struct fsm *net2) {
b--;
if (b == -1) {
- current_start = 0;
- current_final = (point_a+a)->final;
- } else {
+ current_start = 0;
+ current_final = (point_a+a)->final;
+ } else {
current_start = (a == 0 && b == 0) ? 1 : 0;
- current_final = (((point_a+a)->final == 1) && ((point_b+b)->final == 0)) ? 1 : 0;
- }
+ current_final = (((point_a+a)->final == 1) && ((point_b+b)->final == 0)) ? 1 : 0;
+ }
fsm_state_set_current_state(current_state, current_final, current_start);
@@ -2376,7 +2376,7 @@ struct fsm *fsm_quotient_right(struct fsm *net1, struct fsm *net2) {
struct fsm *fsm_ignore(struct fsm *net1, struct fsm *net2, int operation) {
struct fsm_state *fsm1, *fsm2, *new_fsm;
struct fsm *Result;
- short *handled_states1, *handled_states2;
+ short int *handled_states1, *handled_states2;
int i, j, k, state_add_counter = 0, malloc_size, splices = 0, returns, target, splice_size, start_splice, states1, states2, lines1, lines2, *return_state;
net1 = fsm_minimize(net1);
@@ -2487,13 +2487,13 @@ void fsm_compact(struct fsm *net) {
struct fsm_state *fsm;
struct sigma *sig, *sigprev, *sign;
- _Bool *potential;
+ Boolean *potential;
int i, j, prevstate, numsymbols, in, out, state, target, removable;
fsm = net->states;
numsymbols = sigma_max(net->sigma);
- potential = xxmalloc(sizeof(_Bool)*(numsymbols+1));
+ potential = xxmalloc(sizeof(Boolean)*(numsymbols+1));
checktable = xxmalloc(sizeof(struct checktable)*(numsymbols+1));
for (i=0; i <= numsymbols; i++) {
@@ -2524,7 +2524,7 @@ void fsm_compact(struct fsm *net) {
}
*(potential+j) = 0;
}
- }
+ }
if ((fsm+i)->state_no == -1)
break;
@@ -2569,9 +2569,9 @@ void fsm_compact(struct fsm *net) {
j++;
}
else if (*(potential+in) == 1 && in > 2) {
- i++;
+ i++;
} else {
- i++;
+ i++;
j++;
}
} while ((fsm+i)->state_no != -1);
@@ -2589,7 +2589,7 @@ void fsm_compact(struct fsm *net) {
sigprev = sig;
sign = sig->next;
}
- }
+ }
xxfree(potential);
xxfree(checktable);
sigma_cleanup(net,0);
@@ -2603,7 +2603,7 @@ int fsm_symbol_occurs(struct fsm *net, char *symbol, int side) {
return 0;
}
for (i=0, fsm = net->states; (fsm+i)->state_no != -1; i++) {
- if (side == M_UPPER && (fsm+i)->in == sym)
+ if (side == M_UPPER && (fsm+i)->in == sym)
return 1;
if (side == M_LOWER && (fsm+i)->out == sym)
return 1;
@@ -2619,7 +2619,7 @@ struct fsm *fsm_equal_substrings(struct fsm *net, struct fsm *left, struct fsm *
/* every X occurring in different substrings ... left X right ... is identical. */
/* Caveat: there is no reliable termination condition for the loop that extracts */
- /* identities. This means that if run on languages where there are potentially */
+ /* identities. This means that if run on languages where there are potentially */
/* infinite-length identical delimited substrings, it will not terminate. */
/* For example: _eq(l a* r l a* r, l , r) will not terminate. */
@@ -2661,7 +2661,7 @@ struct fsm *fsm_equal_substrings(struct fsm *net, struct fsm *left, struct fsm *
/* 5. Leq = Cleanup(Leq) */
/* Cleanup removes LB RB sequences and, at the same time filters out */
- /* any strings where we find both LB RB and LB X RB where X is not 0. */
+ /* any strings where we find both LB RB and LB X RB where X is not 0. */
/* since we know such sequences could not possibly be identical */
/* Cleanup is implemented by composing Leq with */
/* \LB* [LB:0 RB:0 \LB*]* | ~$[LB RB] */
@@ -2835,7 +2835,7 @@ struct fsm *fsm_left_rewr(struct fsm *net, struct fsm *rewr) {
if (innum == relabelin) {
seensource = 1;
if (fsm_read_is_final(inh, currstate)) {
- outnum = relabelout;
+ outnum = relabelout;
}
}
fsm_construct_add_arc_nums(outh, fsm_get_arc_source(inh), fsm_get_arc_target(inh), innum, outnum);
@@ -3015,9 +3015,6 @@ struct fsm *fsm_mark_fsm_tail(struct fsm *net, struct fsm *marker) {
return(newnet);
}
-#ifndef ORIGINAL
-struct fsm *fsm_context_restrict(struct fsm *X, struct fsmcontexts *LR);
-#else
struct fsm *fsm_context_restrict(struct fsm *X, struct fsmcontexts *LR) {
struct fsm *Var, *Notvar, *UnionL, *UnionP, *Result, *Word;
@@ -3085,7 +3082,6 @@ struct fsm *fsm_context_restrict(struct fsm *X, struct fsmcontexts *LR) {
fsm_clear_contexts(pairs);
return(Result);
}
-#endif
struct fsm *fsm_flatten(struct fsm *net, struct fsm *epsilon) {
struct fsm *newnet;
@@ -3103,7 +3099,7 @@ struct fsm *fsm_flatten(struct fsm *net, struct fsm *epsilon) {
fsm_destroy(epsilon);
return NULL;
}
- epssym = strdup(fsm_get_arc_in(eps));
+ epssym = xxstrdup(fsm_get_arc_in(eps));
fsm_read_done(eps);
outh = fsm_construct_init(net->name);
@@ -3112,14 +3108,14 @@ struct fsm *fsm_flatten(struct fsm *net, struct fsm *epsilon) {
fsm_construct_copy_sigma(outh, net->sigma);
while (fsm_get_next_arc(inh)) {
- target = fsm_get_arc_target(inh);
+ target = fsm_get_arc_target(inh);
in = fsm_get_arc_num_in(inh);
out = fsm_get_arc_num_out(inh);
- if (in == EPSILON || out == EPSILON) {
+ if (in == EPSILON || out == EPSILON) {
instring = fsm_get_arc_in(inh);
outstring = fsm_get_arc_out(inh);
if (in == EPSILON) { instring = epssym; }
- if (out == EPSILON) { outstring = epssym; }
+ if (out == EPSILON) { outstring = epssym; }
fsm_construct_add_arc(outh, fsm_get_arc_source(inh), maxstate, instring, instring);
fsm_construct_add_arc(outh, maxstate, target, outstring, outstring);
diff --git a/back-ends/foma/define.c b/back-ends/foma/define.c
index a79e7f4..a83f599 100644
--- a/back-ends/foma/define.c
+++ b/back-ends/foma/define.c
@@ -36,14 +36,16 @@ int remove_defined(struct defined_networks *def, char *string) {
int exists = 0;
/* Undefine all */
if (string == NULL) {
- for (d = def; d != NULL; d = d_next) {
- d_next = d->next;
- fsm_destroy(d->net);
- xxfree(d->name);
- }
- return 0;
+ for (d = def; d != NULL; d = d_next) {
+ d_next = d->next;
+ if (d->net != NULL)
+ fsm_destroy(d->net);
+ if (d->name != NULL)
+ xxfree(d->name);
+ }
+ return 0;
}
- d_prev = NULL;
+ d_prev = NULL;
for (d = def; d != NULL; d_prev = d, d = d->next) {
if (d->name != NULL && strcmp(d->name, string) == 0) {
exists = 1;
@@ -65,8 +67,9 @@ int remove_defined(struct defined_networks *def, char *string) {
} else {
fsm_destroy(d->net);
xxfree(d->name);
- d->next = NULL;
- d->name = NULL;
+ d->next = NULL;
+ d->name = NULL;
+ d->net = NULL;
}
} else {
fsm_destroy(d->net);
diff --git a/back-ends/foma/determinize.c b/back-ends/foma/determinize.c
index 30f69f7..9e267be 100644
--- a/back-ends/foma/determinize.c
+++ b/back-ends/foma/determinize.c
@@ -31,7 +31,7 @@
static int fsm_linecount, num_states, num_symbols, epsilon_symbol, *single_sigma_array, *double_sigma_array, limit, num_start_states, op;
-static _Bool *finals, deterministic, numss;
+static Boolean *finals, deterministic, numss;
struct e_closure_memo {
int state;
@@ -151,7 +151,7 @@ int fsm_isstarfree(struct fsm *net) {
if (v == (curr_ptr+1)->state_no) {
ptr_stack_push(curr_ptr+1);
}
- if (*(dfs_map+vp) == DFS_WHITE) {
+ if (*(dfs_map+vp) == DFS_WHITE) {
curr_ptr = (state_array+vp)->transitions;
goto nopop;
}
@@ -303,7 +303,7 @@ static struct fsm *fsm_subset(struct fsm *net, int operation) {
if (operation == SUBSET_TEST_STAR_FREE) {
mainloop++;
if ((U = e_closure(j)) != -1) {
- single_symbol_to_symbol_pair(minsym, &symbol_in, &symbol_out);
+ single_symbol_to_symbol_pair(minsym, &symbol_in, &symbol_out);
fsm_state_add_arc(T, symbol_in, symbol_out, U, (T_ptr+T)->finalstart, T == 0 ? 1 : 0);
if (star_free_mark == 1) {
//fsm_state_add_arc(T, maxsigma, maxsigma, U, (T_ptr+T)->finalstart, T == 0 ? 1 : 0);
@@ -528,7 +528,7 @@ static int initial_e_closure(struct fsm *net) {
struct fsm_state *fsm;
int i,j;
- finals = xxcalloc(num_states, sizeof(_Bool));
+ finals = xxcalloc(num_states, sizeof(Boolean));
num_start_states = 0;
fsm = net->states;
@@ -581,7 +581,7 @@ static void memoize_e_closure(struct fsm_state *fsm) {
state = (fsm+i)->state_no;
if (state != laststate) {
- if (!int_stack_isempty()) {
+ if (!int_stack_isempty()) {
deterministic = 0;
ptr = e_closure_memo+laststate;
ptr->target = e_closure_memo+int_stack_pop();
@@ -645,7 +645,7 @@ static void sigma_to_pairs(struct fsm *net) {
fsm = net->states;
- epsilon_symbol = -1;
+ epsilon_symbol = -1;
maxsigma = sigma_max(net->sigma);
maxsigma++;
@@ -750,7 +750,7 @@ INLINE static int hashf(int *set, int setsize) {
unsigned int hashval, sum = 0;
hashval = 6703271;
for (i = 0; i < setsize; i++) {
- hashval = (unsigned int) (*(set+i) + 1103 * setsize) * hashval;
+ hashval = (unsigned int) (*(set+i) + 1103 * setsize) * hashval;
sum += *(set+i) + i;
}
hashval = hashval + sum * 31;
@@ -772,8 +772,8 @@ static unsigned int move_set(int *set, int setsize) {
return(old_offset);
}
-static int nhash_insert(int hashval, int *set, int setsize) {
- struct nhash_list *tableptr;
+static int nhash_insert(int hashval, int *set, int setsize) {
+ struct nhash_list *tableptr;
int i, fs = 0;
current_setnum++;
diff --git a/back-ends/foma/dynarray.c b/back-ends/foma/dynarray.c
index 50838b8..a5f96ce 100644
--- a/back-ends/foma/dynarray.c
+++ b/back-ends/foma/dynarray.c
@@ -36,7 +36,7 @@ struct foma_reserved_symbols {
static size_t current_fsm_size;
static unsigned int current_fsm_linecount, current_state_no, current_final, current_start, current_trans, num_finals, num_initials, arity, statecount;
-static _Bool is_deterministic, is_epsilon_free;
+static Boolean is_deterministic, is_epsilon_free;
static struct fsm_state *current_fsm_head;
static unsigned int mainloop, ssize, arccount;
@@ -141,7 +141,7 @@ void fsm_state_add_arc(int state_no, int in, int out, int target, int final_stat
cptr->out = out;
cptr->target = target;
cptr->final_state = final_state;
- cptr->start_state = start_state;
+ cptr->start_state = start_state;
current_fsm_linecount++;
}
@@ -196,7 +196,7 @@ void fsm_construct_check_size(struct fsm_construct_handle *handle, int state_no)
struct fsm_state_list *sl;
oldsize = handle->fsm_state_list_size;
if (oldsize <= state_no) {
- newsize = next_power_of_two(state_no);
+ newsize = next_power_of_two(state_no);
handle->fsm_state_list = xxrealloc(handle->fsm_state_list, newsize*sizeof(struct fsm_state_list));
handle->fsm_state_list_size = newsize;
sl = handle->fsm_state_list;
@@ -321,10 +321,10 @@ void fsm_construct_copy_sigma(struct fsm_construct_handle *handle, struct sigma
/* Insert into hashtable */
hash = fsm_construct_hash_sym(symbol);
- fh = (handle->fsm_sigma_hash)+hash;
+ fh = (handle->fsm_sigma_hash)+hash;
if (fh->symbol == NULL) {
fh->symbol = symdup;
- fh->sym = symnum;
+ fh->sym = symnum;
} else {
newfh = xxcalloc(1,sizeof(struct fsm_sigma_hash));
newfh->next = fh->next;
@@ -370,10 +370,10 @@ int fsm_construct_add_symbol(struct fsm_construct_handle *handle, char *symbol)
/* Insert into hashtable */
hash = fsm_construct_hash_sym(symbol);
- fh = (handle->fsm_sigma_hash)+hash;
+ fh = (handle->fsm_sigma_hash)+hash;
if (fh->symbol == NULL) {
fh->symbol = symdup;
- fh->sym = symnum;
+ fh->sym = symnum;
} else {
newfh = xxcalloc(1,sizeof(struct fsm_sigma_hash));
newfh->next = fh->next;
@@ -452,7 +452,7 @@ struct fsm *fsm_construct_done(struct fsm_construct_handle *handle) {
fsm_state_close(net);
net->sigma = fsm_construct_convert_sigma(handle);
- if (handle->name != NULL) {
+ if (handle->name != NULL) {
strncpy(net->name, handle->name, 40);
xxfree(handle->name);
} else {
@@ -616,7 +616,7 @@ int fsm_get_arc_source(struct fsm_read_handle *handle) {
}
int fsm_get_arc_target(struct fsm_read_handle *handle) {
- if (handle->arcs_cursor == NULL) { return -1;}
+ if (handle->arcs_cursor == NULL) { return -1;}
return(handle->arcs_cursor->target);
}
@@ -642,7 +642,7 @@ char *fsm_get_arc_in(struct fsm_read_handle *handle) {
}
int fsm_get_arc_num_in(struct fsm_read_handle *handle) {
- if (handle->arcs_cursor == NULL) { return -1;}
+ if (handle->arcs_cursor == NULL) { return -1;}
return(handle->arcs_cursor->in);
}
diff --git a/back-ends/foma/extract.c b/back-ends/foma/extract.c
index 83b5e49..fea08d2 100644
--- a/back-ends/foma/extract.c
+++ b/back-ends/foma/extract.c
@@ -39,6 +39,7 @@ struct fsm *fsm_lower(struct fsm *net) {
fsm_state_end_state();
xxfree(net->states);
fsm_state_close(net);
+ fsm_update_flags(net,NO,NO,NO,UNK,UNK,UNK);
sigma_cleanup(net,0);
return(net);
}
diff --git a/back-ends/foma/flags.c b/back-ends/foma/flags.c
index bee7f38..fcafa10 100644
--- a/back-ends/foma/flags.c
+++ b/back-ends/foma/flags.c
@@ -49,7 +49,7 @@ struct flags {
/* or neither. */
/* The languages FAIL, SUCCEED is then the union of all symbols that cause */
/* compatibility or incompatibility. */
-/* We intersect all these filters, creating a large filter that we compose both on */
+/* We intersect all these filters, creating a large filter that we compose both on */
/* the upper side of the network and the lower side: */
/* RESULT = FILTER .o. ORIGINAL .o. FILTER */
/* We can't simply intersect the language with FILTER because the lower side flags */
@@ -57,7 +57,7 @@ struct flags {
/* Finally, we replace the affected arcs with EPSILON arcs, and call */
/* sigma_cleanup() to purge the symbols not occurring on arcs. */
-///
+///
///Eliminate a flag from a network. If called with name = NULL, eliminate all flags.
///
@@ -71,7 +71,7 @@ struct fsm *flag_eliminate(struct fsm *net, char *name) {
flags = flag_extract(net);
/* Check that flag actually exists in net */
- if (name != NULL) {
+ if (name != NULL) {
for (found = 0, f = flags; f != NULL; f = f->next) {
if (strcmp(name,f->name) == 0)
found = 1;
@@ -151,7 +151,7 @@ struct fsm *flag_create_symbol(int type, char *name, char *value) {
strcat(string, ".");
strcat(string, name);
if (strcmp(value,"") != 0) {
- strcat(string, ".");
+ strcat(string, ".");
strcat(string, value);
}
strcat(string, "@");
@@ -322,8 +322,8 @@ struct flags *flag_extract (struct fsm *net) {
flags->type = flag_get_type(sigma->symbol);
flags->name = flag_get_name(sigma->symbol);
flags->value = flag_get_value(sigma->symbol);
- }
- }
+ }
+ }
return(flags);
}
@@ -339,7 +339,7 @@ int flag_check(char *s) {
if (*(s+i) == '@') { i++; goto s1; } return 0;
s1:
if (*(s+i) == 'C') { i++; goto s4; }
- if (*(s+i) == 'N' || *(s+i) == 'E' || *(s+i) == 'U' || *(s+i) == 'P') { i++; goto s2; }
+ if (*(s+i) == 'N' || *(s+i) == 'E' || *(s+i) == 'U' || *(s+i) == 'P') { i++; goto s2; }
if (*(s+i) == 'R' || *(s+i) == 'D') { i++; goto s3; } return 0;
s2:
if (*(s+i) == '.') { i++; goto s5; } return 0;
@@ -348,21 +348,21 @@ int flag_check(char *s) {
s4:
if (*(s+i) == '.') { i++; goto s7; } return 0;
s5:
- if (*(s+i) != '.' && *(s+i) != '\0') { i++; goto s8; } return 0;
+ if (*(s+i) != '.' && *(s+i) != '\0') { i++; goto s8; } return 0;
s6:
- if (*(s+i) != '.' && *(s+i) != '\0') { i++; goto s9; } return 0;
+ if (*(s+i) != '.' && *(s+i) != '\0') { i++; goto s9; } return 0;
s7:
if (*(s+i) != '.' && *(s+i) != '\0') { i++; goto s10; } return 0;
s8:
- if (*(s+i) == '.') { i++; goto s7; }
- if (*(s+i) != '.' && *(s+i) != '\0') { i++; goto s8; } return 0;
+ if (*(s+i) == '.') { i++; goto s7; }
+ if (*(s+i) != '.' && *(s+i) != '\0') { i++; goto s8; } return 0;
s9:
if (*(s+i) == '@') { i++; goto s11; }
if (*(s+i) == '.') { i++; goto s7; }
if (*(s+i) != '.' && *(s+i) != '\0') { i++; goto s9; } return 0;
s10:
- if (*(s+i) == '@') {i++; goto s11;}
+ if (*(s+i) == '@') {i++; goto s11;}
if (*(s+i) != '.' && *(s+i) != '\0') { i++; goto s10; } return 0;
s11:
if (*(s+i) == '\0') {return 1;} return 0;
@@ -371,25 +371,25 @@ int flag_check(char *s) {
int flag_get_type(char *string) {
if (strncmp(string+1,"U.",2) == 0) {
return FLAG_UNIFY;
- }
+ }
if (strncmp(string+1,"C.",2) == 0) {
return FLAG_CLEAR;
- }
+ }
if (strncmp(string+1,"D.",2) == 0) {
return FLAG_DISALLOW;
- }
+ }
if (strncmp(string+1,"N.",2) == 0) {
return FLAG_NEGATIVE;
- }
+ }
if (strncmp(string+1,"P.",2) == 0) {
return FLAG_POSITIVE;
- }
+ }
if (strncmp(string+1,"R.",2) == 0) {
return FLAG_REQUIRE;
- }
+ }
if (strncmp(string+1,"E.",2) == 0) {
return FLAG_EQUAL;
- }
+ }
return 0;
}
@@ -491,7 +491,7 @@ struct fsm *flag_twosided(struct fsm *net) {
maxstate++;
for (i = 0; i < tail; i++) {
- if ((fsm+i)->target == -1)
+ if ((fsm+i)->target == -1)
continue;
if ((*(isflag+(fsm+i)->in) || *(isflag+(fsm+i)->out)) && (fsm+i)->in != (fsm+i)->out) {
if (*(isflag+(fsm+i)->in) && !*(isflag+(fsm+i)->out)) {
diff --git a/back-ends/foma/foma.c b/back-ends/foma/foma.c
index d9b47f4..bd1dade 100644
--- a/back-ends/foma/foma.c
+++ b/back-ends/foma/foma.c
@@ -106,7 +106,7 @@ char *rl_gets(char *prompt) {
save it on the history. */
if (use_readline == 1) {
if (line_read && *line_read)
- add_history(line_read);
+ add_history(line_read);
}
#endif
return (line_read);
@@ -167,7 +167,7 @@ int main(int argc, char *argv[]) {
}
}
- if (!pipe_mode && !quiet_mode)
+ if (!pipe_mode && !quiet_mode)
printf("%s%i.%i.%i%s (svn r%i)%s",disclaimer1,MAJOR_VERSION,MINOR_VERSION,BUILD_VERSION,STATUS_VERSION,SVN_REV,disclaimer2);
#if defined(ORIGINAL) || defined(HAVE_READLINE)
rl_basic_word_break_characters = " >";
@@ -226,7 +226,7 @@ static char **my_completion(const char *text, int start, int end) {
smatch = start;
matches = rl_completion_matches ((char*)text, &my_generator);
- return (matches);
+ return (matches);
}
char *my_generator(const char *text, int state) {
@@ -244,7 +244,7 @@ char *my_generator(const char *text, int state) {
list_index++;
if (strncmp (name, text, len) == 0) {
- nummatches++;
+ nummatches++;
/* Can't use xxstrdup here */
return(strdup(name+smatch));
}
@@ -257,7 +257,7 @@ char *my_generator(const char *text, int state) {
/* Can't use xxstrdup here */
if (strncmp (name, text, len) == 0)
return(strdup(name+smatch));
- }
+ }
}
/* If no names matched, then return NULL. */
diff --git a/back-ends/foma/foma.h b/back-ends/foma/foma.h
index b550d2e..5845417 100644
--- a/back-ends/foma/foma.h
+++ b/back-ends/foma/foma.h
@@ -35,7 +35,7 @@ struct stack_entry {
struct apply_med_handle *amedh;
struct fsm *fsm;
struct stack_entry *next;
- struct stack_entry *previous;
+ struct stack_entry *previous;
};
/* Quantifier & Logic-related */
diff --git a/back-ends/foma/fomalib.h b/back-ends/foma/fomalib.h
index 673ad31..3a46bd2 100644
--- a/back-ends/foma/fomalib.h
+++ b/back-ends/foma/fomalib.h
@@ -63,6 +63,7 @@ extern "C" {
#define OP_RIGHTWARD_REPLACE 2
#define OP_LEFTWARD_REPLACE 3
#define OP_DOWNWARD_REPLACE 4
+#define OP_TWO_LEVEL_REPLACE 5
/* Arrow types in fsmrules */
#define ARROW_RIGHT 1
@@ -167,7 +168,7 @@ struct fsmcontexts {
struct fsmrules {
struct fsm *left;
- struct fsm *right;
+ struct fsm *right;
struct fsm *right2; /*Only needed for A -> B ... C rules*/
struct fsm *cross_product;
struct fsmrules *next;
@@ -285,6 +286,7 @@ FEXPORT struct fsm *fsm_add_loop(struct fsm *net, struct fsm *marker, int finals
FEXPORT struct fsm *fsm_add_sink(struct fsm *net, int final);
FEXPORT struct fsm *fsm_left_rewr(struct fsm *net, struct fsm *rewr);
FEXPORT struct fsm *fsm_flatten(struct fsm *net, struct fsm *epsilon);
+FEXPORT struct fsm *fsm_unflatten(struct fsm *net, char *epsilon_sym, char *repeat_sym);
FEXPORT struct fsm *fsm_close_sigma(struct fsm *net, int mode);
FEXPORT char *fsm_network_to_char(struct fsm *net);
@@ -448,17 +450,6 @@ int sh_get_value(struct sh_handle *sh);
/* Trie construction */
/*********************/
-#ifndef ORIGINAL
- #ifndef __cplusplus
- #ifndef bool
- #define bool int
- #define false((bool)0)
- #define true((bool)1)
- #define _Bool bool
- #endif
- #endif
-#endif // #ifndef ORIGINAL
-
struct trie_hash {
char *insym;
char *outsym;
@@ -468,7 +459,7 @@ struct trie_hash {
};
struct trie_states {
- _Bool is_final;
+ Boolean is_final;
};
struct fsm_trie_handle {
@@ -504,7 +495,7 @@ struct fsm_read_handle {
int sigma_list_size;
struct fsm *net;
unsigned char *lookuptable;
- _Bool has_unknowns;
+ Boolean has_unknowns;
};
FEXPORT struct fsm_read_handle *fsm_read_init(struct fsm *net);
diff --git a/back-ends/foma/fomalibconf.h b/back-ends/foma/fomalibconf.h
index 9dd33f7..34138b5 100644
--- a/back-ends/foma/fomalibconf.h
+++ b/back-ends/foma/fomalibconf.h
@@ -26,21 +26,21 @@ struct fsm_trans_list {
struct fsm_trans_list *next;
};
-#ifndef ORIGINAL
- #ifndef __cplusplus
- #ifndef bool
- #define bool int
- #define false ((bool)0)
- #define true ((bool)1)
- #endif
+#ifndef __cplusplus
+ #ifndef bool
+ #define bool int
+ #define false ((bool)0)
+ #define true ((bool)1)
#endif
-#define _Bool bool
-#endif // #ifndef ORIGINAL
+#endif // __cplusplus
+
+// _Bool changed to Boolean in HFST
+#define Boolean bool
struct fsm_state_list {
- _Bool used;
- _Bool is_final;
- _Bool is_initial;
+ Boolean used;
+ Boolean is_final;
+ Boolean is_initial;
short int num_trans;
int state_number;
struct fsm_trans_list *fsm_trans_list;
@@ -88,9 +88,9 @@ struct apply_med_handle {
uint8_t *nletterbits;
int astarcount;
int heapcount;
- int heap_size;
+ int heap_size;
int agenda_size;
- int maxdepth;
+ int maxdepth;
int maxsigma;
int wordlen;
int utf8len;
@@ -119,19 +119,19 @@ struct apply_med_handle {
struct state_array *state_array;
struct fsm *net;
struct fsm_state *curr_ptr;
- _Bool hascm;
+ Boolean hascm;
};
struct apply_handle {
int ptr;
- int curr_ptr;
+ int curr_ptr;
int ipos;
int opos;
int mode;
int printcount;
int *numlines;
- int *statemap;
+ int *statemap;
int *marks;
struct sigma_trie {
@@ -164,7 +164,7 @@ struct apply_handle {
char *epsilon_symbol;
int print_pairs;
int apply_stack_ptr;
- int apply_stack_top;
+ int apply_stack_top;
int oldflagneg;
int outstringtop;
int iterate_old;
@@ -189,8 +189,8 @@ struct apply_handle {
struct flag_list {
char *name;
char *value;
- short neg;
- struct flag_list *next;
+ short int neg;
+ struct flag_list *next;
} *flag_list;
struct flag_lookup {
@@ -230,7 +230,7 @@ void fsm_state_set_current_state(int state_no, int final_state, int start_state)
void fsm_state_add_arc(int state_no, int in, int out, int target, int final_state, int start_state);
/* Call fsm_state_close() when done with arcs to a state */
-void fsm_state_close();
+void fsm_state_close(struct fsm *net);
/* Call this when done with entire FSM */
void fsm_state_end_state();
@@ -242,10 +242,6 @@ FEXPORT void fsm_count(struct fsm *net);
void fsm_sort_lines(struct fsm *net);
void fsm_update_flags(struct fsm *net, int det, int pru, int min, int eps, int loop, int completed);
-/* Rewrite-related functions */
-struct fsm *rewrite_cp_to_fst(struct fsm *net, char *lower_symbol, char *zero_symbol);
-struct fsm *rewrite_cp(struct fsm *U, struct fsm *L);
-
int sort_cmp(const void *a, const void *b);
int find_arccount(struct fsm_state *fsm);
diff --git a/back-ends/foma/iface.c b/back-ends/foma/iface.c
index caed6df..d657d39 100644
--- a/back-ends/foma/iface.c
+++ b/back-ends/foma/iface.c
@@ -30,7 +30,7 @@
#ifdef ZLIB
#include "zlib.h"
#endif
-#endif // #ifdef ORIGINAL
+#endif // #ifdef ORIGINAL
extern int g_show_flags;
extern int g_obey_flags;
@@ -63,6 +63,7 @@ extern int foma_net_print(struct fsm *net, gzFile outfile);
#ifdef ZLIB
extern int foma_net_print(struct fsm *net, gzFile outfile);
#endif
+int snprintf(char *str, size_t size, const char *format, ...);
#endif // #ifdef ORIGINAL
static char *sigptr(struct sigma *sigma, int number);
@@ -171,7 +172,7 @@ struct global_help {
{"print upper-words","prints words on the upper side of top FSM","Short form: upper-words"},
{"print upper-words > filename","prints words on the upper side of top FSM to file","Short form:upper-words"},
{"print words","prints words of top FSM","Short form: words"},
- {"print words > filename","prints words of top FSM to file","Short form: words"},
+ {"print words > filename","prints words of top FSM to file","Short form: words"},
{"prune net","makes top network coaccessible",""},
{"push (defined) <name>","adds a defined FSM to top of stack",""},
{"quit","exit foma",""},
@@ -442,9 +443,9 @@ int iface_apply_file(char *infilename, char *outfilename, int direction) {
if (direction == AP_D)
result = apply_down(ah,NULL);
if (direction == AP_U)
- result = apply_up(ah,NULL);
+ result = apply_up(ah,NULL);
if (result == NULL)
- break;
+ break;
fprintf(OUTFILE,"%s\n", result);
}
}
@@ -480,7 +481,7 @@ void iface_apply_down(char *word) {
void iface_apply_up(char *word) {
int i;
char *result;
- struct apply_handle *ah;
+ struct apply_handle *ah;
if (!iface_stack_check(1)) {
return;
}
@@ -511,7 +512,7 @@ void iface_close() {
void iface_compact() {
if (iface_stack_check(1)) {
- fsm_compact(stack_find_top()->fsm);
+ fsm_compact(stack_find_top()->fsm);
sigma_sort(stack_find_top()->fsm);
stack_add(fsm_topsort(fsm_minimize(stack_pop())));
}
@@ -550,7 +551,7 @@ void iface_crossproduct() {
struct fsm *one, *two;
if (iface_stack_check(2)) {
one = stack_pop();
- two = stack_pop();
+ two = stack_pop();
stack_add(fsm_topsort(fsm_minimize(fsm_cross_product(one,two))));
}
}
@@ -599,14 +600,14 @@ void iface_ignore() {
struct fsm *one, *two;
if (iface_stack_check(2)) {
one = stack_pop();
- two = stack_pop();
+ two = stack_pop();
stack_add(fsm_topsort(fsm_minimize(fsm_ignore(one,two,OP_IGNORE_ALL))));
}
}
void iface_intersect() {
if (iface_stack_check(2)) {
- while (stack_size()>1)
+ while (stack_size()>1)
stack_add(fsm_topsort(fsm_minimize(fsm_intersect(stack_pop(),stack_pop()))));
}
}
@@ -671,7 +672,7 @@ void iface_pop() {
else {
net = stack_pop();
fsm_destroy(net);
- }
+ }
}
void iface_lower_words(int limit) {
@@ -697,7 +698,7 @@ void iface_lower_words(int limit) {
void iface_name_net(char *name) {
if (iface_stack_check(1)) {
- strncpy(stack_find_top()->fsm->name, name, 40);
+ strncpy(stack_find_top()->fsm->name, name, 40);
iface_print_name();
}
}
@@ -735,14 +736,14 @@ void iface_print_cmatrix_att(char *filename) {
printf("No confusion matrix defined.\n");
} else {
if (filename == NULL) {
- outfile = stdout;
+ outfile = stdout;
} else {
outfile = fopen(filename,"w");
printf("Writing confusion matrix to file '%s'.\n", filename);
- }
+ }
cmatrix_print_att(stack_find_top()->fsm, outfile);
}
- }
+ }
}
void iface_print_cmatrix() {
@@ -752,7 +753,7 @@ void iface_print_cmatrix() {
} else {
cmatrix_print(stack_find_top()->fsm);
}
- }
+ }
}
void iface_print_defined() {
@@ -822,7 +823,7 @@ void iface_apply_random(char *(*applyer)(), int limit) {
if (result != NULL) {
for (tempresults = results; tempresults - results < limit; tempresults++) {
if (tempresults->string == NULL) {
- tempresults->string = strdup(result);
+ tempresults->string = xxstrdup(result);
tempresults->count = 1;
break;
}
@@ -861,7 +862,7 @@ void iface_print_shortest_string() {
if (iface_stack_check(1)) {
one = fsm_copy(stack_find_top()->fsm);
/* L - ?+ [[L .o. [?:"@TMP@"]*].l .o. "@TMP@":?*].l; */
- if (stack_find_top()->fsm->arity == 1) {
+ if (stack_find_top()->fsm->arity == 1) {
Result = fsm_minimize(fsm_minus(fsm_copy(one),fsm_concat(fsm_kleene_plus(fsm_identity()),fsm_lower(fsm_compose(fsm_lower(fsm_compose(fsm_copy(one),fsm_kleene_star(fsm_cross_product(fsm_identity(),fsm_symbol("@TMP@"))))),fsm_kleene_star(fsm_cross_product(fsm_symbol("@TMP@"),fsm_identity())))))));
ah = apply_init(Result);
word = apply_words(ah);
@@ -901,7 +902,7 @@ void iface_print_shortest_string_size() {
ResultU = fsm_minimize(fsm_lower(fsm_compose(oneu,fsm_kleene_star(fsm_cross_product(fsm_identity(),fsm_symbol("a"))))));
ResultL = fsm_minimize(fsm_lower(fsm_compose(onel,fsm_kleene_star(fsm_cross_product(fsm_identity(),fsm_symbol("a"))))));
printf("Shortest acyclic upper path length: %i\n",(ResultU->statecount)-1);
- printf("Shortest acyclic lower path length: %i\n",(ResultL->statecount)-1);
+ printf("Shortest acyclic lower path length: %i\n",(ResultL->statecount)-1);
}
}
}
@@ -917,7 +918,7 @@ int iface_read_att(char *filename) {
} else {
stack_add(tempnet);
return 0;
- }
+ }
}
int iface_read_prolog(char *filename) {
@@ -1030,7 +1031,7 @@ void iface_save_defined(char *filename) {
save_defined(g_defines, filename);
}
-#if defined(ORIGINAL) || defined(ZLIB)
+#if defined(ORIGINAL) || defined(ZLIB)
void iface_save_stack(char *filename) {
gzFile outfile;
struct stack_entry *stack_ptr;
@@ -1075,7 +1076,7 @@ void iface_show_variable(char *name) {
return;
}
}
- printf("*There is no global variable '%s'.\n",name);
+ printf("*There is no global variable '%s'.\n",name);
}
void iface_set_variable(char *name, char *value) {
@@ -1265,7 +1266,7 @@ void iface_words_file(char *filename, int type) {
}
apply_reset_enumerator(ah);
fclose(outfile);
- }
+ }
}
void iface_words(int limit) {
@@ -1343,15 +1344,15 @@ void iface_pairs_call(int limit, int random) {
if (iface_stack_check(1)) {
ah = stack_get_ah();
apply_set_show_flags(ah, g_show_flags);
- apply_set_obey_flags(ah, g_obey_flags);
+ apply_set_obey_flags(ah, g_obey_flags);
apply_set_space_symbol(ah, "\001");
apply_set_epsilon(ah, "\002");
apply_set_separator(ah, "\003");
for (i = limit; i > 0; i--) {
- if (random == 1)
+ if (random == 1)
result = apply_random_words(ah);
else
- result = apply_words(ah);
+ result = apply_words(ah);
if (result == NULL)
break;
iface_split_result(result, &upper, &lower);
@@ -1361,7 +1362,7 @@ void iface_pairs_call(int limit, int random) {
}
apply_set_space_symbol(ah, " ");
apply_set_epsilon(ah, "0");
- apply_set_separator(ah, ":");
+ apply_set_separator(ah, ":");
apply_reset_enumerator(ah);
}
}
@@ -1390,12 +1391,12 @@ void iface_pairs_file(char *filename) {
}
ah = stack_get_ah();
apply_set_show_flags(ah, g_show_flags);
- apply_set_obey_flags(ah, g_obey_flags);
+ apply_set_obey_flags(ah, g_obey_flags);
apply_set_space_symbol(ah, "\001");
apply_set_epsilon(ah, "\002");
apply_set_separator(ah, "\003");
for (;;) {
- result = apply_words(ah);
+ result = apply_words(ah);
if (result == NULL)
break;
iface_split_result(result, &upper, &lower);
@@ -1405,7 +1406,7 @@ void iface_pairs_file(char *filename) {
}
apply_set_space_symbol(ah, " ");
apply_set_epsilon(ah, "0");
- apply_set_separator(ah, ":");
+ apply_set_separator(ah, ":");
apply_reset_enumerator(ah);
fclose(outfile);
}
@@ -1436,7 +1437,7 @@ int iface_write_att(char *filename) {
}
void iface_write_prolog(char *filename) {
- if (iface_stack_check(1))
+ if (iface_stack_check(1))
foma_write_prolog(stack_find_top()->fsm, filename);
}
@@ -1514,10 +1515,10 @@ static int print_net(struct fsm *net, char *filename) {
fprintf(out,"Arity: %i\n", net->arity);
for (; stateptr->state_no != -1; stateptr++) {
if (stateptr->state_no != previous_state) {
- if (stateptr->start_state) {
+ if (stateptr->start_state) {
fprintf(out,"S");
}
- if (stateptr->final_state) {
+ if (stateptr->final_state) {
fprintf(out,"f");
}
if (stateptr->in==-1) {
@@ -1576,7 +1577,7 @@ void print_mem_size(struct fsm *net) {
} else if (s >= 1048576 && s < 1073741824) {
sprintf(size, "%.1f MB. ", sf/1048576);
} else if (s >= 1073741824) {
- sprintf(size, "%.1f GB. ", sf/1073741824);
+ sprintf(size, "%.1f GB. ", sf/1073741824);
}
fprintf(stdout, "%s", size);
fflush(stdout);
@@ -1656,8 +1657,8 @@ static int print_dot(struct fsm *net, char *filename) {
}
printed = xxcalloc(net->linecount,sizeof(printed));
- /* Go through arcs */
- for (i=0; (stateptr+i)->state_no != -1; i++) {
+ /* Go through arcs */
+ for (i=0; (stateptr+i)->state_no != -1; i++) {
if ((stateptr+i)->target == -1 || printed[i] == 1)
continue;
fprintf(dotfile,"%i -> %i [label=\"", (stateptr+i)->state_no, (stateptr+i)->target);
@@ -1681,7 +1682,7 @@ static int print_dot(struct fsm *net, char *filename) {
}
}
}
- fprintf(dotfile,"\"];\n");
+ fprintf(dotfile,"\"];\n");
}
diff --git a/back-ends/foma/io.c b/back-ends/foma/io.c
index 9b50b69..d17da4e 100644
--- a/back-ends/foma/io.c
+++ b/back-ends/foma/io.c
@@ -126,7 +126,7 @@ void escape_print(FILE *stream, char* string) {
if (strchr(string, '"') != NULL) {
for (i = 0; *(string+i) != '\0'; i++) {
if (*(string+i) == '"') {
- fprintf(stream, "\\\"");
+ fprintf(stream, "\\\"");
} else {
fputc(*(string+i), stream);
}
@@ -183,10 +183,10 @@ int foma_write_prolog (struct fsm *net, char *filename) {
instring = sigma_string(i, net->sigma);
if (strcmp(instring,"0") == 0) {
instring = "%0";
- }
+ }
fprintf(out, "symbol(%s, \"", identifier);
escape_print(out, instring);
- fprintf(out, "\").\n");
+ fprintf(out, "\").\n");
}
}
@@ -218,11 +218,11 @@ int foma_write_prolog (struct fsm *net, char *filename) {
escape_print(out, instring);
fprintf(out, "\").\n");
}
- else if (net->arity == 2) {
+ else if (net->arity == 2) {
fprintf(out, "\"");
escape_print(out, instring);
fprintf(out, "\":\"");
- escape_print(out, outstring);
+ escape_print(out, outstring);
fprintf(out, "\").\n");
}
else if (net->arity == 1) {
@@ -348,7 +348,7 @@ struct fsm *fsm_read_prolog (char *filename) {
if (fsm_construct_check_symbol(outh, temp) == -1) {
fsm_construct_add_symbol(outh, temp);
- }
+ }
continue;
}
if (strstr(buf, "arc(") == buf) {
@@ -379,7 +379,7 @@ struct fsm *fsm_read_prolog (char *filename) {
temp_ptr = strstr(temp_ptr2, "\"");
temp_ptr++;
- if (arity == 2) {
+ if (arity == 2) {
temp_ptr2 = strstr(temp_ptr, "\":");
} else {
temp_ptr2 = strstr(temp_ptr, "\").");
@@ -422,8 +422,8 @@ struct fsm *fsm_read_prolog (char *filename) {
strcpy(out,"?");
}
- if (arity == 1) {
- fsm_construct_add_arc(outh, source, target, in, in);
+ if (arity == 1) {
+ fsm_construct_add_arc(outh, source, target, in, in);
} else {
fsm_construct_add_arc(outh, source, target, in, out);
}
@@ -463,11 +463,11 @@ char *spacedtext_get_next_line(char **text) {
ret = *text;
if (**text == '\0')
return NULL;
- for (t = *text; *t != '\0' && *t != '\n'; t++) {
+ for (t = *text; *t != '\0' && *t != '\n'; t++) {
}
if (*t == '\0')
*text = t;
- else
+ else
*text = t+1;
*t = '\0';
return(ret);
@@ -690,7 +690,7 @@ static INLINE int explode_line(char *buf, int *values) {
/* ##sigma## */
/* ...SIGMA LINES... */
/* ##states## */
-/* ...TRANSITION LINES... */
+/* ...TRANSITION LINES... */
/* ##end## */
/* Several networks may be concatenated in one file */
@@ -767,7 +767,7 @@ struct fsm *io_net_read(struct io_buf_handle *iobh, char **net_name) {
#ifdef ORIGINAL
sscanf(buf, "%i %i %i %i %i %lld %i %i %i %i %i %i %s", &net->arity, &net->arccount, &net->statecount, &net->linecount, &net->finalcount, &net->pathcount, &net->is_deterministic, &net->is_pruned, &net->is_minimized, &net->is_epsilon_free, &net->is_loop_free, &extras, buf);
#else
- sscanf(buf, "%i %i %i %i %i LONG_LONG_SPECIFIER %i %i %i %i %i %i %s", &net->arity, &net->arccount, &net->statecount, &net->linecount, &net->finalcount, &net->pathcount, &net->is_deterministic, &net->is_pruned, &net->is_minimized, &net->is_epsilon_free, &net->is_loop_free, &extras, buf);
+ sscanf(buf, "%i %i %i %i %i "LONG_LONG_SPECIFIER" %i %i %i %i %i %i %s", &net->arity, &net->arccount, &net->statecount, &net->linecount, &net->finalcount, &net->pathcount, &net->is_deterministic, &net->is_pruned, &net->is_minimized, &net->is_epsilon_free, &net->is_loop_free, &extras, buf);
#endif
strcpy(net->name, buf);
*net_name = xxstrdup(buf);
@@ -913,11 +913,11 @@ int foma_net_print(struct fsm *net, gzFile outfile) {
extras = (net->is_completed) | (net->arcs_sorted_in << 2) | (net->arcs_sorted_out << 4);
#ifdef ORIGINAL
- gzprintf(outfile,
+ gzprintf(outfile,
"%i %i %i %i %i %lld %i %i %i %i %i %i %s\n", net->arity, net->arccount, net->statecount, net->linecount, net->finalcount, net->pathcount, net->is_deterministic, net->is_pruned, net->is_minimized, net->is_epsilon_free, net->is_loop_free, extras, net->name);
#else
- gzprintf(outfile,
- "%i %i %i %i %i LONG_LONG_SPECIFIER %i %i %i %i %i %i %s\n", net->arity, net->arccount, net->statecount, net->linecount, net->finalcount, net->pathcount, net->is_deterministic, net->is_pruned, net->is_minimized, net->is_epsilon_free, net->is_loop_free, extras, net->name);
+ gzprintf(outfile,
+ "%i %i %i %i %i "LONG_LONG_SPECIFIER" %i %i %i %i %i %i %s\n", net->arity, net->arccount, net->statecount, net->linecount, net->finalcount, net->pathcount, net->is_deterministic, net->is_pruned, net->is_minimized, net->is_epsilon_free, net->is_loop_free, extras, net->name);
#endif
/* Sigma */
@@ -978,7 +978,7 @@ int net_print_att(struct fsm *net, FILE *outfile) {
}
for (i=0; (fsm+i)->state_no != -1; i++) {
if ((fsm+i)->target != -1) {
- fprintf(outfile, "%i\t%i\t%s\t%s\n",(fsm+i)->state_no,(fsm+i)->target, (sl+(fsm+i)->in)->symbol, (sl+(fsm+i)->out)->symbol);
+ fprintf(outfile, "%i\t%i\t%s\t%s\n",(fsm+i)->state_no,(fsm+i)->target, (sl+(fsm+i)->in)->symbol, (sl+(fsm+i)->out)->symbol);
}
}
prev = -1;
@@ -1003,7 +1003,8 @@ static size_t io_get_gz_file_size(char *filename) {
/* The last four bytes in a .gz file shows the size of the uncompressed data */
infile = fopen(filename, "r");
fseek(infile, -4, SEEK_END);
- fread(&bytes, 1, 4, infile);
+ if (fread(&bytes, 1, 4, infile) != 4) // value of fread was not checked in foma, this is an HFST addition
+ return -1;
fclose(infile);
for (i = 0 ; i < 4 ; i++) {
ints[i] = bytes[i];
diff --git a/back-ends/foma/lex.cmatrix.c b/back-ends/foma/lex.cmatrix.c
index 86aa51b..2e254a0 100644
--- a/back-ends/foma/lex.cmatrix.c
+++ b/back-ends/foma/lex.cmatrix.c
@@ -1,7 +1,7 @@
#line 3 "lex.cmatrix.c"
-#define YY_INT_ALIGNED short int
+#define YY_INT_ALIGNED short
/* A lexical scanner generated by flex */
@@ -52,7 +52,7 @@
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h,
- * if you want the limit (max/min) macros for int types.
+ * if you want the limit (max/min) macros for int types.
*/
#ifndef __STDC_LIMIT_MACROS
#define __STDC_LIMIT_MACROS 1
@@ -67,10 +67,10 @@ typedef int32_t flex_int32_t;
typedef uint32_t flex_uint32_t;
#else
typedef signed char flex_int8_t;
-typedef short int flex_int16_t;
+typedef short flex_int16_t;
typedef int flex_int32_t;
-typedef unsigned char flex_uint8_t;
-typedef unsigned short int flex_uint16_t;
+typedef unsigned char flex_uint8_t;
+typedef unsigned short flex_uint16_t;
typedef unsigned int flex_uint32_t;
/* Limits of integral types. */
@@ -1525,7 +1525,7 @@ static int yy_get_next_buffer (void)
/** Immediately switch to a different input stream.
* @param input_file A readable stream.
- *
+ *
* @note This function does not reset the start condition to @c INITIAL .
*/
void cmatrixrestart (FILE * input_file )
@@ -1543,7 +1543,7 @@ static int yy_get_next_buffer (void)
/** Switch to a different input buffer.
* @param new_buffer The new input buffer.
- *
+ *
*/
void cmatrix_switch_to_buffer (YY_BUFFER_STATE new_buffer )
{
@@ -1587,7 +1587,7 @@ static void cmatrix_load_buffer_state (void)
/** Allocate and initialize an input buffer state.
* @param file A readable stream.
* @param size The character buffer size in bytes. When in doubt, use @c YY_BUF_SIZE.
- *
+ *
* @return the allocated buffer state.
*/
YY_BUFFER_STATE cmatrix_create_buffer (FILE * file, int size )
@@ -1616,7 +1616,7 @@ static void cmatrix_load_buffer_state (void)
/** Destroy the buffer.
* @param b a buffer created with cmatrix_create_buffer()
- *
+ *
*/
void cmatrix_delete_buffer (YY_BUFFER_STATE b )
{
@@ -1667,7 +1667,7 @@ extern int isatty (int );
/** Discard all buffered characters. On the next scan, YY_INPUT will be called.
* @param b the buffer state to be flushed, usually @c YY_CURRENT_BUFFER.
- *
+ *
*/
void cmatrix_flush_buffer (YY_BUFFER_STATE b )
{
@@ -1696,7 +1696,7 @@ extern int isatty (int );
* the current state. This function will allocate the stack
* if necessary.
* @param new_buffer The new state.
- *
+ *
*/
void cmatrixpush_buffer_state (YY_BUFFER_STATE new_buffer )
{
@@ -1726,7 +1726,7 @@ void cmatrixpush_buffer_state (YY_BUFFER_STATE new_buffer )
/** Removes and deletes the top of the stack, if present.
* The next element becomes the new top.
- *
+ *
*/
void cmatrixpop_buffer_state (void)
{
@@ -1793,8 +1793,8 @@ static void cmatrixensure_buffer_stack (void)
/** Setup the input buffer state to scan directly from a user-specified character buffer.
* @param base the character buffer
* @param size the size in bytes of the character buffer
- *
- * @return the newly allocated buffer state object.
+ *
+ * @return the newly allocated buffer state object.
*/
YY_BUFFER_STATE cmatrix_scan_buffer (char * base, yy_size_t size )
{
@@ -1828,7 +1828,7 @@ YY_BUFFER_STATE cmatrix_scan_buffer (char * base, yy_size_t size )
/** Setup the input buffer state to scan a string. The next call to cmatrixlex() will
* scan from a @e copy of @a str.
* @param yystr a NUL-terminated string to scan
- *
+ *
* @return the newly allocated buffer state object.
* @note If you want to scan bytes that may contain NUL values, then use
* cmatrix_scan_bytes() instead.
@@ -1843,7 +1843,7 @@ YY_BUFFER_STATE cmatrix_scan_string (yyconst char * yystr )
* scan from a @e copy of @a bytes.
* @param yybytes the byte buffer to scan
* @param _yybytes_len the number of bytes in the buffer pointed to by @a bytes.
- *
+ *
* @return the newly allocated buffer state object.
*/
YY_BUFFER_STATE cmatrix_scan_bytes (yyconst char * yybytes, int _yybytes_len )
@@ -1906,7 +1906,7 @@ static void yy_fatal_error (yyconst char* msg )
/* Accessor methods (get/set functions) to struct members. */
/** Get the current line number.
- *
+ *
*/
int cmatrixget_lineno (void)
{
@@ -1915,7 +1915,7 @@ int cmatrixget_lineno (void)
}
/** Get the input stream.
- *
+ *
*/
FILE *cmatrixget_in (void)
{
@@ -1923,7 +1923,7 @@ FILE *cmatrixget_in (void)
}
/** Get the output stream.
- *
+ *
*/
FILE *cmatrixget_out (void)
{
@@ -1931,7 +1931,7 @@ FILE *cmatrixget_out (void)
}
/** Get the length of the current token.
- *
+ *
*/
int cmatrixget_leng (void)
{
@@ -1939,7 +1939,7 @@ int cmatrixget_leng (void)
}
/** Get the current token.
- *
+ *
*/
char *cmatrixget_text (void)
@@ -1949,7 +1949,7 @@ char *cmatrixget_text (void)
/** Set the current line number.
* @param line_number
- *
+ *
*/
void cmatrixset_lineno (int line_number )
{
@@ -1960,7 +1960,7 @@ void cmatrixset_lineno (int line_number )
/** Set the input stream. This does not discard the current
* input buffer.
* @param in_str A readable stream.
- *
+ *
* @see cmatrix_switch_to_buffer
*/
void cmatrixset_in (FILE * in_str )
diff --git a/back-ends/foma/lex.interface.c b/back-ends/foma/lex.interface.c
index de9d2ea..820afaa 100644
--- a/back-ends/foma/lex.interface.c
+++ b/back-ends/foma/lex.interface.c
@@ -415,7 +415,7 @@ struct yy_trans_info
flex_int32_t yy_verify;
flex_int32_t yy_nxt;
};
-static yyconst flex_int16_t yy_acclist[1632] =
+static yyconst flex_int16_t yy_acclist[1635] =
{ 0,
211, 211, 152, 152, 213, 210, 212, 207, 210, 212,
208, 210, 212, 207, 209, 210, 212, 181, 207, 210,
@@ -522,84 +522,84 @@ static yyconst flex_int16_t yy_acclist[1632] =
51, 51, 51, 2, 3, 3, 2, 4, 2, 20,
2, 2, 2, 22, 22, 28, 33, 33, 35, 37,
88, 88, 88, 41, 41, 54, 54, 77, 77, 77,
- 45, 58, 58, 20, 58, 58, 58, 58, 57, 57,
- 57, 20, 57, 57, 57, 57, 46, 46, 46, 71,
- 55, 55, 87, 87, 87, 87, 81, 84, 84, 84,
- 92, 93, 87, 87, 87, 20, 99, 66, 101, 103,
-
- 101, 103, 95, 95, 95, 91, 91, 91, 117, 117,
- 108, 120, 9, 70, 70, 78, 78, 78, 69, 125,
- 125, 73, 124, 123, 124, 123, 124, 20, 124, 124,
- 124, 124, 123, 124, 127, 127, 127, 163, 163, 163,
- 163, 163, 163, 161, 161, 161, 161, 161, 161, 126,
- 84, 126, 48, 15, 16, 17, 18, 49, 21, 21,
- 21, 51, 51, 4, 4, 28, 28, 34, 33, 34,
- 34, 20, 34, 34, 34, 34, 34, 34, 33, 34,
- 34, 34, 35, 37, 37, 37, 88, 88, 88, 41,
- 41, 40, 40, 77, 53, 43, 43, 44, 44, 44,
-
- 45, 45, 45, 57, 57, 46, 46, 46, 76, 47,
- 47, 48, 49, 53, 55, 66, 69, 48, 49, 50,
- 53, 55, 56, 66, 67, 69, 87, 87, 87, 87,
- 123, 81, 81, 82, 82, 84, 84, 84, 84, 84,
- 89, 92, 92, 92, 93, 93, 93, 87, 94, 94,
- 98, 98, 98, 20, 98, 98, 98, 98, 98, 98,
- 98, 99, 99, 99, 103, 102, 103, 102, 91, 91,
- 91, 117, 117, 120, 120, 78, 69, 125, 126, 125,
- 125, 126, 75, 124, 124, 125, 127, 127, 127, 126,
- 48, 85, 15, 16, 17, 17, 17, 18, 49, 21,
-
- 21, 21, 4, 29, 37, 37, 88, 32, 40, 40,
- 53, 43, 43, 45, 45, 8251, 8251, 8251, 20, 8251,
- 8251, 8251, 8251, 8251, 8251, 8251, 8251, 46, 76, 76,
- 76, 47, 47, 48, 49, 53, 55, 66, 69, 48,
- 49, 50, 53, 55, 56,16443, 66, 67, 69, 87,
- 87, 123, 87, 123, 81, 81, 82, 82, 82, 82,
- 82, 84, 84, 84, 87, 84, 84, 88, 87, 91,
- 92, 92, 92, 93, 93, 94, 94, 94, 94, 8288,
- 8288, 20, 8288, 99, 99, 99, 91, 117, 117, 120,
- 120, 69, 122, 122, 74, 74, 75, 75, 75, 124,
-
- 125, 127, 8, 8, 12, 12, 13, 13, 86, 85,
- 85, 86, 86, 20, 86, 86, 86, 86, 15, 15,
- 15, 16, 16, 17, 17, 17, 18, 18, 18, 21,
- 29, 29, 35, 35, 53, 43, 8251, 46, 76, 47,
- 47, 48, 49, 66, 69, 48, 49, 50, 52, 71,
- 66, 69, 73, 87, 123, 124, 87, 123, 87, 124,
- 87, 123, 124, 87, 124, 82, 82, 87, 88, 87,
- 87, 91, 93, 93, 94, 94, 94, 98, 98, 97,
- 98, 97, 101, 101, 105, 105, 106, 106, 105, 106,
- 117, 117, 69, 122, 122, 75, 125, 126, 125, 127,
-
- 1, 5, 8, 83, 12, 12, 13, 13, 86, 14,
- 14, 15, 15, 16, 16, 17, 18, 18, 18, 21,
- 30, 35, 35, 36, 38, 38, 53, 47, 47, 48,
- 49, 53, 69, 48, 49, 53, 69, 82, 82, 87,
- 87, 93, 93, 97, 97, 100, 100, 102, 102, 68,
- 113, 69, 123, 127, 1, 1, 3, 5, 12, 12,
- 13, 13, 14, 14, 17, 18, 21, 38, 38, 38,
- 41, 41, 42, 53, 48, 48, 49, 53, 69, 82,
- 82, 85, 85, 95, 95, 97, 97, 100, 100, 100,
- 68, 121, 69, 123, 3, 12, 12, 13, 13, 14,
-
- 14, 17, 18, 21, 41, 41, 41, 53, 53, 69,
- 85, 95, 95, 95, 97, 97, 90, 69, 123, 124,
- 123, 18, 32, 36, 36, 53, 68, 69, 60, 72,
- 62, 64, 97, 97, 90, 90, 90, 121, 121, 11,
- 11, 36, 36, 40, 40, 40, 42, 42, 53, 68,
- 69, 94, 94, 94, 31, 97, 97, 90, 90, 90,
- 111, 112, 121, 121, 26, 26, 39, 39, 40, 40,
- 40, 42, 42, 53, 69, 61, 61, 63, 63, 65,
- 65, 90, 94, 94, 94, 97, 97, 90, 121, 121,
- 1, 1, 25, 25, 24, 25, 24, 27, 27, 20,
-
- 27, 27, 27, 27, 27, 27, 39, 39, 39, 60,
- 72, 62, 64, 90, 79, 109, 110, 115, 1, 1,
- 1, 108, 28, 28, 79, 29, 116, 29, 114, 80,
- 118
+ 53, 45, 58, 58, 20, 58, 58, 58, 58, 57,
+ 57, 57, 20, 57, 57, 57, 57, 46, 46, 46,
+ 71, 55, 55, 87, 87, 87, 87, 81, 84, 84,
+ 84, 92, 93, 87, 87, 87, 20, 99, 66, 101,
+
+ 103, 101, 103, 95, 95, 95, 91, 91, 91, 117,
+ 117, 108, 120, 9, 70, 70, 78, 78, 78, 69,
+ 125, 125, 73, 124, 123, 124, 123, 124, 20, 124,
+ 124, 124, 124, 123, 124, 127, 127, 127, 163, 163,
+ 163, 163, 163, 163, 161, 161, 161, 161, 161, 161,
+ 126, 84, 126, 48, 15, 16, 17, 18, 49, 21,
+ 21, 21, 51, 51, 4, 4, 28, 28, 34, 33,
+ 34, 34, 20, 34, 34, 34, 34, 34, 34, 33,
+ 34, 34, 34, 35, 37, 37, 37, 88, 88, 88,
+ 41, 41, 40, 40, 77, 53, 43, 43, 44, 44,
+
+ 44, 45, 45, 45, 57, 57, 46, 46, 46, 76,
+ 47, 47, 48, 49, 53, 55, 66, 69, 48, 49,
+ 50, 53, 55, 56, 66, 67, 69, 87, 87, 87,
+ 87, 123, 81, 81, 82, 82, 84, 84, 84, 84,
+ 84, 89, 92, 92, 92, 93, 93, 93, 87, 94,
+ 94, 98, 98, 98, 20, 98, 98, 98, 98, 98,
+ 98, 98, 99, 99, 99, 103, 102, 103, 102, 91,
+ 91, 91, 117, 117, 120, 120, 78, 69, 125, 126,
+ 125, 125, 126, 75, 124, 124, 125, 127, 127, 127,
+ 126, 48, 85, 15, 16, 17, 17, 17, 18, 49,
+
+ 21, 21, 21, 4, 29, 37, 37, 88, 32, 40,
+ 40, 53, 43, 43, 45, 45, 8251, 8251, 8251, 20,
+ 8251, 8251, 8251, 8251, 8251, 8251, 8251, 8251, 46, 76,
+ 76, 76, 47, 47, 48, 49, 53, 55, 66, 69,
+ 48, 49, 50, 53, 55, 56,16443, 66, 67, 69,
+ 87, 87, 123, 87, 123, 81, 81, 82, 82, 82,
+ 82, 82, 84, 84, 84, 87, 84, 84, 88, 87,
+ 91, 92, 92, 92, 93, 93, 94, 94, 94, 94,
+ 8288, 8288, 20, 8288, 99, 99, 99, 91, 117, 117,
+ 120, 120, 69, 122, 122, 74, 74, 75, 75, 75,
+
+ 124, 125, 127, 8, 8, 12, 12, 13, 13, 86,
+ 85, 85, 86, 86, 20, 86, 86, 86, 86, 15,
+ 15, 15, 16, 16, 17, 17, 17, 18, 18, 18,
+ 21, 29, 29, 35, 35, 53, 43, 8251, 46, 76,
+ 47, 47, 48, 49, 53, 66, 69, 48, 49, 50,
+ 52, 53, 71, 66, 69, 73, 87, 123, 124, 87,
+ 123, 87, 124, 87, 123, 124, 87, 124, 82, 82,
+ 87, 88, 87, 87, 91, 93, 93, 94, 94, 94,
+ 98, 98, 97, 98, 97, 101, 101, 105, 105, 106,
+ 106, 105, 106, 117, 117, 69, 122, 122, 75, 125,
+
+ 126, 125, 127, 1, 5, 8, 83, 12, 12, 13,
+ 13, 86, 14, 14, 15, 15, 16, 16, 17, 18,
+ 18, 18, 21, 30, 35, 35, 36, 38, 38, 53,
+ 47, 47, 48, 49, 53, 69, 48, 49, 53, 69,
+ 82, 82, 87, 87, 93, 93, 97, 97, 100, 100,
+ 102, 102, 68, 113, 69, 123, 127, 1, 1, 3,
+ 5, 12, 12, 13, 13, 14, 14, 17, 18, 21,
+ 38, 38, 38, 41, 41, 42, 53, 48, 48, 49,
+ 53, 69, 82, 82, 85, 85, 95, 95, 97, 97,
+ 100, 100, 100, 68, 121, 69, 123, 3, 12, 12,
+
+ 13, 13, 14, 14, 17, 18, 21, 41, 41, 41,
+ 53, 53, 69, 85, 95, 95, 95, 97, 97, 90,
+ 69, 123, 124, 123, 18, 32, 36, 36, 53, 68,
+ 69, 60, 72, 62, 64, 97, 97, 90, 90, 90,
+ 121, 121, 11, 11, 36, 36, 40, 40, 40, 42,
+ 42, 53, 68, 69, 94, 94, 94, 31, 97, 97,
+ 90, 90, 90, 111, 112, 121, 121, 26, 26, 39,
+ 39, 40, 40, 40, 42, 42, 53, 69, 61, 61,
+ 63, 63, 65, 65, 90, 94, 94, 94, 97, 97,
+ 90, 121, 121, 1, 1, 25, 25, 24, 25, 24,
+
+ 27, 27, 20, 27, 27, 27, 27, 27, 27, 39,
+ 39, 39, 60, 72, 62, 64, 90, 79, 109, 110,
+ 115, 1, 1, 1, 108, 28, 28, 79, 29, 116,
+ 29, 114, 80, 118
} ;
-static yyconst flex_int16_t yy_accept[2914] =
+static yyconst flex_int16_t yy_accept[2917] =
{ 0,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
@@ -760,168 +760,168 @@ static yyconst flex_int16_t yy_accept[2914] =
937, 938, 939, 941, 942, 943, 944, 945, 946, 946,
947, 947, 947, 947, 947, 948, 949, 949, 950, 950,
950, 950, 951, 951, 951, 952, 953, 954, 954, 955,
- 956, 956, 957, 958, 959, 960, 961, 961, 961, 961,
- 961, 961, 961, 961, 961, 962, 963, 964, 966, 967,
- 968, 969, 969, 970, 971, 971, 972, 974, 975, 976,
- 977, 978, 979, 980, 981, 981, 981, 981, 981, 981,
- 981, 982, 982, 982, 982, 982, 982, 982, 982, 982,
-
- 983, 983, 983, 983, 983, 983, 983, 983, 983, 983,
- 984, 985, 986, 987, 987, 987, 988, 988, 988, 988,
- 989, 990, 991, 991, 991, 991, 991, 991, 991, 991,
- 991, 991, 991, 991, 991, 991, 991, 991, 991, 992,
- 992, 992, 993, 994, 995, 996, 996, 996, 996, 996,
- 996, 996, 997, 997, 997, 997, 997, 997, 998, 999,
- 1000, 1001, 1001, 1002, 1003, 1003, 1003, 1003, 1004, 1005,
- 1006, 1006, 1006, 1006, 1006, 1006, 1006, 1006, 1006, 1006,
- 1007, 1008, 1009, 1010, 1011, 1012, 1012, 1012, 1012, 1012,
- 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1020, 1020,
-
- 1020, 1021, 1022, 1023, 1024, 1026, 1027, 1028, 1030, 1030,
- 1031, 1032, 1033, 1035, 1035, 1035, 1035, 1035, 1035, 1035,
- 1036, 1037, 1038, 1038, 1039, 1039, 1039, 1039, 1039, 1040,
- 1040, 1040, 1040, 1040, 1041, 1041, 1041, 1041, 1041, 1042,
- 1042, 1042, 1042, 1042, 1043, 1043, 1043, 1043, 1043, 1044,
- 1044, 1044, 1044, 1044, 1045, 1045, 1045, 1045, 1045, 1046,
- 1046, 1046, 1046, 1046, 1047, 1047, 1047, 1047, 1047, 1048,
- 1048, 1048, 1048, 1048, 1049, 1049, 1049, 1049, 1049, 1050,
- 1050, 1050, 1050, 1050, 1050, 1050, 1050, 1050, 1050, 1051,
- 1051, 1052, 1053, 1053, 1053, 1053, 1053, 1053, 1053, 1053,
-
- 1054, 1054, 1054, 1054, 1054, 1054, 1055, 1055, 1055, 1056,
- 1056, 1056, 1057, 1057, 1057, 1058, 1059, 1060, 1061, 1062,
- 1063, 1063, 1064, 1065, 1066, 1066, 1067, 1068, 1068, 1068,
- 1068, 1068, 1069, 1071, 1072, 1074, 1075, 1076, 1077, 1078,
- 1079, 1081, 1082, 1083, 1083, 1083, 1083, 1084, 1084, 1085,
- 1086, 1087, 1087, 1087, 1087, 1088, 1089, 1090, 1090, 1091,
- 1092, 1092, 1092, 1092, 1092, 1093, 1094, 1095, 1096, 1097,
- 1098, 1098, 1099, 1100, 1100, 1101, 1101, 1102, 1103, 1104,
- 1104, 1104, 1104, 1104, 1104, 1104, 1104, 1104, 1104, 1104,
- 1105, 1105, 1106, 1107, 1108, 1109, 1109, 1109, 1110, 1111,
-
- 1112, 1113, 1114, 1115, 1116, 1116, 1117, 1118, 1119, 1120,
- 1121, 1122, 1123, 1124, 1124, 1124, 1124, 1125, 1126, 1126,
- 1127, 1127, 1128, 1129, 1130, 1132, 1133, 1134, 1135, 1136,
- 1136, 1137, 1138, 1139, 1140, 1140, 1140, 1141, 1141, 1141,
- 1141, 1142, 1142, 1142, 1142, 1143, 1144, 1145, 1146, 1147,
- 1148, 1149, 1149, 1149, 1149, 1149, 1150, 1151, 1151, 1151,
- 1151, 1151, 1152, 1153, 1154, 1156, 1157, 1158, 1159, 1160,
- 1161, 1162, 1163, 1164, 1165, 1165, 1165, 1165, 1166, 1167,
- 1167, 1168, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169,
- 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169,
-
- 1169, 1169, 1169, 1169, 1169, 1169, 1170, 1171, 1172, 1173,
- 1174, 1174, 1174, 1175, 1176, 1177, 1177, 1177, 1178, 1178,
- 1178, 1178, 1180, 1181, 1183, 1183, 1183, 1184, 1185, 1185,
- 1186, 1186, 1186, 1187, 1187, 1188, 1189, 1190, 1190, 1190,
- 1190, 1190, 1190, 1190, 1190, 1190, 1190, 1191, 1191, 1191,
- 1191, 1191, 1191, 1191, 1193, 1193, 1193, 1193, 1193, 1193,
- 1194, 1194, 1194, 1195, 1196, 1197, 1198, 1198, 1198, 1199,
- 1200, 1201, 1202, 1203, 1204, 1204, 1204, 1204, 1204, 1205,
- 1205, 1205, 1205, 1205, 1205, 1205, 1205, 1206, 1207, 1207,
- 1207, 1207, 1207, 1208, 1209, 1209, 1209, 1209, 1209, 1210,
-
- 1211, 1211, 1212, 1213, 1214, 1214, 1215, 1216, 1217, 1218,
- 1218, 1219, 1221, 1222, 1223, 1224, 1225, 1225, 1226, 1226,
- 1226, 1226, 1227, 1228, 1228, 1229, 1230, 1231, 1232, 1233,
- 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1241, 1242, 1243,
- 1243, 1244, 1245, 1247, 1247, 1247, 1247, 1248, 1249, 1249,
- 1249, 1250, 1250, 1251, 1253, 1255, 1256, 1257, 1258, 1259,
- 1260, 1261, 1262, 1262, 1263, 1264, 1265, 1265, 1266, 1267,
- 1268, 1268, 1269, 1270, 1270, 1270, 1271, 1272, 1273, 1274,
- 1275, 1276, 1276, 1276, 1276, 1276, 1277, 1278, 1279, 1280,
- 1280, 1281, 1282, 1284, 1284, 1284, 1284, 1284, 1284, 1284,
-
- 1285, 1286, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287,
- 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287,
- 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287,
- 1287, 1287, 1287, 1287, 1288, 1289, 1290, 1290, 1290, 1291,
- 1292, 1292, 1293, 1294, 1295, 1295, 1296, 1297, 1298, 1299,
- 1300, 1301, 1301, 1302, 1302, 1303, 1303, 1303, 1304, 1304,
- 1304, 1305, 1305, 1305, 1305, 1306, 1307, 1308, 1309, 1310,
- 1311, 1312, 1313, 1314, 1316, 1317, 1318, 1319, 1319, 1319,
- 1320, 1321, 1322, 1323, 1324, 1325, 1326, 1327, 1328, 1329,
- 1330, 1331, 1331, 1331, 1331, 1331, 1331, 1332, 1333, 1333,
-
- 1333, 1333, 1334, 1335, 1335, 1335, 1335, 1335, 1335, 1335,
- 1335, 1335, 1335, 1335, 1336, 1337, 1338, 1339, 1340, 1341,
- 1342, 1343, 1344, 1344, 1345, 1346, 1347, 1348, 1350, 1350,
- 1351, 1351, 1351, 1352, 1352, 1353, 1354, 1357, 1359, 1361,
- 1364, 1366, 1367, 1368, 1368, 1368, 1368, 1368, 1368, 1369,
- 1369, 1370, 1371, 1372, 1372, 1372, 1373, 1374, 1375, 1375,
- 1375, 1375, 1375, 1376, 1377, 1378, 1378, 1378, 1379, 1380,
- 1381, 1382, 1383, 1383, 1383, 1384, 1384, 1385, 1385, 1385,
- 1385, 1386, 1387, 1387, 1388, 1389, 1389, 1390, 1390, 1391,
- 1391, 1391, 1391, 1391, 1391, 1391, 1391, 1391, 1391, 1391,
-
- 1391, 1391, 1391, 1391, 1391, 1391, 1391, 1391, 1391, 1391,
- 1392, 1393, 1393, 1393, 1393, 1394, 1395, 1396, 1397, 1397,
- 1399, 1400, 1400, 1401, 1402, 1402, 1403, 1404, 1404, 1405,
- 1406, 1407, 1408, 1409, 1410, 1410, 1410, 1411, 1412, 1413,
- 1414, 1415, 1416, 1417, 1418, 1419, 1420, 1421, 1421, 1421,
- 1421, 1421, 1421, 1421, 1421, 1422, 1422, 1422, 1423, 1424,
- 1425, 1426, 1427, 1427, 1427, 1427, 1427, 1427, 1427, 1427,
- 1428, 1429, 1430, 1431, 1432, 1433, 1434, 1435, 1436, 1437,
- 1437, 1437, 1437, 1437, 1437, 1438, 1438, 1439, 1440, 1440,
- 1440, 1440, 1440, 1440, 1441, 1441, 1442, 1442, 1443, 1444,
-
- 1444, 1444, 1444, 1444, 1444, 1444, 1445, 1446, 1447, 1448,
- 1449, 1450, 1450, 1451, 1451, 1451, 1451, 1451, 1451, 1451,
- 1451, 1451, 1451, 1451, 1451, 1451, 1451, 1451, 1451, 1451,
- 1451, 1451, 1452, 1452, 1452, 1452, 1452, 1452, 1452, 1453,
- 1454, 1455, 1456, 1457, 1458, 1459, 1459, 1460, 1461, 1462,
- 1463, 1463, 1464, 1465, 1466, 1467, 1468, 1468, 1468, 1468,
- 1468, 1468, 1468, 1468, 1468, 1468, 1468, 1468, 1468, 1469,
- 1470, 1471, 1471, 1471, 1471, 1472, 1472, 1473, 1474, 1475,
- 1476, 1477, 1478, 1479, 1479, 1479, 1479, 1480, 1481, 1482,
- 1482, 1482, 1482, 1482, 1483, 1484, 1484, 1484, 1485, 1485,
-
- 1486, 1486, 1486, 1487, 1488, 1489, 1490, 1491, 1491, 1492,
- 1492, 1492, 1492, 1492, 1492, 1492, 1492, 1492, 1492, 1492,
- 1492, 1492, 1492, 1492, 1492, 1492, 1492, 1492, 1492, 1492,
- 1492, 1492, 1492, 1493, 1494, 1495, 1495, 1495, 1496, 1496,
- 1497, 1498, 1499, 1500, 1501, 1502, 1503, 1504, 1505, 1505,
- 1505, 1505, 1505, 1505, 1505, 1505, 1505, 1505, 1505, 1505,
- 1505, 1505, 1505, 1505, 1506, 1507, 1507, 1508, 1508, 1508,
- 1509, 1510, 1510, 1510, 1510, 1510, 1510, 1510, 1511, 1511,
- 1511, 1511, 1511, 1512, 1512, 1512, 1513, 1514, 1514, 1515,
- 1515, 1515, 1516, 1517, 1518, 1518, 1518, 1518, 1518, 1518,
-
- 1518, 1518, 1518, 1518, 1518, 1518, 1518, 1518, 1518, 1518,
- 1518, 1518, 1518, 1518, 1518, 1518, 1518, 1519, 1521, 1522,
- 1522, 1522, 1522, 1523, 1523, 1523, 1523, 1523, 1523, 1523,
- 1523, 1523, 1524, 1524, 1525, 1526, 1526, 1526, 1526, 1526,
- 1526, 1526, 1527, 1527, 1527, 1527, 1527, 1527, 1528, 1529,
- 1530, 1531, 1532, 1533, 1533, 1533, 1533, 1533, 1533, 1534,
- 1535, 1536, 1537, 1538, 1538, 1538, 1538, 1538, 1538, 1538,
- 1538, 1538, 1538, 1538, 1538, 1538, 1538, 1538, 1538, 1538,
- 1538, 1538, 1539, 1540, 1540, 1540, 1541, 1542, 1542, 1542,
- 1542, 1542, 1542, 1542, 1542, 1542, 1543, 1544, 1544, 1544,
-
- 1545, 1546, 1547, 1548, 1549, 1550, 1550, 1550, 1550, 1550,
- 1550, 1551, 1552, 1552, 1552, 1552, 1552, 1552, 1552, 1552,
- 1553, 1554, 1555, 1556, 1556, 1557, 1558, 1559, 1560, 1561,
- 1561, 1561, 1561, 1561, 1561, 1561, 1561, 1561, 1562, 1562,
- 1563, 1563, 1563, 1563, 1563, 1563, 1564, 1565, 1565, 1565,
- 1565, 1565, 1566, 1567, 1567, 1567, 1567, 1567, 1568, 1569,
- 1570, 1571, 1572, 1573, 1574, 1575, 1575, 1575, 1575, 1575,
- 1575, 1576, 1577, 1578, 1579, 1580, 1581, 1582, 1583, 1584,
- 1585, 1586, 1586, 1587, 1588, 1589, 1589, 1589, 1589, 1589,
- 1589, 1589, 1589, 1589, 1589, 1589, 1589, 1589, 1589, 1589,
-
- 1590, 1591, 1592, 1593, 1594, 1595, 1596, 1597, 1598, 1599,
- 1599, 1599, 1600, 1602, 1603, 1604, 1605, 1606, 1606, 1607,
- 1607, 1607, 1607, 1607, 1608, 1609, 1610, 1611, 1612, 1613,
- 1614, 1614, 1615, 1616, 1616, 1616, 1616, 1617, 1618, 1618,
- 1619, 1619, 1619, 1619, 1619, 1620, 1621, 1622, 1622, 1622,
- 1622, 1622, 1622, 1622, 1622, 1622, 1622, 1622, 1622, 1622,
- 1622, 1622, 1622, 1623, 1623, 1623, 1623, 1623, 1624, 1624,
- 1624, 1624, 1624, 1624, 1624, 1624, 1624, 1624, 1624, 1625,
- 1625, 1626, 1626, 1626, 1626, 1626, 1626, 1626, 1626, 1627,
- 1627, 1627, 1627, 1627, 1628, 1628, 1628, 1629, 1629, 1630,
-
- 1630, 1630, 1630, 1630, 1630, 1630, 1630, 1630, 1631, 1631,
- 1631, 1632, 1632
+ 956, 956, 957, 958, 959, 960, 961, 962, 962, 962,
+ 962, 962, 962, 962, 962, 963, 964, 965, 967, 968,
+ 969, 970, 970, 971, 972, 972, 973, 975, 976, 977,
+ 978, 979, 980, 981, 982, 982, 982, 982, 982, 982,
+ 982, 983, 983, 983, 983, 983, 983, 983, 983, 983,
+
+ 984, 984, 984, 984, 984, 984, 984, 984, 984, 984,
+ 985, 986, 987, 988, 988, 988, 989, 989, 989, 989,
+ 990, 991, 992, 992, 992, 992, 992, 992, 992, 992,
+ 992, 992, 992, 992, 992, 992, 992, 992, 992, 993,
+ 993, 993, 994, 995, 996, 997, 997, 997, 997, 997,
+ 997, 997, 998, 998, 998, 998, 998, 998, 999, 1000,
+ 1001, 1002, 1002, 1003, 1004, 1004, 1004, 1004, 1005, 1006,
+ 1007, 1007, 1007, 1007, 1007, 1007, 1007, 1007, 1007, 1007,
+ 1008, 1009, 1010, 1011, 1012, 1013, 1013, 1013, 1013, 1013,
+ 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1021, 1021,
+
+ 1021, 1022, 1023, 1024, 1025, 1027, 1028, 1029, 1031, 1031,
+ 1032, 1033, 1034, 1036, 1036, 1036, 1036, 1036, 1036, 1036,
+ 1037, 1038, 1039, 1039, 1040, 1040, 1040, 1040, 1040, 1041,
+ 1041, 1041, 1041, 1041, 1042, 1042, 1042, 1042, 1042, 1043,
+ 1043, 1043, 1043, 1043, 1044, 1044, 1044, 1044, 1044, 1045,
+ 1045, 1045, 1045, 1045, 1046, 1046, 1046, 1046, 1046, 1047,
+ 1047, 1047, 1047, 1047, 1048, 1048, 1048, 1048, 1048, 1049,
+ 1049, 1049, 1049, 1049, 1050, 1050, 1050, 1050, 1050, 1051,
+ 1051, 1051, 1051, 1051, 1051, 1051, 1051, 1051, 1051, 1052,
+ 1052, 1053, 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054,
+
+ 1055, 1055, 1055, 1055, 1055, 1055, 1056, 1056, 1056, 1057,
+ 1057, 1057, 1058, 1058, 1058, 1059, 1060, 1061, 1062, 1063,
+ 1064, 1064, 1065, 1066, 1067, 1067, 1068, 1069, 1069, 1069,
+ 1069, 1069, 1070, 1072, 1073, 1075, 1076, 1077, 1078, 1079,
+ 1080, 1082, 1083, 1084, 1084, 1084, 1084, 1085, 1085, 1086,
+ 1087, 1088, 1088, 1088, 1088, 1089, 1090, 1091, 1091, 1092,
+ 1093, 1093, 1093, 1093, 1093, 1094, 1095, 1096, 1096, 1096,
+ 1097, 1098, 1099, 1099, 1100, 1101, 1101, 1102, 1102, 1103,
+ 1104, 1105, 1105, 1105, 1105, 1105, 1105, 1105, 1105, 1105,
+ 1105, 1105, 1106, 1106, 1107, 1108, 1109, 1110, 1110, 1110,
+
+ 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1117, 1118, 1119,
+ 1120, 1121, 1122, 1123, 1124, 1125, 1125, 1125, 1125, 1126,
+ 1127, 1127, 1128, 1128, 1129, 1130, 1131, 1133, 1134, 1135,
+ 1136, 1137, 1137, 1138, 1139, 1140, 1141, 1141, 1141, 1142,
+ 1142, 1142, 1142, 1143, 1143, 1143, 1143, 1144, 1145, 1146,
+ 1147, 1148, 1149, 1150, 1150, 1150, 1150, 1150, 1151, 1152,
+ 1152, 1152, 1152, 1152, 1153, 1154, 1155, 1157, 1158, 1159,
+ 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1166, 1166, 1166,
+ 1167, 1168, 1168, 1169, 1170, 1170, 1170, 1170, 1170, 1170,
+ 1170, 1170, 1170, 1170, 1170, 1170, 1170, 1170, 1170, 1170,
+
+ 1170, 1170, 1170, 1170, 1170, 1170, 1170, 1170, 1171, 1172,
+ 1173, 1174, 1175, 1175, 1175, 1176, 1177, 1178, 1178, 1178,
+ 1179, 1179, 1179, 1179, 1181, 1182, 1184, 1184, 1184, 1185,
+ 1186, 1186, 1187, 1187, 1187, 1188, 1188, 1189, 1190, 1191,
+ 1191, 1191, 1191, 1191, 1191, 1191, 1191, 1191, 1191, 1192,
+ 1192, 1192, 1192, 1192, 1192, 1192, 1194, 1194, 1194, 1194,
+ 1194, 1194, 1195, 1195, 1195, 1196, 1197, 1198, 1199, 1199,
+ 1199, 1200, 1201, 1202, 1203, 1204, 1205, 1205, 1205, 1205,
+ 1205, 1206, 1206, 1206, 1206, 1206, 1206, 1206, 1206, 1207,
+ 1208, 1208, 1208, 1208, 1208, 1209, 1210, 1210, 1210, 1210,
+
+ 1210, 1211, 1212, 1212, 1213, 1214, 1215, 1215, 1216, 1217,
+ 1218, 1219, 1219, 1220, 1222, 1223, 1224, 1225, 1226, 1226,
+ 1227, 1227, 1227, 1227, 1228, 1229, 1229, 1230, 1231, 1232,
+ 1233, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1241, 1242,
+ 1243, 1244, 1244, 1245, 1246, 1248, 1248, 1248, 1248, 1249,
+ 1250, 1250, 1250, 1251, 1251, 1252, 1254, 1256, 1257, 1258,
+ 1259, 1260, 1261, 1262, 1263, 1263, 1264, 1265, 1266, 1266,
+ 1267, 1268, 1269, 1269, 1270, 1271, 1271, 1271, 1272, 1273,
+ 1274, 1275, 1276, 1277, 1277, 1277, 1277, 1277, 1278, 1279,
+ 1280, 1281, 1281, 1282, 1283, 1285, 1285, 1285, 1285, 1285,
+
+ 1285, 1285, 1286, 1287, 1288, 1288, 1288, 1288, 1288, 1288,
+ 1288, 1288, 1288, 1288, 1288, 1288, 1288, 1288, 1288, 1288,
+ 1288, 1288, 1288, 1288, 1288, 1288, 1288, 1288, 1288, 1288,
+ 1288, 1288, 1288, 1288, 1288, 1288, 1289, 1290, 1291, 1291,
+ 1291, 1292, 1293, 1293, 1294, 1295, 1296, 1296, 1297, 1298,
+ 1299, 1300, 1301, 1302, 1302, 1303, 1303, 1304, 1304, 1304,
+ 1305, 1305, 1305, 1306, 1306, 1306, 1306, 1307, 1308, 1309,
+ 1310, 1311, 1312, 1313, 1314, 1315, 1317, 1318, 1319, 1320,
+ 1320, 1320, 1321, 1322, 1323, 1324, 1325, 1326, 1327, 1328,
+ 1329, 1330, 1331, 1332, 1332, 1332, 1332, 1332, 1332, 1333,
+
+ 1334, 1334, 1334, 1334, 1335, 1336, 1336, 1336, 1336, 1336,
+ 1336, 1336, 1336, 1336, 1336, 1336, 1337, 1338, 1339, 1340,
+ 1341, 1342, 1343, 1344, 1345, 1346, 1347, 1348, 1349, 1350,
+ 1352, 1353, 1354, 1354, 1354, 1355, 1355, 1356, 1357, 1360,
+ 1362, 1364, 1367, 1369, 1370, 1371, 1371, 1371, 1371, 1371,
+ 1371, 1372, 1372, 1373, 1374, 1375, 1375, 1375, 1376, 1377,
+ 1378, 1378, 1378, 1378, 1378, 1379, 1380, 1381, 1381, 1381,
+ 1382, 1383, 1384, 1385, 1386, 1386, 1386, 1387, 1387, 1388,
+ 1388, 1388, 1388, 1389, 1390, 1390, 1391, 1392, 1392, 1393,
+ 1393, 1394, 1394, 1394, 1394, 1394, 1394, 1394, 1394, 1394,
+
+ 1394, 1394, 1394, 1394, 1394, 1394, 1394, 1394, 1394, 1394,
+ 1394, 1394, 1395, 1396, 1396, 1396, 1396, 1397, 1398, 1399,
+ 1400, 1400, 1402, 1403, 1403, 1404, 1405, 1405, 1406, 1407,
+ 1407, 1408, 1409, 1410, 1411, 1412, 1413, 1413, 1413, 1414,
+ 1415, 1416, 1417, 1418, 1419, 1420, 1421, 1422, 1423, 1424,
+ 1424, 1424, 1424, 1424, 1424, 1424, 1424, 1425, 1425, 1425,
+ 1426, 1427, 1428, 1429, 1430, 1430, 1430, 1430, 1430, 1430,
+ 1430, 1430, 1431, 1432, 1433, 1434, 1435, 1436, 1437, 1438,
+ 1439, 1439, 1440, 1440, 1440, 1440, 1440, 1440, 1441, 1441,
+ 1442, 1443, 1443, 1443, 1443, 1443, 1443, 1444, 1444, 1445,
+
+ 1445, 1446, 1447, 1447, 1447, 1447, 1447, 1447, 1447, 1448,
+ 1449, 1450, 1451, 1452, 1453, 1453, 1454, 1454, 1454, 1454,
+ 1454, 1454, 1454, 1454, 1454, 1454, 1454, 1454, 1454, 1454,
+ 1454, 1454, 1454, 1454, 1454, 1455, 1455, 1455, 1455, 1455,
+ 1455, 1455, 1456, 1457, 1458, 1459, 1460, 1461, 1462, 1462,
+ 1463, 1464, 1465, 1466, 1466, 1467, 1468, 1469, 1470, 1471,
+ 1471, 1471, 1471, 1471, 1471, 1471, 1471, 1471, 1471, 1471,
+ 1471, 1471, 1472, 1473, 1474, 1474, 1474, 1474, 1475, 1475,
+ 1476, 1477, 1478, 1479, 1480, 1481, 1482, 1482, 1482, 1482,
+ 1483, 1484, 1485, 1485, 1485, 1485, 1485, 1486, 1487, 1487,
+
+ 1487, 1488, 1488, 1489, 1489, 1489, 1490, 1491, 1492, 1493,
+ 1494, 1494, 1495, 1495, 1495, 1495, 1495, 1495, 1495, 1495,
+ 1495, 1495, 1495, 1495, 1495, 1495, 1495, 1495, 1495, 1495,
+ 1495, 1495, 1495, 1495, 1495, 1495, 1496, 1497, 1498, 1498,
+ 1498, 1499, 1499, 1500, 1501, 1502, 1503, 1504, 1505, 1506,
+ 1507, 1508, 1508, 1508, 1508, 1508, 1508, 1508, 1508, 1508,
+ 1508, 1508, 1508, 1508, 1508, 1508, 1508, 1509, 1510, 1510,
+ 1511, 1511, 1511, 1512, 1513, 1513, 1513, 1513, 1513, 1513,
+ 1513, 1514, 1514, 1514, 1514, 1514, 1515, 1515, 1515, 1516,
+ 1517, 1517, 1518, 1518, 1518, 1519, 1520, 1521, 1521, 1521,
+
+ 1521, 1521, 1521, 1521, 1521, 1521, 1521, 1521, 1521, 1521,
+ 1521, 1521, 1521, 1521, 1521, 1521, 1521, 1521, 1521, 1521,
+ 1522, 1524, 1525, 1525, 1525, 1525, 1526, 1526, 1526, 1526,
+ 1526, 1526, 1526, 1526, 1526, 1527, 1527, 1528, 1529, 1529,
+ 1529, 1529, 1529, 1529, 1529, 1530, 1530, 1530, 1530, 1530,
+ 1530, 1531, 1532, 1533, 1534, 1535, 1536, 1536, 1536, 1536,
+ 1536, 1536, 1537, 1538, 1539, 1540, 1541, 1541, 1541, 1541,
+ 1541, 1541, 1541, 1541, 1541, 1541, 1541, 1541, 1541, 1541,
+ 1541, 1541, 1541, 1541, 1541, 1542, 1543, 1543, 1543, 1544,
+ 1545, 1545, 1545, 1545, 1545, 1545, 1545, 1545, 1545, 1546,
+
+ 1547, 1547, 1547, 1548, 1549, 1550, 1551, 1552, 1553, 1553,
+ 1553, 1553, 1553, 1553, 1554, 1555, 1555, 1555, 1555, 1555,
+ 1555, 1555, 1555, 1556, 1557, 1558, 1559, 1559, 1560, 1561,
+ 1562, 1563, 1564, 1564, 1564, 1564, 1564, 1564, 1564, 1564,
+ 1564, 1565, 1565, 1566, 1566, 1566, 1566, 1566, 1566, 1567,
+ 1568, 1568, 1568, 1568, 1568, 1569, 1570, 1570, 1570, 1570,
+ 1570, 1571, 1572, 1573, 1574, 1575, 1576, 1577, 1578, 1578,
+ 1578, 1578, 1578, 1578, 1579, 1580, 1581, 1582, 1583, 1584,
+ 1585, 1586, 1587, 1588, 1589, 1589, 1590, 1591, 1592, 1592,
+ 1592, 1592, 1592, 1592, 1592, 1592, 1592, 1592, 1592, 1592,
+
+ 1592, 1592, 1592, 1593, 1594, 1595, 1596, 1597, 1598, 1599,
+ 1600, 1601, 1602, 1602, 1602, 1603, 1605, 1606, 1607, 1608,
+ 1609, 1609, 1610, 1610, 1610, 1610, 1610, 1611, 1612, 1613,
+ 1614, 1615, 1616, 1617, 1617, 1618, 1619, 1619, 1619, 1619,
+ 1620, 1621, 1621, 1622, 1622, 1622, 1622, 1622, 1623, 1624,
+ 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625,
+ 1625, 1625, 1625, 1625, 1625, 1625, 1626, 1626, 1626, 1626,
+ 1626, 1627, 1627, 1627, 1627, 1627, 1627, 1627, 1627, 1627,
+ 1627, 1627, 1628, 1628, 1629, 1629, 1629, 1629, 1629, 1629,
+ 1629, 1629, 1630, 1630, 1630, 1630, 1630, 1631, 1631, 1631,
+
+ 1632, 1632, 1633, 1633, 1633, 1633, 1633, 1633, 1633, 1633,
+ 1633, 1634, 1634, 1634, 1635, 1635
} ;
static yyconst flex_int32_t yy_ec[256] =
@@ -969,7 +969,7 @@ static yyconst flex_int32_t yy_meta[85] =
8, 8, 11, 11
} ;
-static yyconst flex_int16_t yy_base[3463] =
+static yyconst flex_int16_t yy_base[3466] =
{ 0,
0, 9, 93, 0, 177, 0, 259, 263, 269, 0,
353, 0, 437, 0, 521, 0, 605, 0, 689, 0,
@@ -984,12 +984,12 @@ static yyconst flex_int16_t yy_base[3463] =
3061, 0, 3145, 0, 2866, 3227, 3251, 0, 3335, 0,
3419, 0, 3503, 0, 3587, 0, 3671, 0, 3755, 0,
- 3839, 0,11432,26921,26921,26921,26921,26921, 0, 3922,
- 4004, 4085, 0, 1817, 2719,11381, 2729, 1799, 2836,11393,
- 3200, 1822, 2848, 232, 1420,11380, 3898,11365, 3899, 3916,
- 3932, 1802,11351, 2841,11354, 4165, 0, 0,26921,26921,
- 26921,26921,26921,26921, 3165,11375,26921,26921,26921, 0,
- 0, 0, 4249,11370, 0,26921,11369, 4321, 4340, 4359,
+ 3839, 0,11461,26921,26921,26921,26921,26921, 0, 3922,
+ 4004, 4085, 0, 1817, 2719,11408, 2729, 1799, 2836,11410,
+ 3200, 1822, 2848, 232, 1420,11397, 3898,11374, 3899, 3916,
+ 3932, 1802,11384, 2841,11380, 4165, 0, 0,26921,26921,
+ 26921,26921,26921,26921, 3165,11377,26921,26921,26921, 0,
+ 0, 0, 4249,11372, 0,26921,11371, 4321, 4340, 4359,
26921,26921, 3908, 3985, 4427, 0, 0, 4019, 4036, 4507,
0, 0, 4591, 0, 0, 0, 4046, 0, 4671, 2744,
@@ -1006,33 +1006,33 @@ static yyconst flex_int16_t yy_base[3463] =
4831, 6031, 0, 0, 4844, 4854, 6111, 0, 0, 4864,
4874, 6191, 0, 0,26921, 3921, 3959, 4983,26921, 2,
- 26921,26921,26921, 4927, 4937, 4947,11372,26921, 4957, 0,
+ 26921,26921,26921, 4927, 4937, 4947,11374,26921, 4957, 0,
1796, 3926, 0, 3943, 0, 0, 6257, 0, 0, 4991,
5004, 6336, 0, 0, 5014, 5024, 6416, 0, 0, 5034,
5071, 6496, 0, 0, 5084, 5094, 6576, 0, 0, 0,
0, 0, 0,26921, 0, 0,26921, 5104, 5114, 6656,
- 0, 0, 0, 0, 0,11340, 5151, 5164, 6736, 0,
- 0, 0, 0, 0,11339, 5174, 5184, 6816, 0, 0,
+ 0, 0, 0, 0, 0,11342, 5151, 5164, 6736, 0,
+ 0, 0, 0, 0,11341, 5174, 5184, 6816, 0, 0,
5194, 5231, 6896, 0, 0, 5244, 5254, 6976, 0, 0,
- 26921,26921,26921,26921, 0, 0, 0,11372, 5264, 5274,
+ 26921,26921,26921,26921, 0, 0, 0,11374, 5264, 5274,
7056, 0, 0, 5311, 5324, 7136, 0, 0, 5334, 5344,
7216, 0, 0,26921,26921, 0, 0, 0, 5354, 5391,
7296, 0, 0, 5404, 5414, 7376, 0, 0, 5424, 5434,
7456, 0, 0, 5471, 5484, 7536, 0, 0, 0, 7619,
- 7700, 2868,26921, 7780, 0, 0, 7863, 0, 4030,11327,
- 3915, 2601, 5538,11328, 5540, 2616, 5541, 1424, 1776,11314,
- 5551,11293, 5542, 5597, 5613, 3195,11301, 4062,11300, 0,
- 4067,11301, 3890,11279,11269, 2768,11274, 2606,11276, 2723,
- 11259, 5594, 2832,11243,11240,11228, 5557,11244, 2839,11237,
-
- 11210,11189,11194, 3887,11204, 3923,11161, 3922,11159,11149,
- 4685, 3903, 4686,11149,11141, 5623, 1801, 5583,11141, 2847,
- 5677,11130,11132,11099, 3932, 4109, 4055, 5662,11116, 4148,
- 11115,11087, 5665, 3910, 4123,11084,11055, 4434,11058, 5674,
- 11036, 4121, 7914,11044, 3949,11031,11036, 4687, 4955, 0,
+ 7700, 2868,26921, 7780, 0, 0, 7863, 0, 4030,11329,
+ 3915, 2601, 5538,11351, 5540, 2616, 5541, 1424, 1776,11338,
+ 5551,11309, 5542, 5597, 5613, 3195,11318, 4062,11308, 0,
+ 4067,11308, 3890,11287,11284, 2768,11292, 2606,11287, 2723,
+ 11265, 5594, 2832,11269,11264,11231, 5557,11246, 2839,11239,
+
+ 11246,11225,11198, 3887,11208, 3923,11204, 3922,11202,11151,
+ 4685, 3903, 4686,11154,11147, 5623, 1801, 5583,11149, 2847,
+ 5677,11137,11139,11128, 3932, 4109, 4055, 5662,11142, 4148,
+ 11119,11115, 5665, 3910, 4123,11112,11069, 4434,11083, 5674,
+ 11061, 4121, 7914,11058, 3949,11033,11037, 4687, 4955, 0,
0,26921, 0, 0, 0,26921,26921, 0, 0, 7998,
- 5649,11059,26921,11058, 8070, 8089, 8108, 0, 8127, 5652,
+ 5649,11060,26921,11059, 8070, 8089, 8108, 0, 8127, 5652,
8195, 0, 0, 0, 0, 5662, 8275, 0, 0, 0,
0, 8359, 0, 5682, 8439, 4149, 4960, 5724, 8519, 0,
0, 0, 0, 4961, 0, 5711, 8599, 0, 0, 0,
@@ -1095,30 +1095,30 @@ static yyconst flex_int16_t yy_base[3463] =
7141, 7340,12270, 7382,12354,12438, 6676, 5086, 5250, 6582,
5330, 5331, 6751, 6909, 5404, 6916,12520, 6584, 6585, 7579,
7143, 7420, 5247, 5089, 7607, 5804, 7341, 7608, 7448, 7500,
- 8199,10840,12604, 6993,26921,26921,10780, 0, 0, 7641,
+ 8199,10828,12604, 6993,26921,26921,10780, 0, 0, 7641,
7384, 5405, 5169,10803, 7501, 7642, 7529, 7726,12688, 7463,
12772, 5328,10802, 7684, 6632, 7822, 7304, 5566, 6326, 5644,
5811, 5890, 5891, 6727, 6588, 5964,10787, 6652,10756, 5492,
- 10768,10767, 6730,10740,10714, 7823, 8082, 7729, 7541, 7852,
+ 10768,10767, 6730,10728,10714, 7823, 8082, 7729, 7541, 7852,
5329,10722, 7670, 6429, 8083, 6187, 5714, 5408, 6665, 5409,
6800, 5489,10700,10678,10684,10671,10682,10660,10658, 8200,
8443, 7671, 7731, 7000, 8267, 8084, 7543, 7824, 8202, 7542,
8444, 8085, 7075, 8201, 8281, 8207, 7078, 5690, 7540, 6738,
- 8214, 8446, 7682, 8280, 6051,10641, 6509, 5718, 6660,10634,
+ 8214, 8446, 7682, 8280, 6051,10642, 6509, 5718, 6660,10635,
10634,10590, 8445, 8458, 7763, 8526, 5808, 8459, 8090, 8524,
5996, 5246, 8562, 8591, 8525, 8532, 8527, 8534,10602, 7049,
10589, 6430, 8533, 5886, 8592, 8096, 8594, 8599, 8451, 8606,
10627, 8097, 8609, 8617, 8611, 8670, 8614, 6516, 8616, 6743,
- 12856, 8215, 7105, 6791,10576,10566, 8644, 5889,10564, 8672,
+ 12856, 8215, 7105, 6791,10576,10566, 8644, 5889,10563, 8672,
0, 8643, 8679, 8692, 8703, 8723, 8783, 8803, 8863, 8883,
- 8943, 8963, 9023, 9043, 9103, 9123, 9183,10591, 9203, 9263,
+ 8943, 8963, 9023, 9043, 9103, 9123, 9183,10590, 9203, 9263,
9283,10582, 9343, 9363, 9423, 9443, 9503, 9523,10553, 9583,
9603, 9663, 9683,10552, 9743, 9763, 9823, 0, 8275, 8759,
9903, 9913, 9993,10003,10059,10139,10149,10204,10234,10285,
10297,10365,10377,10445,10551,10457,10525,10537,10550,10605,
- 10617,10685,10697,10765,10777,10538,10845,10857,10925,10937,
+ 10617,10685,10697,10765,10777,10526,10845,10857,10925,10937,
10511,11005,11017,11085,10454, 5997, 6077,12939,13020, 6154,
13102,26921,26921,13184, 0, 0, 8213, 7301, 7140, 6302,
13268, 8686,26921, 6771,26921, 7609, 0, 0, 0, 8786,
@@ -1136,276 +1136,276 @@ static yyconst flex_int16_t yy_base[3463] =
9165,10416, 8535, 9166, 9187,10404, 9086, 9206, 9243, 8269,
9231, 9244, 9072, 9319, 9073,10407,10401,10380,10347,10356,
- 10348,10346,10315,10313, 6312,10318,10262,10246, 8465, 9153,
- 9245, 8467, 9886, 9324,10243, 8726, 9366,10241, 9234, 9233,
- 9310, 8787, 9268, 9246, 9347, 9087, 9395, 9403,10227,10239,
- 10187,10195, 9474,10207,10184, 8788, 9404,10167, 9325, 9348,
- 10113, 9479, 8913, 8946, 9312, 9484, 9485, 9152, 9393,10149,
- 9406,10147, 9559,13772, 9507, 9564,10113, 9550, 9313, 9409,
- 9428, 9487,10087,10045,10044, 9606, 9639, 9472, 9551, 9473,
- 9630, 9644, 9666,10032,10008, 9553, 9569, 9966,11148, 9567,
- 9632, 9486, 9566,10026, 9633, 9719, 9712, 9686, 9978, 9646,
- 10002, 9713, 5981, 9647, 9668, 9725, 9790, 9728, 9791, 9766,
+ 10348,10346,10314,10305, 6312,10276,10248,10231, 8465, 9153,
+ 9245, 8467, 9886, 9324,10242, 8726, 9366,10240, 9234, 9233,
+ 9310, 8787, 9268, 9246, 9347, 9087, 9395, 9403,10221,10209,
+ 10186,10194, 9474,10206,10149, 8788, 9404,10127, 9325, 9348,
+ 10110, 9479, 8913, 8946, 9312, 9484, 9485, 9152, 9393,10147,
+ 9406,10138, 9559,13772, 9507, 9564,10097, 9550, 9313, 9409,
+ 9428, 9487,10049,10044,10018, 9606, 9639, 9472, 9551, 9473,
+ 9630, 9644, 9666,10027,10003, 9553, 9569, 9966,11148, 9567,
+ 9632, 9486, 9566, 9982, 9633, 9719, 9712, 9686, 9977, 9646,
+ 9993, 9713, 5981, 9647, 9668, 9725, 9790, 9728, 9791, 9766,
9748, 9804, 9809, 9792,13856, 9871,26921,26921, 9885, 0,
- 0, 0, 9908, 9951, 9806, 9807, 9954, 9931, 9793, 9844,
- 9915, 9872, 9959,10309,10389,10469,10549,10629,10709,10789,
+ 0, 0, 9908, 9951, 9806, 9807, 9926, 9920, 9793, 9844,
+ 9907, 9872, 9959,10309,10389,10469,10549,10629,10709,10789,
10869,10949,11029,11165,11178,11188,11268,11317,11327,11348,
11407,11454,11488,11499,11515,11809,11819,11829,11839,11955,
11981,11991,10217,11144,12954,12971,12984,13001,13938,13948,
13978,13989,14029,14040,14070,14080,14121,14131,14161,14172,
14212,14223,14253,14263,14304,14314,14344,14355,14395,14406,
14436,14446, 0,14527,14608,11798,10113,11940,14690,10123,
- 9846, 9873, 9879,11473, 9912,11468,10188,11941,11149,11800,
+ 9846, 9873, 9879,11473, 9890,11468,10188,11941,11149,11800,
11958,11151,11963,12044,10125,12046,12522,11295,12524,12940,
- 9899,12942,12945,11298,12952,12959,13186, 9883,13188, 9884,
- 9909,10124, 9906,11296,13940,12962,11304,13941,13950,13947,
+ 9897,12942,12945,11298,12952,12959,13186, 9867,13188, 9884,
+ 9909,10124, 9874,11296,13940,12962,11304,13941,13950,13947,
13959,13952,14774,26921,26921,14042,14528,14856, 0, 0,
- 12001, 9845, 9822,14133,14225,11470,14135,14186,14316, 9790,
+ 12001, 9814, 9808,14133,14225,11470,14135,14186,14316, 9789,
14318,14408,11498,14369, 9915,14460,11309,14501,14534,11310,
- 14539,14546, 9803, 9787,14551,11410, 9918,14588,14611, 9784,
- 14609,14621,11411,14623, 9920, 9921,14858, 9756,14860,14927,
- 11412,14973,11493, 9923,15056,11514,15101,11580,11596, 9952,
- 9953,11499,15059,26921,15068,15071,11817,15081,15184, 9773,
-
- 15090,15185,15186,15195,15197,15198,15232, 9724, 9720,11805,
- 11942, 9722,11807, 9709, 9705, 9686, 9690, 9695, 9691,11950,
- 9683, 9959,15207,11812,15244,15246, 9667,15258, 9967,15269,
- 9968,15249,11949,15259,15271,15292,10043, 9645, 9660, 9650,
- 9998,10055, 9644, 9622,15304, 9606,15306,15318, 9613,15317,
- 10005,15329,15351, 9602, 9565,15352,11974,15363, 9603,12004,
- 15372,15377,15423,26921,26921,15506, 0, 0, 0,12954,
- 10086,15508, 9549,15509,15511,12530,15518,15521,15528, 9546,
- 26921, 9526,15559,15566,15569,15571,15576, 9528, 9520,15578,
- 15588,15589,15619,15626,15631,15636,15637,15648, 9485, 9466,
-
- 9465, 9454, 6932, 9446, 6041,10045,15678,12533,15689, 9443,
- 15679,15690,15696, 9404,10046,15731,12544,15738,15701, 9417,
- 15743,12535,12542,13065,15772,13067,15757,15818,12953,10048,
- 10053,15769,10058, 9410,15901,26921,15903,15904,15906,15923,
- 15918, 9393, 9396, 9373,15949,15956,13000,15961, 9378,15968,
- 9362,15974, 9365,16020,16103, 9359,16104,16105, 9361,16114,
- 16116, 9336,16123,16125, 9303,16126,16171, 9316,16173,16174,
- 16176,26921,16185, 9328,16188,16219,13196,16226,16233,16236,
- 9300, 9285,16238,16239, 9289,16284,16245,26921,16286, 9286,
- 16291,13197,10085,16293,16296,16306, 9260, 9263,16305,13194,
-
- 16339,16348,16351,26921,16353,16360,26921,16362, 0,13199,
- 26921,26921, 0, 0, 0,10112,10114, 0, 0,13204,
- 0, 0, 0, 0,16365,10115,16374,13206,16399, 9258,
- 16408,16411,16428,16417,16429,16438, 9232, 9215,13209,14072,
- 13960,26921,14073, 9209, 9201, 9189, 9182,26921, 9171,14166,
- 14088, 9151,10143,14255,14642,16458,26921,16459,13207,16471,
- 10148,10154,16477,10189,16489,14347,16496,16506,10335,10194,
- 9151,10415,14438,10256, 9161,10495,16518,26921,16527,16539,
- 9144,16548,16545, 9129, 9133,16557,14349,16560,10195,16582,
- 9147,13953, 9124,16591,10257,14164,16602,10197,10258,16603,
-
- 26921,16605,16608, 9098,16617,16620, 9062, 9048,16650,16651,
- 16663,16662,11110,11111,16669,16672,16681,16710,16715,16720,
- 16727,16730,16740,16741, 9034, 9013, 9019, 8999, 8993, 8992,
- 8985, 8987, 8968,10202,16775, 8972,16778,16787,16789,26921,
- 16796,16790,16835, 8955,16837,16838, 6116,10203,16844,14272,
- 14536,16858,14256, 8953,16847,16864,16889,16937, 8919, 8917,
- 10575,17020,17021, 8886,17022, 8901,17023, 8893,17032, 0,
- 14573,26921,10655,26921, 0, 0, 0,17041, 8887,17070,
- 8845,17071,17080, 8844,17082,26921,17092,17091, 8830,17093,
- 17105,17139,17142, 8836,17144,16476,17151,14559,17153, 8842,
-
- 8829,17156, 8809,17162,17173, 8802,17203, 8818,17204,17214,
- 8772, 8766,17215,17234,17221,14614,17260,10204,17263, 8751,
- 17272,17273,17283,17282,17320, 8740, 8698,14439, 8695,14656,
- 8669, 8673,26921, 8684,10259,14865,14870,14877,17285,14885,
- 10269,17330,10338,17332,17337,17339,17349,17380,17386, 8664,
- 10735,10815,14928,10300, 8666,10895,17391, 8655,17398,17401,
- 8650, 8651,17411,14622,10337,17432,17444,10349,14872,17446,
- 10340,11135,17449, 8634,17458,17461,26921, 8598,17467,17487,
- 17506,14879,17509,17512,14929,17521,10350,11138,10417,11060,
- 17524,17543,17554,17566,17580,17585,17591,17592,17597,17611,
-
- 8596, 8595, 8571, 8578, 8590, 8562, 8525, 8531, 8531,17614,
- 26921,17634,17644,17645,17657,17656,26921,10418,17659,14930,
- 14935, 8528,17671,17682, 8505,10420,10975,17702,26921,17704,
- 8514,17705, 8504,10429,11055,14940,17716, 8444,17717,26921,
- 17719,26921,17739,17750,26921,17762,17764,17765,17787, 8441,
- 17799,17810, 8424, 8417,17816, 8276, 8257,17817,26921,17822,
- 17828,10430,17851, 8248,17837,17863, 8186, 8176,17876,17886,
- 17885,26921,17888,17898,17911,17923, 8162, 8091,15337,14942,
- 8094, 8065, 8057,11229,15364,15110,17936,11322,17937,17946,
- 17948,17958,17949,17991, 8051,11228, 8078,17997,26921,18000,
-
- 18006, 8055, 7876,18003,18017,18018,11311,18020,10497,18051,
- 26921,18063,18065,18068,18070,10498,11320,18088,18085,18111,
- 18122,18123,18133,18144,18153,18171, 7852, 7851, 7844, 7844,
- 7847,26921, 7841, 7843, 7827,18156,15058,18174,18183,10500,
- 18194,18204,15076,11989,10509,18209,18216, 7826,18221, 7806,
- 10510,18231, 7805,18243,18254,18264,18266,15061,18269, 7770,
- 18291,18303, 7786, 7775, 6892, 7762,18314,15691,18316,14941,
- 10577,18321, 7766,18326,18333, 7757,10578,18336,18359,18371,
- 26921,26921,15371,10545, 7740, 7737,15596,18374,10590,18384,
- 18386,18396,18393, 0,12536, 7722,18405,18431, 7735,14943,
-
- 18419,18438,18444,11150,18450,15119,10657,18456,18457,18469,
- 18479,11553,10658,18491,18504,18498,18510,18521,18516,18533,
- 18547,18558, 7734, 7725, 7703, 7719, 7681, 7685, 7683, 7674,
- 18564, 7646,18559,18584,15397,18585, 7605,12947,18597,18607,
- 26921,18610,26921,18619,26921,18630,18632,18644,18649, 7608,
- 18661, 7593,18667,18678, 7600, 7586, 7592, 7572,18679, 7585,
- 18692, 7553,18690,18709,15131, 7537,10660,18712,15699,18729,
- 15519, 7511, 7523, 7496, 7496, 7513, 7478,15586,18727,18738,
- 18750,18739,13942, 7478,18760,18785,15639, 7478,15398,18795,
- 18797,18812,11457,18798,18815,10669,18832,18845,18850,18862,
-
- 18875,18880,18897,18909, 7463, 7445, 7439, 7424, 7427, 7412,
- 7423, 7403,18914, 7402,18926,15777,18943,15646,15649,18936,
- 7386,18948,18955,18957, 7377,18969, 7357,18986,19000, 7344,
- 7347,26921, 7322,19003, 7324,19006, 7301,19012,10670,19023,
- 7281,15913, 7263, 7274, 7261, 7225, 7216, 7216,15986,19021,
- 19033,19054,19060, 7197,19066,10737,19071,19080,19083,10738,
- 19081,15196,19093,19114,19126,19128,19131,19145,19151,19157,
- 19174, 7214, 7205, 7182, 7149, 7149, 7146, 7119, 7123,19186,
- 7115,19192, 7101,19203, 7100,19205,15206,19212, 7062,19223,
- 7044,19225,19238, 7044, 7044,19250,26921,19260, 7041,19272,
-
- 15698,10740,19275, 7020,15746, 7026, 7013, 7024, 6997, 6956,
- 26921,15921,19273,16732,19295,16895,19310,16897, 6955,19293,
- 15966,15911,19298,19320,19332,10749,10750,19341,15791,19343,
- 19358,19363,19378,19388,19397, 6959, 6956,26921, 6942,26921,
- 6948, 6907, 6885,19400, 6902,19409, 6852,19419, 6847,19431,
- 10817,19477,15969,19560,19561, 6829, 6834,19563,10818,19566,
- 15995,10820,19572,26921,15979, 6823, 6801, 6799, 6771, 6743,
- 16106,19609, 6252,19611, 6445,19614, 6447,14039,19621,16419,
- 16113,19620,19634,26921,10829,19640,19659,19669,19671,19681,
- 19682, 6725, 6715, 6691, 6670, 6638, 6620,19688, 6634,19700,
-
- 26921,19719,10830,19729,16182,19730,10897,26921,19731, 0,
- 16547,26921,26921,16115, 0, 0, 0,16183,10898,19741,
- 19742, 6570, 6568,19748,16193,10900,10909,26921,10910,10977,
- 6512,14079,19768,19787,19790,19793,26921,26921, 6515,26921,
- 6501, 6513,19802, 6353,19805,16244,10978,16256,16304,10980,
- 19820,19839, 6320, 6233,11458,11599,11814, 6205,19840,19850,
- 19851, 6172,26921, 6115,19881, 6054,19861,10989, 5794, 5717,
- 19888,19898,19903, 5678, 5654,19908, 5620,19915,10990, 5589,
- 5553,19918,19933,19945, 5494, 5006,19956, 4947,11057, 4915,
- 19966,19968, 4120,26921,19978, 4037,11058, 3978,26921,19987,
-
- 3948, 3928,20001, 3214, 3204,20013, 2832,26921,20016, 2699,
- 26921,26921,20062,20075,20088,20101,20114,20127,20140,20153,
- 20166,20179,20192,20205,20218,20231,20244,20254,20267,20280,
- 20290,20303,20316,20329,20342,20352,20365,20378,20388,20401,
- 20414,20427,20437,20450,20463,20473,20486,20499,20509,20522,
- 20535,20545,20558,20571,20581,20594,20607,20617,20630,20643,
- 20653,20666,20679,20689,20702,20715,20725,20738,20751,20761,
- 20774,20787,20797,20810,20823,20833,20846,20859,20872,20885,
- 20895,20908,20921,20931,20944,20957,20967,20980,20993,21003,
- 21016,21029,21042,21052,21065,21078,21088,21101,21114,21124,
-
- 21137,21150,21160,21173,21186,21199,21212, 2648,21222,21235,
- 21248,21261,21274,21287,21300,21310,21323,21336,21346,21359,
- 21372,21382,21395,21408,21418,21431,21444,21457,21470,21480,
- 21493,21506,21519,21529,21542,21555,21568,21578,21591,21604,
- 21614,21627,21640,21650,21663,21676,21689,21702,21715,21725,
- 21738,21751,21761,21774,21787,21797,21810,21823,21836,21849,
- 21862,21872,21885,21898,21908,21921,21934,21944,21957,21970,
- 21980,21993,22006,22019,22029,22042,22055,22068,22081,22094,
- 22107,22120,22133,22146,22156,22169,22182,22195,22205,22218,
- 22231,22244,22257,22267,22280,22293,22306,22319,22332,22345,
-
- 22355,22368,22381,22394,22404,22417,22430,22443,22453,22466,
- 22479,22492,22502,22515,22528,22541,22551,22564,22577,22590,
- 22600,22613,22626,22639,22649,22662,22675,22688,22698,22711,
- 22724,22737,22747,22760,22773,22786,22796,22809,22822,22835,
- 22845,22858,22871,22884,22897,22910,22920,22933,22946,22959,
- 22969,22982,22995,23008,23018,23031,23044,23057,23067,23080,
- 23093,23106,23119,23129,23142,23155,23168,23178,23191,23204,
- 23217,23227,23240,23253,23266,23276,23289,23302,23315,23328,
- 23341,23354,23367,23380,23393,23406, 2619,23419,23432,23445,
- 23458,23471,23484,23497,23510,23523,23536,23549,23562,23575,
-
- 23585,23598,23611,23624,23634,23647,23660,23673,23683,23696,
- 23709,23722,23732,23745,23758,23771,23784,23797,23807,23820,
- 23833,23846,23859,23869,23882,23895,23908,23921,23931,23944,
- 23957,23970,23980,23993,24006,24019,24029,24042,24055,24068,
- 24081,24094,24104,24117,24130,24143,24153,24166,24179,24192,
- 24202,24215,24228,24241,24254,24267,24277,24290,24303,24316,
- 24326,24339,24352,24365,24375,24388,24401,24414,24424,24437,
- 24450,24463,24473,24486,24499,24512,24525,24538,24551,24564,
- 24577,24590,24603,24616,24629,24642,24655,24668,24681,24694,
- 24707,24720,24733,24746,24759,24772,24785,24798,24811,24824,
-
- 24837,24850,24863,24876,24889,24902,24915,24928,24941,24954,
- 24967,24980,24993,25006,25019,25032,25045,25058,25071,25084,
- 25097,25110,25123,25136,25149,25162,25175,25188, 1819,25201,
- 25214,25227,25240,25253,25266,25279,25292,25305,25318,25331,
- 25344,25357,25370,25383,25396,25409,25422,25435,25448,25458,
- 25471,25484,25497,25510,25523,25536,25549,25562,25575,25588,
- 25601,25614,25627,25640,25653,25666,25679,25692,25705,25718,
- 25731,25744,25757,25770,25783,25796,25809,25822,25835,25848,
- 25861,25874,25887,25900,25913,25926,25939,25952,25965,25978,
- 25991,26004,26017,26030,26043,26053,26066,26079,26092,26105,
-
- 26118,26131,26144,26157,26170,26183,26196,26209,26222,26235,
- 26248,26261,26274,26287,26300,26313,26326,26339,26352,26365,
- 26378,26391,26404,26417,26430,26443,26456,26469,26482,26495,
- 26508,26521,26534,26547,26560,26573,26586,26599,26612,26625,
- 26638,26651,26663,26673,26686,26699,26712,26725,26738,26751,
- 26764,26777,26790,26803,26816,26829,26842,26855,26868,26881,
- 26894,26907
+ 14539,14546, 9801, 9784,14551,11410, 9918,14609,11412,14588,
+ 14611, 9756,14614,14621,11411,14631, 9920, 9921,14858, 9755,
+ 14860,14927,11493,14973,11580, 9923,15056,11596,15101,11805,
+ 11807, 9952, 9953,11499,15059,26921,15068,15071,11817,15081,
+
+ 15184, 9742,15090,15185,15186,15195,15197,15198,15232, 9710,
+ 9718,11942,11949, 9721,11948, 9692, 9699, 9685, 9688, 9689,
+ 9684,11950, 9682, 9959,15207,11812,15244,15246, 9645,15258,
+ 9967,15269, 9968,15249,12004,15259,15271,15292,10043, 9644,
+ 9658, 9636, 9998,10055, 9638, 9606,15304, 9595,15306,15318,
+ 9606,15317,10005,15329,15351, 9579, 9564,15352,12533,15363,
+ 9588,12530,15372,15377,15423,26921,26921,15506, 0, 0,
+ 0,12954,10086,15508, 9547,15509,15511,13000,15518,15521,
+ 15528, 9521,26921, 9519,15559,15566,15569,15571,15576, 9526,
+ 9489,15578,15588,15589,15619,15626,15631,15636,15637,15648,
+
+ 9470, 9449, 9464, 9437, 6932, 9441, 6041,10045,15678,12535,
+ 15689, 9420,15679,15690,15696, 9403,10046,15731,12544,15738,
+ 15701, 9416,15743,12541,13065,13067,15772,14072,15757,15818,
+ 12536,10048,10053,15769,10058, 9396,15901,26921,15903,15904,
+ 15906,15923,15918, 9387, 9383, 9372,15949,15956,12543,15961,
+ 9372,15968, 9355,15974, 9359,16020,16103, 9357,16104,16105,
+ 9336,16114,16116, 9317,16123,16125, 9302,16126,16171, 9303,
+ 16173,16174,16176,26921,16185, 9323,16188,16219,13196,16226,
+ 16233,16236, 9299, 9277,16238,16239, 9286,16284,16245,26921,
+ 16286, 9261,16291,13197,10085,16293,16296,16306, 9258, 9260,
+
+ 16305,12953,16339,16348,16351,26921,16353,16360,26921,16362,
+ 0,13194,26921,26921, 0, 0, 0,10112,10114, 0,
+ 0,13203, 0, 0, 0, 0,16365,10115,16374,13205,
+ 16399, 9246,16408,16411,16428,16417,16429,16438, 9211, 9214,
+ 13206,14166,13960,26921,14073, 9197, 9194, 9164, 9180,26921,
+ 9159,14255,14088, 9150,10143,14256,14656,16458,26921,16459,
+ 13953,16471,10148,10154,16477,10189,16489,14347,16496,16506,
+ 10335,10194, 9149,10415,14438,10256, 9142,10495,16518,26921,
+ 16527,16539, 9132,16548,16545, 9128, 9126,16557,14349,16560,
+ 10195,16582, 9124,14164, 9123,16591,10257,14536,16602,10197,
+
+ 10258,16603,26921,16605,16608, 9076,16617,16620, 9049, 9034,
+ 16650,16651,16663,16662,11110,11111,16669,16672,16681,16710,
+ 16715,16720,16727,16730,16740,16741, 9006, 9011, 9016, 8984,
+ 8990, 8990, 8972, 8982, 8967,10202,16775, 8964,16778,16787,
+ 16789,26921,16796,16790,16835, 8948,16837,16838, 6116,10203,
+ 16844,14559,14573,16858,14271, 8927,16847,16864,16889,16937,
+ 8899, 8899,10575,17020,17021, 8885,17022, 8899,17023, 8887,
+ 17032, 0,14869,26921,10655,26921, 0, 0, 0,17041,
+ 8857,17070, 8844,17071,17080, 8830,17082,26921,17092,17091,
+ 8829,17093,17105,17139,17142, 8835,17144,16476,17151,14619,
+
+ 17153, 8841, 8811,17156, 8802,17162,17173, 8800,17203, 8780,
+ 17204,17214, 8760, 8758,17215,17234,17221,14866,17260,10204,
+ 17263, 8739,17272,17273,17283,17282,17320, 8694, 8681,14439,
+ 10259,14871, 8669, 8673,26921, 8684,10337,14876,14877,14930,
+ 17285,14936,10269,17330,10338,17332,17337,17339,17349,17380,
+ 17386, 8664,10735,10815,14928,10300, 8666,10895,17391, 8655,
+ 17398,17401, 8650, 8651,17411,14874,10340,17432,17444,10349,
+ 14879,17446,10350,11135,17449, 8634,17458,17461,26921, 8598,
+ 17467,17487,17506,14939,17509,17512,14941,17521,10417,11138,
+ 10418,11060,17524,17543,17554,17566,17580,17585,17591,17592,
+
+ 17597,17611, 8596, 8595, 8571, 8578, 8590, 8562, 8525, 8531,
+ 8531,17614,26921,17634,17644,17645,17657,17656,26921,10420,
+ 17659,14942,15057, 8528,17671,17682, 8505,10429,10975,17702,
+ 26921,17704, 8514,17705, 8504,10430,11055,15062,17716, 8444,
+ 17717,26921,17719,26921,17739,17750,26921,17762,17764,17765,
+ 17787, 8441,17799,17810, 8424, 8417,17816, 8276, 8257,17817,
+ 26921,17822,17828,10497,17851, 8248,17837,17863, 8186, 8176,
+ 17876,17886,17885,26921,17888,17898,17911,17923, 8162, 8091,
+ 11229,15337,15069, 8094, 8065, 8057,11458,15364,15110,17936,
+ 11322,17937,17946,17948,17958,17949,17991, 8051,11228, 8078,
+
+ 17997,26921,18000,18006, 8055, 7876,18003,18017,18018,11311,
+ 18020,10498,18051,26921,18063,18065,18068,18070,10510,11320,
+ 18088,18085,18111,18122,18123,18133,18144,18153,18171, 7852,
+ 7851, 7844, 7844, 7847,26921, 7841, 7843, 7827,18156,15116,
+ 18174,18183,10500,18194,18204,15196,11597,10509,18209,18216,
+ 7826,18221, 7806,10577,18231, 7805,18243,18254,18264,18266,
+ 15119,18269, 7770,18291,18303, 7786, 7775, 6892, 7762,18314,
+ 15076,18316,15131,10578,18321, 7766,18326,18333, 7757,10580,
+ 18336,18359,18371,26921,26921,15371,10625, 7740, 7737,15596,
+ 18374,10589,18384,18386,18396,18393, 0,11989, 7722,18405,
+
+ 18431, 7735,15397,18419,18438,18444,11150,18450,15206,10590,
+ 18456,18457,18469,18479,11553,10657,18491,18504,18498,18510,
+ 18521,18516,18533,18547,18558, 7734, 7725, 7703, 7719, 7681,
+ 7685, 7683, 7674,18564, 7646,18559,18584,15398,18585, 7605,
+ 12947,18597,18607,26921,18610,26921,18619,26921,18630,18632,
+ 18644,18649, 7608,18661, 7593,18667,18678, 7600, 7586, 7592,
+ 7572,18679, 7585,18692, 7553,18690,18709,15519, 7537,10658,
+ 18712,15691,18729,15586, 7511, 7523, 7496, 7496, 7513, 7478,
+ 16115,18727,18738,18750,18739,13942, 7478,18760,18785,15639,
+ 7478,15646,18795,18797,18812,11457,18798,18815,10670,18832,
+
+ 18845,18850,18862,18875,18880,18897,18909, 7463, 7445, 7439,
+ 7424, 7427, 7412, 7423, 7403,18914, 7402,18926,15699,18943,
+ 15649,15746,18936, 7386,18948,18955,18957, 7377,18969, 7357,
+ 18986,19000, 7344, 7347,26921, 7322,19003, 7324,19006, 7301,
+ 19012,10737,19023, 7281,15913, 7263, 7274, 7261, 7225, 7216,
+ 7216,15986,19021,19033,19054,19060, 7197,19066,10738,19071,
+ 19080,19083,10750,19081,15791,19093,19114,19126,19128,19131,
+ 19145,19151,19157,19174, 7214, 7205, 7182, 7149, 7149, 7146,
+ 7119, 7123,19186, 7115,19192, 7101,19203, 7100,19205,15520,
+ 19212, 7062,19223, 7044,19225,19238, 7044, 7044,19250,26921,
+
+ 19260, 7041,19272,15698,10740,19275, 7020,15921, 7026, 7013,
+ 7024, 6997, 6956,26921,16113,19273,16732,19295,16895,19310,
+ 16897, 6955,19293,15777,15788,19298,19320,19332,10749,10817,
+ 19341,15912,19343,19358,19363,19378,19388,19397, 6959, 6956,
+ 26921, 6942,26921, 6948, 6907, 6885,19400, 6902,19409, 6852,
+ 19419, 6847,19431,10829,19477,15966,19560,19561, 6829, 6834,
+ 19563,10818,19566,15915,10820,19572,26921,15969, 6823, 6801,
+ 6799, 6771, 6743,15979,19609, 6252,19611, 6445,19614, 6447,
+ 14039,19621,15995,16106,19620,19634,26921,10830,19640,19659,
+ 19669,19671,19681,19682, 6725, 6715, 6691, 6670, 6638, 6620,
+
+ 19688, 6634,19700,26921,19719,10897,19729,16182,19730,10898,
+ 26921,19731, 0,16419,26921,26921,16183, 0, 0, 0,
+ 16256,10900,19741,19742, 6570, 6568,19748,16193,10909,10910,
+ 26921,10977,10978, 6512,14079,19768,19787,19790,19793,26921,
+ 26921, 6515,26921, 6501, 6513,19802, 6353,19805,16244,10980,
+ 16294,16304,10989,19820,19839, 6320, 6233,11814,13036,13066,
+ 6205,19840,19850,19851, 6172,26921, 6115,19881, 6054,19861,
+ 10990, 5794, 5717,19888,19898,19903, 5678, 5654,19908, 5620,
+ 19915,11057, 5589, 5553,19918,19933,19945, 5494, 5006,19956,
+ 4947,11058, 4915,19966,19968, 4120,26921,19978, 4037,11069,
+
+ 3978,26921,19987, 3948, 3928,20001, 3214, 3204,20013, 2832,
+ 26921,20016, 2699,26921,26921,20062,20075,20088,20101,20114,
+ 20127,20140,20153,20166,20179,20192,20205,20218,20231,20244,
+ 20254,20267,20280,20290,20303,20316,20329,20342,20352,20365,
+ 20378,20388,20401,20414,20427,20437,20450,20463,20473,20486,
+ 20499,20509,20522,20535,20545,20558,20571,20581,20594,20607,
+ 20617,20630,20643,20653,20666,20679,20689,20702,20715,20725,
+ 20738,20751,20761,20774,20787,20797,20810,20823,20833,20846,
+ 20859,20872,20885,20895,20908,20921,20931,20944,20957,20967,
+ 20980,20993,21003,21016,21029,21042,21052,21065,21078,21088,
+
+ 21101,21114,21124,21137,21150,21160,21173,21186,21199,21212,
+ 2648,21222,21235,21248,21261,21274,21287,21300,21310,21323,
+ 21336,21346,21359,21372,21382,21395,21408,21418,21431,21444,
+ 21457,21470,21480,21493,21506,21519,21529,21542,21555,21568,
+ 21578,21591,21604,21614,21627,21640,21650,21663,21676,21689,
+ 21702,21715,21725,21738,21751,21761,21774,21787,21797,21810,
+ 21823,21836,21849,21862,21872,21885,21898,21908,21921,21934,
+ 21944,21957,21970,21980,21993,22006,22019,22029,22042,22055,
+ 22068,22081,22094,22107,22120,22133,22146,22156,22169,22182,
+ 22195,22205,22218,22231,22244,22257,22267,22280,22293,22306,
+
+ 22319,22332,22345,22355,22368,22381,22394,22404,22417,22430,
+ 22443,22453,22466,22479,22492,22502,22515,22528,22541,22551,
+ 22564,22577,22590,22600,22613,22626,22639,22649,22662,22675,
+ 22688,22698,22711,22724,22737,22747,22760,22773,22786,22796,
+ 22809,22822,22835,22845,22858,22871,22884,22897,22910,22920,
+ 22933,22946,22959,22969,22982,22995,23008,23018,23031,23044,
+ 23057,23067,23080,23093,23106,23119,23129,23142,23155,23168,
+ 23178,23191,23204,23217,23227,23240,23253,23266,23276,23289,
+ 23302,23315,23328,23341,23354,23367,23380,23393,23406, 2619,
+ 23419,23432,23445,23458,23471,23484,23497,23510,23523,23536,
+
+ 23549,23562,23575,23585,23598,23611,23624,23634,23647,23660,
+ 23673,23683,23696,23709,23722,23732,23745,23758,23771,23784,
+ 23797,23807,23820,23833,23846,23859,23869,23882,23895,23908,
+ 23921,23931,23944,23957,23970,23980,23993,24006,24019,24029,
+ 24042,24055,24068,24081,24094,24104,24117,24130,24143,24153,
+ 24166,24179,24192,24202,24215,24228,24241,24254,24267,24277,
+ 24290,24303,24316,24326,24339,24352,24365,24375,24388,24401,
+ 24414,24424,24437,24450,24463,24473,24486,24499,24512,24525,
+ 24538,24551,24564,24577,24590,24603,24616,24629,24642,24655,
+ 24668,24681,24694,24707,24720,24733,24746,24759,24772,24785,
+
+ 24798,24811,24824,24837,24850,24863,24876,24889,24902,24915,
+ 24928,24941,24954,24967,24980,24993,25006,25019,25032,25045,
+ 25058,25071,25084,25097,25110,25123,25136,25149,25162,25175,
+ 25188, 1819,25201,25214,25227,25240,25253,25266,25279,25292,
+ 25305,25318,25331,25344,25357,25370,25383,25396,25409,25422,
+ 25435,25448,25458,25471,25484,25497,25510,25523,25536,25549,
+ 25562,25575,25588,25601,25614,25627,25640,25653,25666,25679,
+ 25692,25705,25718,25731,25744,25757,25770,25783,25796,25809,
+ 25822,25835,25848,25861,25874,25887,25900,25913,25926,25939,
+ 25952,25965,25978,25991,26004,26017,26030,26043,26053,26066,
+
+ 26079,26092,26105,26118,26131,26144,26157,26170,26183,26196,
+ 26209,26222,26235,26248,26261,26274,26287,26300,26313,26326,
+ 26339,26352,26365,26378,26391,26404,26417,26430,26443,26456,
+ 26469,26482,26495,26508,26521,26534,26547,26560,26573,26586,
+ 26599,26612,26625,26638,26651,26663,26673,26686,26699,26712,
+ 26725,26738,26751,26764,26777,26790,26803,26816,26829,26842,
+ 26855,26868,26881,26894,26907
} ;
-static yyconst flex_int16_t yy_def[3463] =
+static yyconst flex_int16_t yy_def[3466] =
{ 0,
- 2913, 2912, 2912, 3, 2912, 5, 2914, 2914, 2912, 9,
- 2912, 11, 2912, 13, 2912, 15, 2912, 17, 2912, 19,
- 2912, 21, 2912, 23, 2912, 25, 2912, 27, 2912, 29,
- 2912, 31, 2912, 33, 2912, 35, 2915, 2915, 2916, 2916,
- 2912, 41, 2912, 43, 2912, 45, 2912, 47, 2917, 2917,
- 2918, 2918, 2919, 2919, 2912, 55, 2912, 57, 2912, 59,
- 2912, 61, 2920, 2920, 2921, 2921, 2920, 2920, 2912, 69,
- 2912, 71, 2912, 73, 2912, 75, 2912, 77, 2922, 2922,
- 2923, 2923, 2912, 83, 2920, 2920, 2924, 2924, 2920, 2920,
- 2912, 91, 2925, 2925, 2920, 2920, 2912, 97, 2912, 99,
-
- 2912, 101, 2912, 103, 2920, 2920, 2912, 107, 2912, 109,
- 2912, 111, 2912, 113, 2912, 115, 2912, 117, 2912, 119,
- 2912, 121, 2912, 2912, 2912, 2912, 2912, 2912, 2926, 2927,
- 2912, 2927, 132, 131, 132, 132, 132, 132, 132, 132,
+ 2916, 2915, 2915, 3, 2915, 5, 2917, 2917, 2915, 9,
+ 2915, 11, 2915, 13, 2915, 15, 2915, 17, 2915, 19,
+ 2915, 21, 2915, 23, 2915, 25, 2915, 27, 2915, 29,
+ 2915, 31, 2915, 33, 2915, 35, 2918, 2918, 2919, 2919,
+ 2915, 41, 2915, 43, 2915, 45, 2915, 47, 2920, 2920,
+ 2921, 2921, 2922, 2922, 2915, 55, 2915, 57, 2915, 59,
+ 2915, 61, 2923, 2923, 2924, 2924, 2923, 2923, 2915, 69,
+ 2915, 71, 2915, 73, 2915, 75, 2915, 77, 2925, 2925,
+ 2926, 2926, 2915, 83, 2923, 2923, 2927, 2927, 2923, 2923,
+ 2915, 91, 2928, 2928, 2923, 2923, 2915, 97, 2915, 99,
+
+ 2915, 101, 2915, 103, 2923, 2923, 2915, 107, 2915, 109,
+ 2915, 111, 2915, 113, 2915, 115, 2915, 117, 2915, 119,
+ 2915, 121, 2915, 2915, 2915, 2915, 2915, 2915, 2929, 2930,
+ 2915, 2930, 132, 131, 132, 132, 132, 132, 132, 132,
132, 132, 132, 132, 132, 132, 132, 132, 132, 132,
- 132, 132, 132, 132, 132, 2927, 2928, 2929, 2912, 2912,
- 2912, 2912, 2912, 2912, 2930, 2912, 2912, 2912, 2912, 2931,
- 2932, 2933, 2912, 2934, 173, 2912, 2934, 2934, 178, 178,
- 2912, 2912, 2935, 2935, 2935, 2936, 2937, 2938, 2938, 2938,
- 2939, 2940, 2912, 2941, 2941, 193, 2942, 2941, 193, 2943,
-
- 2944, 2945, 2945, 2945, 2946, 2947, 2948, 2948, 2948, 2949,
- 2950, 2951, 2951, 2951, 2952, 2953, 2954, 2954, 2954, 2955,
- 2956, 2957, 2957, 2957, 2958, 2959, 2960, 2960, 2960, 2961,
- 2962, 2963, 2963, 2963, 2964, 2965, 2966, 2966, 2966, 2967,
- 2968, 2969, 2969, 2969, 2970, 2971, 2972, 2972, 2972, 2973,
- 2974, 2975, 2975, 2975, 2976, 2977, 2978, 2978, 2978, 2979,
- 2979, 2980, 2980, 2980, 2981, 2982, 2983, 2983, 2983, 2984,
- 2985, 2986, 2986, 2986, 2987, 2988, 2989, 2989, 2989, 2990,
- 2991, 2912, 2912, 2912, 2992, 2992, 2992, 2992, 2912, 2912,
- 2912, 2912, 2912, 2912, 2993, 2993, 2993, 2994, 2995, 2996,
-
- 2996, 2996, 2997, 2998, 2999, 2999, 2999, 3000, 3001, 3002,
- 3002, 3002, 3003, 3004, 2912, 3005, 3005, 3005, 2912, 2912,
- 2912, 2912, 2912, 3006, 3007, 3008, 2912, 2912, 3009, 3010,
- 3010, 3010, 3010, 3010, 3010, 3011, 3012, 3013, 3014, 3015,
- 3015, 3015, 3016, 3017, 3018, 3018, 3018, 3019, 3020, 3021,
- 3021, 3021, 3022, 3023, 3024, 3024, 3024, 3025, 3026, 3027,
- 3027, 3027, 3028, 2912, 3028, 3028, 2912, 3029, 3029, 3029,
- 3030, 3031, 3032, 3032, 3032, 2912, 3033, 3033, 3033, 3034,
- 3035, 3036, 3036, 3036, 2912, 3037, 3037, 3037, 3038, 3039,
- 3040, 3040, 3040, 3041, 3042, 3043, 3043, 3043, 3044, 3045,
-
- 2912, 2912, 2912, 2912, 3046, 3047, 3048, 2912, 3049, 3049,
- 3049, 3050, 3051, 3052, 3052, 3052, 3053, 3054, 3055, 3055,
- 3055, 3056, 3057, 2912, 2912, 3058, 3059, 3060, 3061, 3061,
- 3061, 3062, 3063, 3064, 3064, 3064, 3065, 3066, 3067, 3067,
- 3067, 3068, 3069, 3070, 3070, 3070, 3071, 3072, 3073, 3074,
- 3074, 2912, 2912, 3074, 3075, 3076, 451, 457, 451, 451,
+ 132, 132, 132, 132, 132, 2930, 2931, 2932, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2933, 2915, 2915, 2915, 2915, 2934,
+ 2935, 2936, 2915, 2937, 173, 2915, 2937, 2937, 178, 178,
+ 2915, 2915, 2938, 2938, 2938, 2939, 2940, 2941, 2941, 2941,
+ 2942, 2943, 2915, 2944, 2944, 193, 2945, 2944, 193, 2946,
+
+ 2947, 2948, 2948, 2948, 2949, 2950, 2951, 2951, 2951, 2952,
+ 2953, 2954, 2954, 2954, 2955, 2956, 2957, 2957, 2957, 2958,
+ 2959, 2960, 2960, 2960, 2961, 2962, 2963, 2963, 2963, 2964,
+ 2965, 2966, 2966, 2966, 2967, 2968, 2969, 2969, 2969, 2970,
+ 2971, 2972, 2972, 2972, 2973, 2974, 2975, 2975, 2975, 2976,
+ 2977, 2978, 2978, 2978, 2979, 2980, 2981, 2981, 2981, 2982,
+ 2982, 2983, 2983, 2983, 2984, 2985, 2986, 2986, 2986, 2987,
+ 2988, 2989, 2989, 2989, 2990, 2991, 2992, 2992, 2992, 2993,
+ 2994, 2915, 2915, 2915, 2995, 2995, 2995, 2995, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2996, 2996, 2996, 2997, 2998, 2999,
+
+ 2999, 2999, 3000, 3001, 3002, 3002, 3002, 3003, 3004, 3005,
+ 3005, 3005, 3006, 3007, 2915, 3008, 3008, 3008, 2915, 2915,
+ 2915, 2915, 2915, 3009, 3010, 3011, 2915, 2915, 3012, 3013,
+ 3013, 3013, 3013, 3013, 3013, 3014, 3015, 3016, 3017, 3018,
+ 3018, 3018, 3019, 3020, 3021, 3021, 3021, 3022, 3023, 3024,
+ 3024, 3024, 3025, 3026, 3027, 3027, 3027, 3028, 3029, 3030,
+ 3030, 3030, 3031, 2915, 3031, 3031, 2915, 3032, 3032, 3032,
+ 3033, 3034, 3035, 3035, 3035, 2915, 3036, 3036, 3036, 3037,
+ 3038, 3039, 3039, 3039, 2915, 3040, 3040, 3040, 3041, 3042,
+ 3043, 3043, 3043, 3044, 3045, 3046, 3046, 3046, 3047, 3048,
+
+ 2915, 2915, 2915, 2915, 3049, 3050, 3051, 2915, 3052, 3052,
+ 3052, 3053, 3054, 3055, 3055, 3055, 3056, 3057, 3058, 3058,
+ 3058, 3059, 3060, 2915, 2915, 3061, 3062, 3063, 3064, 3064,
+ 3064, 3065, 3066, 3067, 3067, 3067, 3068, 3069, 3070, 3070,
+ 3070, 3071, 3072, 3073, 3073, 3073, 3074, 3075, 3076, 3077,
+ 3077, 2915, 2915, 3077, 3078, 3079, 451, 457, 451, 451,
451, 451, 451, 451, 451, 451, 451, 451, 451, 451,
451, 451, 451, 451, 451, 451, 451, 451, 451, 457,
457, 451, 451, 451, 451, 451, 451, 451, 451, 451,
@@ -1415,333 +1415,333 @@ static yyconst flex_int16_t yy_def[3463] =
451, 451, 451, 451, 451, 451, 451, 451, 451, 451,
451, 451, 451, 451, 451, 451, 451, 451, 451, 451,
451, 451, 451, 451, 451, 451, 451, 451, 451, 451,
- 451, 451, 2912, 451, 451, 451, 451, 451, 451, 454,
- 3077, 2912, 3078, 3079, 3080, 2912, 2912, 3081, 3082, 2912,
- 2912, 3083, 2912, 3083, 3083, 565, 565, 565, 565, 3084,
- 3084, 3085, 3086, 571, 3087, 3088, 3088, 3089, 3090, 577,
- 3091, 2912, 3092, 3093, 582, 3094, 3095, 3093, 3093, 3096,
- 3097, 585, 589, 3098, 3099, 3100, 3100, 3101, 3102, 597,
-
- 3103, 3104, 3104, 3105, 3106, 603, 3107, 3108, 3108, 3109,
- 3110, 609, 3111, 3112, 3112, 3113, 3114, 615, 3115, 3116,
- 3116, 3117, 3118, 621, 3119, 3120, 3120, 3121, 3122, 627,
- 3123, 3124, 3124, 3125, 3126, 633, 3127, 3128, 3128, 3129,
- 3130, 639, 3131, 3132, 3132, 3133, 3134, 645, 3135, 3136,
- 3136, 3137, 3138, 651, 3139, 3140, 3140, 3141, 3142, 657,
- 3143, 3144, 3145, 3146, 3146, 3147, 3148, 665, 3149, 3150,
- 3150, 3151, 3152, 671, 3153, 3154, 3154, 3155, 3156, 677,
- 3157, 3158, 3158, 3159, 3160, 683, 3161, 3162, 3163, 3163,
- 3164, 3165, 690, 3166, 3167, 3167, 3168, 3169, 696, 3170,
-
- 3171, 3171, 3172, 3173, 702, 3174, 3175, 3175, 3176, 3177,
- 708, 3178, 3179, 2912, 2912, 2912, 2912, 3180, 3181, 3182,
- 3183, 2912, 3184, 3185, 3186, 3187, 3188, 3188, 3188, 3188,
- 3188, 3188, 3189, 3190, 3191, 3192, 2912, 3193, 3194, 3195,
- 3196, 3188, 3188, 3197, 3188, 3198, 3188, 3188, 3188, 3188,
- 3188, 3199, 3200, 3200, 3201, 3202, 754, 3203, 3204, 3204,
- 3205, 3206, 760, 3207, 3208, 3208, 3209, 3210, 766, 3211,
- 3212, 3212, 3213, 3214, 772, 3215, 3216, 3217, 3218, 3218,
- 3219, 3220, 780, 3221, 3222, 2912, 3223, 3223, 3224, 3225,
- 788, 3226, 3227, 2912, 3228, 3228, 3229, 3230, 796, 3231,
-
- 3232, 3232, 3233, 3234, 802, 3235, 3236, 3236, 3237, 3238,
- 808, 3239, 2912, 3240, 3241, 2912, 3242, 3242, 3243, 3244,
- 818, 3245, 3246, 3246, 3247, 3248, 824, 3249, 3250, 3250,
- 3251, 3252, 830, 3253, 2912, 3254, 3255, 3256, 3256, 3257,
- 3258, 839, 3259, 3260, 3260, 3261, 3262, 845, 3263, 3264,
- 3264, 3265, 3266, 851, 3267, 3268, 3268, 3269, 3270, 857,
- 3271, 3272, 3272, 863, 863, 863, 863, 863, 863, 2912,
- 863, 863, 863, 863, 863, 863, 863, 863, 2912, 863,
+ 451, 451, 2915, 451, 451, 451, 451, 451, 451, 454,
+ 3080, 2915, 3081, 3082, 3083, 2915, 2915, 3084, 3085, 2915,
+ 2915, 3086, 2915, 3086, 3086, 565, 565, 565, 565, 3087,
+ 3087, 3088, 3089, 571, 3090, 3091, 3091, 3092, 3093, 577,
+ 3094, 2915, 3095, 3096, 582, 3097, 3098, 3096, 3096, 3099,
+ 3100, 585, 589, 3101, 3102, 3103, 3103, 3104, 3105, 597,
+
+ 3106, 3107, 3107, 3108, 3109, 603, 3110, 3111, 3111, 3112,
+ 3113, 609, 3114, 3115, 3115, 3116, 3117, 615, 3118, 3119,
+ 3119, 3120, 3121, 621, 3122, 3123, 3123, 3124, 3125, 627,
+ 3126, 3127, 3127, 3128, 3129, 633, 3130, 3131, 3131, 3132,
+ 3133, 639, 3134, 3135, 3135, 3136, 3137, 645, 3138, 3139,
+ 3139, 3140, 3141, 651, 3142, 3143, 3143, 3144, 3145, 657,
+ 3146, 3147, 3148, 3149, 3149, 3150, 3151, 665, 3152, 3153,
+ 3153, 3154, 3155, 671, 3156, 3157, 3157, 3158, 3159, 677,
+ 3160, 3161, 3161, 3162, 3163, 683, 3164, 3165, 3166, 3166,
+ 3167, 3168, 690, 3169, 3170, 3170, 3171, 3172, 696, 3173,
+
+ 3174, 3174, 3175, 3176, 702, 3177, 3178, 3178, 3179, 3180,
+ 708, 3181, 3182, 2915, 2915, 2915, 2915, 3183, 3184, 3185,
+ 3186, 2915, 3187, 3188, 3189, 3190, 3191, 3191, 3191, 3191,
+ 3191, 3191, 3192, 3193, 3194, 3195, 2915, 3196, 3197, 3198,
+ 3199, 3191, 3191, 3200, 3191, 3201, 3191, 3191, 3191, 3191,
+ 3191, 3202, 3203, 3203, 3204, 3205, 754, 3206, 3207, 3207,
+ 3208, 3209, 760, 3210, 3211, 3211, 3212, 3213, 766, 3214,
+ 3215, 3215, 3216, 3217, 772, 3218, 3219, 3220, 3221, 3221,
+ 3222, 3223, 780, 3224, 3225, 2915, 3226, 3226, 3227, 3228,
+ 788, 3229, 3230, 2915, 3231, 3231, 3232, 3233, 796, 3234,
+
+ 3235, 3235, 3236, 3237, 802, 3238, 3239, 3239, 3240, 3241,
+ 808, 3242, 2915, 3243, 3244, 2915, 3245, 3245, 3246, 3247,
+ 818, 3248, 3249, 3249, 3250, 3251, 824, 3252, 3253, 3253,
+ 3254, 3255, 830, 3256, 2915, 3257, 3258, 3259, 3259, 3260,
+ 3261, 839, 3262, 3263, 3263, 3264, 3265, 845, 3266, 3267,
+ 3267, 3268, 3269, 851, 3270, 3271, 3271, 3272, 3273, 857,
+ 3274, 3275, 3275, 863, 863, 863, 863, 863, 863, 2915,
+ 863, 863, 863, 863, 863, 863, 863, 863, 2915, 863,
863, 863, 863, 863, 863, 863, 863, 863, 863, 863,
863, 863, 863, 863, 863, 863, 863, 863, 863, 863,
- 863, 863, 863, 863, 2912, 863, 863, 2912, 863, 863,
- 863, 863, 2912, 863, 863, 863, 863, 2912, 863, 863,
- 863, 863, 863, 863, 2912, 863, 863, 863, 863, 863,
- 2912, 863, 863, 863, 863, 863, 863, 863, 863, 863,
- 863, 863, 863, 863, 2912, 863, 863, 863, 863, 2912,
- 863, 863, 863, 2912, 863, 863, 863, 863, 863, 863,
- 863, 863, 863, 863, 863, 2912, 863, 863, 863, 863,
- 863, 863, 863, 2912, 3273, 2912, 863, 2912, 863, 3272,
- 3274, 3275, 863, 863, 863, 863, 863, 863, 2912, 863,
- 863, 2912, 863, 980, 3276, 3277, 3278, 2912, 3279, 3280,
-
- 3281, 3282, 3283, 3284, 3285, 3286, 3287, 3288, 3289, 3290,
- 3291, 3292, 3293, 3294, 3295, 3296, 3297, 3298, 3299, 3300,
- 3301, 3302, 2912, 3303, 3303, 3304, 3305, 3306, 3304, 3303,
- 3307, 3308, 3309, 3310, 3311, 3312, 3313, 3314, 3315, 3316,
- 2912, 2912, 3317, 3317, 3318, 3319, 3320, 3318, 3317, 3321,
- 3322, 3323, 3324, 3325, 3326, 3327, 3328, 3329, 2912, 3330,
- 3331, 3332, 3333, 3334, 3335, 2912, 3336, 2912, 3337, 3338,
- 3339, 3340, 3341, 3342, 3343, 3344, 3345, 3346, 3347, 3348,
- 3349, 3349, 1082, 2912, 3350, 1082, 1082, 2912, 3351, 1082,
- 1082, 2912, 1082, 1082, 1082, 1082, 1082, 1082, 1082, 1082,
-
- 2912, 1082, 2912, 3352, 2912, 2912, 1082, 1082, 1082, 1082,
- 1082, 1082, 1082, 2912, 1082, 1082, 3349, 1082, 1082, 1082,
- 2912, 1082, 1082, 1082, 1082, 1082, 1082, 1082, 2912, 1082,
- 1082, 1082, 2912, 3353, 2912, 2912, 1117, 3354, 3355, 1082,
- 2912, 1082, 1082, 2912, 1082, 1082, 2912, 1082, 2912, 3356,
- 2912, 1082, 2912, 1082, 1082, 1082, 2912, 1082, 1082, 1082,
- 1082, 1082, 1082, 1082, 1082, 1082, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 1082, 1082, 1082, 2912, 1082,
- 1082, 2912, 1082, 1082, 1082, 1082, 1082, 1082, 1082, 1082,
- 1082, 1082, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 1082,
-
- 2912, 1082, 1082, 2912, 1082, 1082, 2912, 1082, 1082, 2912,
- 1082, 1082, 2912, 1082, 1082, 1082, 2912, 1082, 3357, 1082,
- 1082, 1082, 2912, 1082, 1082, 1082, 1082, 1082, 1082, 2912,
- 2912, 2912, 1082, 1082, 2912, 1082, 1082, 1082, 2912, 1082,
- 2912, 1082, 1082, 2912, 1082, 1082, 1082, 1082, 1082, 1082,
- 2912, 1082, 1082, 1082, 1082, 2912, 1082, 1082, 2912, 1082,
- 2912, 2912, 1082, 2912, 1082, 1082, 1082, 1082, 1082, 1082,
- 2912, 3358, 1082, 1082, 2912, 2912, 1082, 1082, 2912, 1082,
- 3359, 3360, 3360, 3361, 3362, 3363, 3361, 3361, 3364, 3365,
- 3366, 3367, 3367, 3368, 3369, 3370, 3361, 3360, 3364, 3365,
-
- 3366, 3360, 3361, 3361, 3364, 3365, 3366, 3364, 3361, 3371,
- 3372, 3373, 3368, 3367, 3374, 3375, 3376, 3377, 3378, 3378,
- 3379, 3380, 3381, 3379, 3379, 3382, 3383, 3384, 3385, 3385,
- 3386, 3387, 3388, 3379, 3378, 3382, 3383, 3384, 3378, 3379,
- 3379, 3382, 3383, 3384, 3382, 3379, 3389, 3390, 3391, 3386,
- 3385, 3392, 3393, 3394, 2912, 2912, 2912, 3395, 3395, 1359,
- 2912, 2912, 2912, 3395, 3396, 3397, 3398, 1359, 1359, 1359,
- 2912, 3399, 2912, 1359, 2912, 1371, 1364, 3396, 3397, 3399,
- 2912, 3399, 1359, 2912, 1359, 2912, 1359, 1359, 1359, 1359,
- 1359, 1359, 1359, 2912, 1359, 1359, 1359, 1359, 2912, 1359,
-
- 1359, 2912, 2912, 2912, 1364, 3396, 3397, 3400, 3401, 1359,
- 2912, 3401, 2912, 2912, 1364, 3396, 3397, 1359, 2912, 3402,
- 2912, 1359, 2912, 1364, 3396, 3397, 1359, 2912, 1359, 1359,
- 1359, 1359, 1359, 1359, 2912, 3403, 1359, 1359, 1359, 1359,
- 2912, 1359, 1359, 1359, 1359, 2912, 1359, 1359, 1359, 2912,
- 1359, 1359, 2912, 2912, 1359, 2912, 1359, 1359, 2912, 1359,
- 1359, 2912, 1359, 2912, 1359, 1359, 2912, 2912, 1364, 3396,
- 3397, 3404, 1359, 2912, 3404, 2912, 2912, 1364, 3396, 3397,
- 1359, 2912, 1359, 1359, 1359, 2912, 1359, 1359, 1359, 1359,
- 1359, 1359, 1359, 1359, 1359, 2912, 2912, 2912, 2912, 2912,
-
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 1359, 1359,
- 2912, 1359, 1359, 1359, 2912, 1359, 1359, 2912, 1359, 1359,
- 2912, 1359, 1359, 1359, 1359, 1359, 1359, 1359, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 1359, 1359, 2912, 1359, 1359,
- 2912, 1359, 2912, 1359, 2912, 1359, 2912, 1359, 1359, 3405,
- 3405, 3405, 1359, 2912, 3406, 1359, 2912, 1359, 1359, 1359,
- 1359, 1359, 2912, 2912, 2912, 1359, 1359, 1359, 2912, 2912,
- 1359, 1359, 1359, 2912, 2912, 1359, 1359, 1359, 2912, 1359,
- 2912, 1359, 1359, 2912, 1359, 1359, 1359, 1359, 2912, 1359,
- 2912, 1359, 2912, 2912, 1359, 2912, 1359, 1359, 2912, 1359,
-
- 1371, 3399, 1359, 1359, 2912, 3407, 2912, 2912, 1605, 1364,
- 3396, 3397, 3407, 3407, 1359, 1359, 2912, 2912, 1359, 1359,
- 2912, 1359, 3408, 3408, 3409, 3410, 3411, 3409, 3409, 3412,
- 3413, 3414, 3415, 3415, 3416, 3417, 3418, 3409, 3409, 3412,
- 3413, 3414, 3412, 3412, 3419, 3420, 3421, 3416, 3416, 3422,
- 3423, 3424, 3425, 3425, 3426, 3427, 3428, 3426, 3426, 3429,
- 3430, 3431, 3432, 3432, 3433, 3434, 3435, 3426, 3426, 3429,
- 3430, 3431, 3429, 3429, 3436, 3437, 3438, 3433, 3433, 3439,
- 3440, 3441, 3442, 3443, 3443, 2912, 1685, 1685, 2912, 3444,
- 2912, 3444, 3444, 1685, 2912, 1685, 2912, 1685, 2912, 1685,
-
- 1685, 2912, 1685, 1685, 2912, 1685, 1685, 2912, 1685, 1685,
- 2912, 1685, 1685, 2912, 1685, 1685, 1685, 2912, 1685, 3445,
- 3445, 3445, 2912, 2912, 1685, 1685, 2912, 1685, 1685, 1685,
- 1685, 1685, 2912, 2912, 2912, 1685, 1685, 3443, 3446, 3447,
- 3448, 2912, 2912, 1685, 1685, 2912, 1685, 1685, 1685, 2912,
- 1685, 1685, 2912, 1685, 2912, 1685, 2912, 1685, 1685, 2912,
- 1685, 1685, 2912, 2912, 1685, 2912, 2912, 1685, 1685, 2912,
- 1685, 1685, 2912, 1685, 2912, 2912, 1685, 2912, 1685, 1685,
- 2912, 2912, 2912, 3449, 1685, 1738, 1782, 3450, 3451, 3452,
- 3452, 3452, 1685, 2912, 1685, 1685, 2912, 1685, 1685, 2912,
-
- 1685, 1685, 1685, 1685, 1685, 1685, 1685, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 1685, 2912, 1685, 1685, 2912, 1685, 2912, 1685,
- 2912, 1685, 2912, 1685, 1685, 1685, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 1685, 2912, 1685, 1685, 2912, 1685,
- 2912, 1685, 1685, 2912, 2912, 1685, 2912, 1685, 3453, 3453,
- 1685, 1685, 2912, 2912, 2912, 1685, 1738, 3446, 3447, 3454,
- 2912, 1685, 2912, 1685, 1685, 2912, 1685, 1685, 1685, 2912,
- 2912, 2912, 1685, 1685, 1685, 1685, 1685, 2912, 2912, 1685,
- 1685, 1685, 1685, 1685, 1685, 1685, 1685, 1685, 2912, 2912,
-
- 2912, 2912, 2912, 2912, 2912, 2912, 1685, 2912, 1685, 2912,
- 1685, 1685, 1685, 2912, 2912, 1685, 2912, 1685, 1685, 2912,
- 1685, 1689, 3444, 3444, 1685, 2912, 1685, 2912, 3455, 3455,
- 3455, 1685, 2912, 2912, 1685, 2912, 1685, 1685, 1685, 1685,
- 1685, 2912, 2912, 2912, 1685, 1685, 3444, 1685, 2912, 1685,
- 2912, 1685, 2912, 2912, 1685, 2912, 1685, 1685, 2912, 1685,
- 1685, 2912, 1685, 1685, 2912, 1685, 1685, 2912, 1685, 1685,
- 1685, 2912, 1685, 2912, 1685, 1685, 2912, 1685, 1685, 1685,
- 2912, 2912, 1685, 1685, 2912, 1685, 1685, 2912, 1685, 2912,
- 1685, 2912, 2912, 1685, 1685, 1685, 2912, 2912, 1685, 2912,
-
- 1685, 1685, 1685, 2912, 1685, 1685, 2912, 1685, 1782, 3449,
- 2912, 2912, 1738, 3446, 3447, 3449, 3449, 1782, 1782, 1738,
- 1787, 1738, 1787, 1787, 1685, 2912, 1685, 2912, 1685, 2912,
- 1685, 1685, 1685, 1685, 1685, 1685, 2912, 2912, 3456, 2912,
- 2912, 2912, 3457, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 1928, 3455, 1685, 2912, 1685, 2912, 1685,
- 2912, 2912, 1685, 2912, 1685, 2912, 1685, 1685, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 1685, 2912, 1685, 1685,
- 2912, 1685, 1685, 2912, 2912, 1685, 2912, 1685, 2912, 1685,
- 3453, 3453, 3453, 1685, 1863, 3454, 1685, 3454, 2912, 1685,
-
- 2912, 1685, 1685, 2912, 1685, 1685, 2912, 2912, 1685, 1685,
- 1685, 1685, 2912, 2912, 1685, 1685, 1685, 1685, 1685, 1685,
- 1685, 1685, 1685, 1685, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 1685, 2912, 1685, 1685, 1685, 2912,
- 1685, 1685, 1685, 2912, 1685, 1685, 2912, 2912, 1685, 2912,
- 3455, 1685, 3444, 2912, 1685, 1685, 1685, 2912, 2912, 2912,
- 3458, 1685, 1685, 2912, 1685, 2912, 1685, 2912, 1685, 1954,
- 3459, 2912, 1954, 2912, 1738, 3446, 3447, 1685, 2912, 1685,
- 2912, 1685, 1685, 2912, 1685, 2912, 1685, 1685, 2912, 1685,
- 1685, 1685, 1685, 2912, 1685, 2912, 1685, 2912, 1685, 2912,
-
- 2912, 1685, 2912, 1685, 1685, 2912, 1685, 2912, 1685, 1685,
- 2912, 2912, 1685, 1685, 1685, 3449, 1685, 2912, 1685, 2912,
- 1685, 1685, 1685, 1685, 1685, 2912, 2912, 3456, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 1928, 3455, 1685, 3455,
- 2912, 1685, 2912, 1685, 1685, 1685, 1685, 1685, 1685, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 1685, 2912, 1685, 1685,
- 2912, 2912, 1685, 2912, 2912, 1685, 1685, 1863, 3454, 1685,
- 3454, 2912, 1685, 2912, 1685, 1685, 2912, 2912, 1685, 1685,
- 1685, 2912, 1685, 1685, 2912, 1685, 2912, 2912, 2912, 2912,
- 1685, 1685, 1685, 1685, 1685, 1685, 1685, 1685, 1685, 1685,
+ 863, 863, 863, 863, 2915, 863, 863, 2915, 863, 863,
+ 863, 863, 2915, 863, 863, 863, 863, 2915, 863, 863,
+ 863, 863, 863, 863, 2915, 863, 863, 863, 863, 863,
+ 2915, 863, 863, 863, 863, 863, 863, 863, 863, 863,
+ 863, 863, 863, 863, 2915, 863, 863, 863, 863, 2915,
+ 863, 863, 863, 2915, 863, 863, 863, 863, 863, 863,
+ 863, 863, 863, 863, 863, 2915, 863, 863, 863, 863,
+ 863, 863, 863, 2915, 3276, 2915, 863, 2915, 863, 3275,
+ 3277, 3278, 863, 863, 863, 863, 863, 863, 2915, 863,
+ 863, 2915, 863, 980, 3279, 3280, 3281, 2915, 3282, 3283,
+
+ 3284, 3285, 3286, 3287, 3288, 3289, 3290, 3291, 3292, 3293,
+ 3294, 3295, 3296, 3297, 3298, 3299, 3300, 3301, 3302, 3303,
+ 3304, 3305, 2915, 3306, 3306, 3307, 3308, 3309, 3307, 3306,
+ 3310, 3311, 3312, 3313, 3314, 3315, 3316, 3317, 3318, 3319,
+ 2915, 2915, 3320, 3320, 3321, 3322, 3323, 3321, 3320, 3324,
+ 3325, 3326, 3327, 3328, 3329, 3330, 3331, 3332, 2915, 3333,
+ 3334, 3335, 3336, 3337, 3338, 2915, 3339, 2915, 3340, 3341,
+ 3342, 3343, 3344, 3345, 3346, 3347, 3348, 3349, 3350, 3351,
+ 3352, 3352, 1082, 2915, 3353, 1082, 1082, 2915, 3354, 1082,
+ 1082, 2915, 1082, 1082, 1082, 1082, 1082, 1082, 1082, 1082,
+
+ 2915, 1082, 2915, 3355, 2915, 2915, 1082, 1082, 1082, 1082,
+ 1082, 1082, 1082, 2915, 1082, 1082, 3352, 1082, 1082, 1082,
+ 2915, 1082, 1082, 1082, 1082, 1082, 1082, 1082, 2915, 1082,
+ 1082, 1082, 2915, 3356, 2915, 2915, 1117, 3357, 3358, 1082,
+ 2915, 1082, 1082, 2915, 1082, 1082, 2915, 1082, 2915, 3359,
+ 2915, 1082, 2915, 1082, 1082, 1082, 2915, 1082, 1082, 1082,
+ 1082, 1082, 1082, 1082, 1082, 1082, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 1082, 1082, 1082, 2915, 1082,
+ 1082, 2915, 1082, 1082, 1082, 1082, 1082, 1082, 1082, 1082,
+ 1082, 1082, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 1082,
+
+ 2915, 1082, 1082, 2915, 1082, 1082, 2915, 1082, 1082, 2915,
+ 1082, 1082, 2915, 1082, 1082, 1082, 2915, 1082, 3360, 1082,
+ 1082, 1082, 2915, 1082, 1082, 1082, 1082, 1082, 1082, 2915,
+ 2915, 2915, 1082, 1082, 2915, 1082, 1082, 1082, 2915, 1082,
+ 2915, 1082, 1082, 2915, 1082, 1082, 1082, 1082, 1082, 1082,
+ 2915, 1082, 1082, 1082, 1082, 2915, 1082, 1082, 2915, 1082,
+ 2915, 2915, 1082, 2915, 1082, 1082, 1082, 1082, 1082, 1082,
+ 2915, 3361, 1082, 1082, 2915, 2915, 1082, 1082, 2915, 1082,
+ 3362, 3363, 3363, 3364, 3365, 3366, 3364, 3364, 3367, 3368,
+ 3369, 3370, 3370, 3371, 3372, 3373, 3364, 3363, 3367, 3368,
+
+ 3369, 3363, 3364, 3364, 3367, 3368, 3369, 3367, 3364, 3374,
+ 3375, 3376, 3371, 3370, 3377, 3378, 3379, 3380, 3381, 3381,
+ 3382, 3383, 3384, 3382, 3382, 3385, 3386, 3387, 3388, 3388,
+ 3389, 3390, 3391, 3382, 3381, 3385, 3386, 3387, 3381, 3382,
+ 3382, 3385, 3386, 3387, 3385, 3382, 3392, 3393, 3394, 3389,
+ 3388, 3395, 3396, 3397, 2915, 2915, 2915, 3398, 3398, 1359,
+ 2915, 2915, 2915, 3398, 3399, 3400, 3401, 1359, 1359, 1359,
+ 2915, 3402, 2915, 1359, 2915, 1371, 1364, 3399, 3400, 3402,
+ 2915, 3402, 1359, 2915, 1359, 2915, 1359, 1359, 1359, 1359,
+ 1359, 1359, 1359, 2915, 1359, 1359, 1359, 1359, 2915, 1359,
+
+ 1359, 2915, 2915, 2915, 1364, 3399, 3400, 3403, 3404, 1359,
+ 2915, 3404, 2915, 2915, 1364, 3399, 3400, 1359, 2915, 3405,
+ 2915, 1359, 2915, 1364, 3399, 3400, 1359, 2915, 1359, 1359,
+ 1359, 1359, 1359, 1359, 2915, 3406, 1359, 1359, 1359, 1359,
+ 2915, 1359, 1359, 1359, 1359, 2915, 1359, 1359, 1359, 2915,
+ 1359, 1359, 2915, 2915, 1359, 2915, 1359, 1359, 2915, 1359,
+ 1359, 2915, 1359, 2915, 1359, 1359, 2915, 2915, 1364, 3399,
+ 3400, 3407, 1359, 2915, 3407, 2915, 2915, 1364, 3399, 3400,
+ 1359, 2915, 1359, 1359, 1359, 2915, 1359, 1359, 1359, 1359,
+ 1359, 1359, 1359, 1359, 1359, 2915, 2915, 2915, 2915, 2915,
+
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 1359, 1359,
+ 2915, 1359, 1359, 1359, 2915, 1359, 1359, 2915, 1359, 1359,
+ 2915, 1359, 1359, 1359, 1359, 1359, 1359, 1359, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 1359, 1359, 2915, 1359, 1359,
+ 2915, 1359, 2915, 1359, 2915, 1359, 2915, 1359, 1359, 3408,
+ 3408, 3408, 1359, 2915, 3409, 1359, 2915, 1359, 1359, 1359,
+ 1359, 1359, 2915, 2915, 2915, 1359, 1359, 1359, 2915, 2915,
+ 1359, 1359, 1359, 2915, 2915, 1359, 1359, 1359, 2915, 1359,
+ 2915, 1359, 1359, 2915, 1359, 1359, 1359, 1359, 2915, 1359,
+ 2915, 1359, 2915, 2915, 1359, 2915, 1359, 1359, 2915, 1359,
+
+ 1371, 3402, 1359, 1359, 2915, 3410, 2915, 2915, 1605, 1364,
+ 3399, 3400, 3410, 3410, 1359, 1359, 2915, 2915, 1359, 1359,
+ 2915, 1359, 3411, 3411, 3412, 3413, 3414, 3412, 3412, 3415,
+ 3416, 3417, 3418, 3418, 3419, 3420, 3421, 3412, 3412, 3415,
+ 3416, 3417, 3415, 3415, 3422, 3423, 3424, 3419, 3419, 3425,
+ 3426, 3427, 3428, 3428, 3429, 3430, 3431, 3429, 3429, 3432,
+ 3433, 3434, 3435, 3435, 3436, 3437, 3438, 3429, 3429, 3432,
+ 3433, 3434, 3432, 3432, 3439, 3440, 3441, 3436, 3436, 3442,
+ 3443, 3444, 3445, 3446, 3446, 2915, 1685, 1685, 2915, 3447,
+ 2915, 3447, 3447, 1685, 2915, 1685, 2915, 1685, 2915, 1685,
+
+ 1685, 2915, 1685, 1685, 2915, 1685, 1685, 2915, 1685, 1685,
+ 2915, 1685, 1685, 2915, 1685, 1685, 1685, 2915, 1685, 3448,
+ 3448, 3448, 2915, 2915, 1685, 1685, 2915, 1685, 1685, 1685,
+ 1685, 1685, 2915, 2915, 2915, 1685, 1685, 3446, 3449, 3450,
+ 3451, 2915, 2915, 1685, 1685, 2915, 1685, 1685, 1685, 2915,
+ 1685, 1685, 2915, 1685, 2915, 1685, 2915, 1685, 1685, 2915,
+ 1685, 1685, 2915, 2915, 1685, 2915, 2915, 1685, 2915, 1685,
+ 1685, 2915, 1685, 1685, 2915, 1685, 2915, 2915, 1685, 2915,
+ 1685, 1685, 2915, 2915, 2915, 3452, 1685, 1738, 1784, 3453,
+ 3454, 3455, 3455, 3455, 1685, 2915, 1685, 1685, 2915, 1685,
+
+ 1685, 2915, 1685, 1685, 1685, 1685, 1685, 1685, 1685, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 1685, 2915, 1685, 1685, 2915, 1685,
+ 2915, 1685, 2915, 1685, 2915, 1685, 1685, 1685, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 1685, 2915, 1685, 1685,
+ 2915, 1685, 2915, 1685, 1685, 2915, 2915, 1685, 2915, 1685,
+ 3456, 3456, 1685, 1685, 2915, 2915, 2915, 1685, 1738, 3449,
+ 3450, 3457, 2915, 1685, 2915, 1685, 1685, 2915, 1685, 1685,
+ 1685, 2915, 2915, 2915, 1685, 1685, 1685, 1685, 1685, 2915,
+ 2915, 1685, 1685, 1685, 1685, 1685, 1685, 1685, 1685, 1685,
+
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 1685, 2915,
+ 1685, 2915, 1685, 1685, 1685, 2915, 2915, 1685, 2915, 1685,
+ 1685, 2915, 1685, 1689, 3447, 3447, 1685, 2915, 1685, 2915,
+ 3458, 3458, 3458, 1685, 2915, 2915, 1685, 2915, 1685, 1685,
+ 1685, 1685, 1685, 2915, 2915, 2915, 1685, 1685, 3447, 1685,
+ 2915, 1685, 2915, 1685, 2915, 2915, 1685, 2915, 1685, 1685,
+ 2915, 1685, 1685, 2915, 1685, 1685, 2915, 1685, 1685, 2915,
+ 1685, 1685, 1685, 2915, 1685, 2915, 1685, 1685, 2915, 1685,
+ 1685, 1685, 2915, 2915, 1685, 1685, 2915, 1685, 1685, 2915,
+ 1685, 2915, 1685, 2915, 2915, 1685, 1685, 1685, 2915, 2915,
+
+ 1685, 2915, 1685, 1685, 1685, 2915, 1685, 1685, 2915, 1685,
+ 1784, 3452, 2915, 2915, 1738, 3449, 3450, 3452, 3452, 1784,
+ 1784, 1738, 1789, 1738, 1789, 1789, 1685, 2915, 1685, 2915,
+ 1685, 2915, 1685, 1685, 1685, 1685, 1685, 1685, 2915, 2915,
+ 3459, 2915, 2915, 2915, 3460, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 1930, 3458, 1685, 2915, 1685,
+ 2915, 1685, 2915, 2915, 1685, 2915, 1685, 2915, 1685, 1685,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 1685, 2915,
+ 1685, 1685, 2915, 1685, 1685, 2915, 2915, 1685, 2915, 1685,
+ 2915, 1685, 3456, 3456, 3456, 1685, 1865, 3457, 1685, 3457,
+
+ 2915, 1685, 2915, 1685, 1685, 2915, 1685, 1685, 2915, 2915,
+ 1685, 1685, 1685, 1685, 2915, 2915, 1685, 1685, 1685, 1685,
+ 1685, 1685, 1685, 1685, 1685, 1685, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 1685, 2915, 1685, 1685,
+ 1685, 2915, 1685, 1685, 1685, 2915, 1685, 1685, 2915, 2915,
+ 1685, 2915, 3458, 1685, 3447, 2915, 1685, 1685, 1685, 2915,
+ 2915, 2915, 3461, 1685, 1685, 2915, 1685, 2915, 1685, 2915,
+ 1685, 1956, 3462, 2915, 1956, 2915, 1738, 3449, 3450, 1685,
+ 2915, 1685, 2915, 1685, 1685, 2915, 1685, 2915, 1685, 1685,
+ 2915, 1685, 1685, 1685, 1685, 2915, 1685, 2915, 1685, 2915,
+
+ 1685, 2915, 2915, 1685, 2915, 1685, 1685, 2915, 1685, 2915,
+ 1685, 1685, 2915, 2915, 1685, 1685, 1685, 3452, 1685, 2915,
+ 1685, 2915, 1685, 1685, 1685, 1685, 1685, 2915, 2915, 3459,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 1930, 3458,
+ 1685, 3458, 2915, 1685, 2915, 1685, 1685, 1685, 1685, 1685,
+ 1685, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 1685, 2915,
+ 1685, 1685, 2915, 2915, 1685, 2915, 2915, 1685, 1685, 1865,
+ 3457, 1685, 3457, 2915, 1685, 2915, 1685, 1685, 2915, 2915,
+ 1685, 1685, 1685, 2915, 1685, 1685, 2915, 1685, 2915, 2915,
+ 2915, 2915, 1685, 1685, 1685, 1685, 1685, 1685, 1685, 1685,
+
+ 1685, 1685, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 1685, 2915, 1685, 1685, 1685, 1685, 1685, 2915, 2915,
+ 1685, 3447, 3447, 2915, 1685, 1685, 2915, 3463, 3461, 1685,
+ 2915, 1685, 2915, 1685, 2915, 3462, 1956, 3462, 1685, 2915,
+ 1685, 2915, 1685, 2915, 1685, 1685, 2915, 1685, 1685, 1685,
+ 1685, 2915, 1685, 1685, 2915, 2915, 1685, 2915, 2915, 1685,
+ 2915, 1685, 1685, 2915, 1685, 2915, 1685, 1685, 2915, 2915,
+ 1685, 1685, 1685, 2915, 1685, 1685, 1685, 1685, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 1685,
+ 2915, 1685, 1685, 1685, 1685, 1685, 1685, 2915, 2915, 2915,
+
+ 1685, 2915, 1685, 1685, 2915, 2915, 1685, 1685, 1685, 2915,
+ 1685, 2915, 1685, 2915, 1685, 1685, 1685, 1685, 2915, 2915,
+ 1685, 1685, 1685, 1685, 1685, 1685, 1685, 1685, 1685, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 1685, 2915,
+ 1685, 1685, 2915, 1685, 1685, 2915, 3464, 3463, 1685, 1685,
+ 2915, 1685, 2915, 3462, 1685, 2915, 1685, 1685, 1685, 1685,
+ 2915, 1685, 2915, 1685, 1685, 2915, 2915, 2915, 2915, 1685,
+ 2915, 1685, 2915, 2915, 1685, 2915, 1685, 1685, 2915, 2915,
+ 1685, 1685, 1685, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 1685, 2915, 1685, 1685, 1685, 1685, 1956, 3462, 2915, 1685,
+
+ 1685, 2915, 2915, 1685, 1685, 1685, 2915, 1685, 2915, 2915,
+ 1685, 1685, 1685, 1685, 2915, 2915, 1685, 1685, 1685, 1685,
+ 1685, 1685, 1685, 1685, 1685, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 1685, 2915, 1685, 1685, 3458, 1685, 2915,
+ 3464, 1685, 1685, 2915, 1685, 2915, 1685, 2915, 1685, 1685,
+ 1685, 1685, 2915, 1685, 2915, 1685, 1685, 2915, 2915, 2915,
+ 2915, 1685, 2915, 1685, 2915, 1685, 1685, 2915, 2915, 2915,
+ 1685, 2915, 1685, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 1685, 1685, 1685, 1685, 3462, 2915, 1685, 1685, 2915,
+ 2915, 2915, 1685, 1685, 1685, 2915, 1685, 1685, 2915, 1685,
+
+ 1685, 1685, 1685, 1685, 1685, 1685, 1685, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 1685, 2915, 1685, 2915, 1685,
+ 3458, 3458, 1685, 2915, 1685, 1685, 1685, 2915, 1685, 2915,
+ 1685, 1685, 2915, 2915, 2915, 2915, 1685, 2915, 1685, 2915,
+ 1685, 2915, 1685, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 1685, 1685, 1685, 1685, 2915, 1685, 2915, 1685,
+ 1685, 1685, 2915, 1685, 2915, 1685, 1685, 1685, 1685, 1685,
+ 1685, 1685, 1685, 1685, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 1685, 2915, 1685, 2915, 1685, 2915, 1685, 2915,
+ 1685, 2915, 1685, 2915, 1685, 1685, 2915, 2915, 1685, 2915,
+
+ 1685, 2915, 1685, 2915, 2915, 1685, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 1685, 2915, 1685, 2915, 1685,
+ 2915, 2915, 1685, 2915, 2915, 1685, 1685, 1685, 2915, 2915,
+ 1685, 2915, 1685, 1685, 1685, 1685, 1685, 1685, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 1685, 2915, 1685, 2915,
+ 1685, 2915, 1685, 2915, 2915, 3465, 1685, 1685, 2915, 2915,
+ 1685, 2915, 1685, 2915, 2915, 1685, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 1685, 2915, 1685, 2915, 1685, 2915,
+ 2915, 1685, 2915, 2915, 1685, 1685, 2915, 2915, 1685, 1685,
+ 1685, 1685, 1685, 1685, 2915, 2915, 2915, 2915, 2915, 2915,
+
+ 1685, 2915, 1685, 2915, 1685, 2915, 1685, 2915, 1685, 2915,
+ 2915, 1685, 2755, 3465, 2915, 2915, 2755, 1738, 3449, 3450,
+ 3465, 3465, 1685, 1685, 2915, 2915, 1685, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 1685, 1685, 1685, 1685, 2915,
+ 2915, 2915, 2915, 2915, 2915, 1685, 2915, 1685, 2915, 2915,
+ 2755, 3465, 3465, 1685, 1685, 2915, 2915, 2915, 2915, 2915,
+ 2915, 1685, 1685, 1685, 2915, 2915, 2915, 1685, 2915, 1685,
+ 2915, 2915, 2915, 1685, 1685, 1685, 2915, 2915, 1685, 2915,
+ 1685, 2915, 2915, 2915, 1685, 1685, 1685, 2915, 2915, 1685,
+ 2915, 2915, 2915, 1685, 1685, 2915, 2915, 1685, 2915, 2915,
+
+ 2915, 2915, 1685, 2915, 2915, 1685, 2915, 2915, 1685, 2915,
+ 2915, 1685, 2915, 2915, 0, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 1685,
- 2912, 1685, 1685, 1685, 1685, 1685, 2912, 2912, 1685, 3444,
- 3444, 2912, 1685, 1685, 2912, 3460, 3458, 1685, 2912, 1685,
- 2912, 1685, 2912, 3459, 1954, 3459, 1685, 2912, 1685, 2912,
- 1685, 2912, 1685, 1685, 2912, 1685, 1685, 1685, 1685, 2912,
- 1685, 1685, 2912, 2912, 1685, 2912, 2912, 1685, 2912, 1685,
- 1685, 2912, 1685, 2912, 1685, 1685, 2912, 2912, 1685, 1685,
- 1685, 2912, 1685, 1685, 1685, 1685, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 1685, 2912, 1685, 1685,
- 1685, 1685, 1685, 1685, 2912, 2912, 2912, 1685, 2912, 1685,
-
- 1685, 2912, 2912, 1685, 1685, 1685, 2912, 1685, 2912, 1685,
- 2912, 1685, 1685, 1685, 1685, 2912, 2912, 1685, 1685, 1685,
- 1685, 1685, 1685, 1685, 1685, 1685, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 1685, 2912, 1685, 1685, 2912,
- 1685, 1685, 2912, 3461, 3460, 1685, 1685, 2912, 1685, 2912,
- 3459, 1685, 2912, 1685, 1685, 1685, 1685, 2912, 1685, 2912,
- 1685, 1685, 2912, 2912, 2912, 2912, 1685, 2912, 1685, 2912,
- 2912, 1685, 2912, 1685, 1685, 2912, 2912, 1685, 1685, 1685,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 1685, 2912, 1685,
- 1685, 1685, 1685, 1954, 3459, 2912, 1685, 1685, 2912, 2912,
-
- 1685, 1685, 1685, 2912, 1685, 2912, 2912, 1685, 1685, 1685,
- 1685, 2912, 2912, 1685, 1685, 1685, 1685, 1685, 1685, 1685,
- 1685, 1685, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 1685, 2912, 1685, 1685, 3455, 1685, 2912, 3461, 1685, 1685,
- 2912, 1685, 2912, 1685, 2912, 1685, 1685, 1685, 1685, 2912,
- 1685, 2912, 1685, 1685, 2912, 2912, 2912, 2912, 1685, 2912,
- 1685, 2912, 1685, 1685, 2912, 2912, 2912, 1685, 2912, 1685,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 1685, 1685,
- 1685, 1685, 3459, 2912, 1685, 1685, 2912, 2912, 2912, 1685,
- 1685, 1685, 2912, 1685, 1685, 2912, 1685, 1685, 1685, 1685,
-
- 1685, 1685, 1685, 1685, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 1685, 2912, 1685, 2912, 1685, 3455, 3455, 1685,
- 2912, 1685, 1685, 1685, 2912, 1685, 2912, 1685, 1685, 2912,
- 2912, 2912, 2912, 1685, 2912, 1685, 2912, 1685, 2912, 1685,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 1685,
- 1685, 1685, 1685, 2912, 1685, 2912, 1685, 1685, 1685, 2912,
- 1685, 2912, 1685, 1685, 1685, 1685, 1685, 1685, 1685, 1685,
- 1685, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 1685,
- 2912, 1685, 2912, 1685, 2912, 1685, 2912, 1685, 2912, 1685,
- 2912, 1685, 1685, 2912, 2912, 1685, 2912, 1685, 2912, 1685,
-
- 2912, 2912, 1685, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 1685, 2912, 1685, 2912, 1685, 2912, 2912, 1685,
- 2912, 2912, 1685, 1685, 1685, 2912, 2912, 1685, 2912, 1685,
- 1685, 1685, 1685, 1685, 1685, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 1685, 2912, 1685, 2912, 1685, 2912, 1685,
- 2912, 2912, 3462, 1685, 1685, 2912, 2912, 1685, 2912, 1685,
- 2912, 2912, 1685, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 1685, 2912, 1685, 2912, 1685, 2912, 2912, 1685, 2912,
- 2912, 1685, 1685, 2912, 2912, 1685, 1685, 1685, 1685, 1685,
- 1685, 2912, 2912, 2912, 2912, 2912, 2912, 1685, 2912, 1685,
-
- 2912, 1685, 2912, 1685, 2912, 1685, 2912, 2912, 1685, 2752,
- 3462, 2912, 2912, 2752, 1738, 3446, 3447, 3462, 3462, 1685,
- 1685, 2912, 2912, 1685, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 1685, 1685, 1685, 1685, 2912, 2912, 2912, 2912,
- 2912, 2912, 1685, 2912, 1685, 2912, 2912, 2752, 3462, 3462,
- 1685, 1685, 2912, 2912, 2912, 2912, 2912, 2912, 1685, 1685,
- 1685, 2912, 2912, 2912, 1685, 2912, 1685, 2912, 2912, 2912,
- 1685, 1685, 1685, 2912, 2912, 1685, 2912, 1685, 2912, 2912,
- 2912, 1685, 1685, 1685, 2912, 2912, 1685, 2912, 2912, 2912,
- 1685, 1685, 2912, 2912, 1685, 2912, 2912, 2912, 2912, 1685,
-
- 2912, 2912, 1685, 2912, 2912, 1685, 2912, 2912, 1685, 2912,
- 2912, 0, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
-
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
-
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
-
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
-
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
-
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915
} ;
static yyconst flex_int16_t yy_nxt[27006] =
{ 0,
- 2912, 125, 126, 717, 127, 128, 129, 717, 129, 130,
+ 2915, 125, 126, 717, 127, 128, 129, 717, 129, 130,
131, 132, 130, 133, 134, 129, 130, 129, 130, 130,
130, 130, 130, 130, 130, 130, 130, 130, 130, 130,
130, 124, 130, 130, 130, 135, 136, 137, 138, 139,
@@ -1942,10 +1942,10 @@ static yyconst flex_int16_t yy_nxt[27006] =
284, 181, 511, 283, 284, 286, 512, 287, 288, 286,
289, 287, 288, 291, 289, 292, 293, 291, 480, 292,
293, 181, 481, 182, 176, 181, 726, 182, 176, 492,
- 317, 315, 318, 319, 317, 315, 318, 319, 493, 2912,
+ 317, 315, 318, 319, 317, 315, 318, 319, 493, 2915,
542, 169, 543, 169, 922, 169, 181, 169, 182, 176,
181, 923, 182, 176, 503, 169, 361, 320, 362, 176,
- 504, 320, 181, 294, 182, 176, 2912, 294, 295, 295,
+ 504, 320, 181, 294, 182, 176, 2915, 294, 295, 295,
181, 295, 182, 296, 295, 295, 295, 295, 295, 295,
295, 295, 295, 295, 295, 295, 295, 295, 295, 295,
295, 295, 295, 295, 295, 295, 295, 295, 295, 295,
@@ -2042,7 +2042,7 @@ static yyconst flex_int16_t yy_nxt[27006] =
169, 169, 169, 169, 169, 169, 169, 169, 169, 169,
169, 169, 169, 169, 169, 169, 169, 169, 169, 370,
370, 370, 370, 370, 370, 371, 371, 371, 372, 169,
- 169, 181, 2911, 182, 176, 181, 583, 182, 176, 169,
+ 169, 181, 2914, 182, 176, 181, 583, 182, 176, 169,
383, 583, 384, 176, 169, 383, 482, 384, 176, 483,
875, 876, 484, 485, 486, 488, 489, 593, 490, 869,
376, 491, 595, 870, 376, 377, 377, 181, 377, 182,
@@ -2055,7 +2055,7 @@ static yyconst flex_int16_t yy_nxt[27006] =
169, 169, 169, 169, 169, 169, 169, 169, 169, 169,
169, 169, 169, 169, 169, 169, 169, 379, 379, 379,
379, 379, 379, 380, 380, 380, 381, 169, 169, 181,
- 2910, 182, 176, 181, 494, 182, 176, 545, 181, 452,
+ 2913, 182, 176, 181, 494, 182, 176, 545, 181, 452,
182, 176, 495, 452, 505, 893, 882, 496, 506, 883,
546, 547, 507, 548, 497, 928, 501, 508, 385, 408,
453, 929, 385, 386, 386, 181, 386, 182, 387, 386,
@@ -2096,8 +2096,8 @@ static yyconst flex_int16_t yy_nxt[27006] =
169, 169, 169, 169, 169, 169, 169, 169, 169, 169,
169, 169, 169, 169, 169, 169, 169, 405, 405, 405,
405, 405, 405, 406, 406, 406, 407, 169, 169, 181,
- 499, 182, 176, 542, 2908, 543, 500, 553, 553, 553,
- 553, 553, 553, 554, 554, 554, 555, 501, 2907, 502,
+ 499, 182, 176, 542, 2911, 543, 500, 553, 553, 553,
+ 553, 553, 553, 554, 554, 554, 555, 501, 2910, 502,
408, 409, 409, 181, 409, 182, 410, 409, 409, 409,
409, 409, 409, 409, 409, 409, 409, 409, 409, 409,
409, 409, 409, 409, 409, 409, 409, 409, 409, 409,
@@ -2177,19 +2177,19 @@ static yyconst flex_int16_t yy_nxt[27006] =
516, 517, 524, 518, 453, 726, 525, 911, 903, 526,
527, 488, 489, 956, 490, 528, 529, 491, 957, 530,
531, 532, 534, 535, 714, 533, 536, 900, 537, 904,
- 538, 901, 715, 938, 716, 539, 939, 540, 2905, 541,
+ 538, 901, 715, 938, 716, 539, 939, 540, 2908, 541,
571, 571, 571, 571, 571, 571, 572, 572, 572, 573,
- 2912, 984, 2904, 985, 454, 454, 454, 454, 454, 454,
+ 2915, 984, 2907, 985, 454, 454, 454, 454, 454, 454,
455, 455, 455, 456, 450, 457, 450, 450, 450, 458,
- 2912, 450, 2902, 450, 450, 450, 450, 450, 450, 450,
+ 2915, 450, 2905, 450, 450, 450, 450, 450, 450, 450,
450, 450, 450, 450, 450, 450, 453, 450, 450, 450,
459, 460, 461, 462, 463, 464, 450, 465, 466, 450,
467, 468, 469, 470, 471, 472, 473, 474, 475, 476,
477, 478, 450, 450, 479, 450, 450, 571, 571, 571,
571, 571, 571, 572, 572, 572, 573, 482, 480, 584,
- 483, 2901, 481, 484, 485, 486, 454, 454, 454, 454,
- 454, 454, 455, 455, 455, 456, 451, 942, 545, 2912,
+ 483, 2904, 481, 484, 485, 486, 454, 454, 454, 454,
+ 454, 454, 455, 455, 455, 456, 451, 942, 545, 2915,
452, 577, 577, 577, 577, 577, 577, 578, 578, 578,
579, 546, 547, 450, 548, 943, 1249, 453, 577, 577,
@@ -2197,8 +2197,8 @@ static yyconst flex_int16_t yy_nxt[27006] =
589, 589, 589, 589, 590, 590, 590, 591, 597, 597,
597, 597, 597, 597, 598, 598, 598, 599, 597, 597,
597, 597, 597, 597, 598, 598, 598, 599, 940, 949,
- 971, 583, 958, 950, 941, 972, 2899, 454, 454, 454,
- 454, 454, 454, 455, 455, 455, 456, 2912, 959, 450,
+ 971, 583, 958, 950, 941, 972, 2902, 454, 454, 454,
+ 454, 454, 454, 455, 455, 455, 456, 2915, 959, 450,
1107, 450, 593, 450, 603, 603, 603, 603, 603, 603,
604, 604, 604, 605, 603, 603, 603, 603, 603, 603,
604, 604, 604, 605, 609, 609, 609, 609, 609, 609,
@@ -2285,17 +2285,17 @@ static yyconst flex_int16_t yy_nxt[27006] =
702, 702, 703, 703, 703, 704, 702, 702, 702, 702,
702, 702, 703, 703, 703, 704, 708, 708, 708, 708,
708, 708, 709, 709, 709, 710, 708, 708, 708, 708,
- 708, 708, 709, 709, 709, 710, 991, 1219, 2898, 1385,
+ 708, 708, 709, 709, 709, 710, 991, 1219, 2901, 1385,
992, 450, 583, 583, 608, 608, 608, 608, 608, 608,
608, 608, 608, 608, 608, 608, 608, 608, 608, 608,
- 608, 608, 608, 595, 1003, 450, 450, 726, 714, 2896,
+ 608, 608, 608, 595, 1003, 450, 450, 726, 714, 2899,
450, 871, 1185, 608, 608, 614, 715, 993, 716, 719,
719, 719, 719, 719, 719, 720, 720, 720, 721, 723,
723, 723, 723, 723, 723, 724, 724, 724, 725, 727,
728, 729, 730, 731, 732, 733, 734, 735, 736, 739,
- 739, 739, 739, 739, 739, 740, 740, 740, 741, 2912,
- 2912, 1388, 2894, 450, 614, 614, 614, 614, 614, 614,
+ 739, 739, 739, 739, 739, 740, 740, 740, 741, 2915,
+ 2915, 1388, 2897, 450, 614, 614, 614, 614, 614, 614,
614, 614, 614, 614, 614, 614, 614, 614, 614, 614,
614, 614, 614, 754, 754, 754, 754, 754, 754, 755,
755, 755, 756, 614, 614, 620, 754, 754, 754, 754,
@@ -2348,31 +2348,31 @@ static yyconst flex_int16_t yy_nxt[27006] =
851, 851, 852, 852, 852, 853, 851, 851, 851, 851,
851, 851, 852, 852, 852, 853, 726, 726, 1500, 1528,
- 2893, 450, 1501, 726, 650, 650, 650, 650, 650, 650,
+ 2896, 450, 1501, 726, 650, 650, 650, 650, 650, 650,
650, 650, 650, 650, 650, 650, 650, 650, 650, 650,
650, 650, 650, 857, 857, 857, 857, 857, 857, 858,
858, 858, 859, 650, 650, 656, 857, 857, 857, 857,
- 857, 857, 858, 858, 858, 859, 494, 505, 863, 2890,
- 499, 506, 521, 1127, 495, 507, 500, 514, 2912, 496,
- 508, 522, 450, 887, 924, 2912, 497, 501, 925, 502,
+ 857, 857, 858, 858, 858, 859, 494, 505, 863, 2893,
+ 499, 506, 521, 1127, 495, 507, 500, 514, 2915, 496,
+ 508, 522, 450, 887, 924, 2915, 497, 501, 925, 502,
862, 888, 726, 516, 517, 878, 518, 889, 450, 879,
726, 890, 891, 1487, 656, 656, 656, 656, 656, 656,
656, 656, 656, 656, 656, 656, 656, 656, 656, 656,
656, 656, 656, 524, 917, 880, 926, 525, 918, 1097,
- 526, 527, 2889, 656, 656, 664, 528, 529, 881, 450,
- 530, 531, 532, 534, 535, 714, 533, 536, 2888, 537,
+ 526, 527, 2892, 656, 656, 664, 528, 529, 881, 450,
+ 530, 531, 532, 534, 535, 714, 533, 536, 2891, 537,
561, 538, 1184, 715, 561, 716, 539, 919, 540, 1093,
- 541, 450, 920, 944, 714, 2912, 953, 945, 921, 998,
+ 541, 450, 920, 944, 714, 2915, 953, 945, 921, 998,
954, 450, 715, 1084, 716, 965, 450, 1085, 930, 966,
- 2886, 450, 931, 1490, 664, 664, 664, 664, 664, 664,
+ 2889, 450, 931, 1490, 664, 664, 664, 664, 664, 664,
664, 664, 664, 664, 664, 664, 664, 664, 664, 664,
664, 664, 664, 932, 946, 584, 1252, 947, 955, 933,
450, 1086, 967, 664, 664, 670, 968, 1117, 450, 969,
- 1549, 2885, 450, 934, 571, 571, 571, 571, 571, 571,
+ 1549, 2888, 450, 934, 571, 571, 571, 571, 571, 571,
572, 572, 572, 573, 577, 577, 577, 577, 577, 577,
- 578, 578, 578, 579, 1119, 1123, 450, 584, 1561, 2881,
+ 578, 578, 578, 579, 1119, 1123, 450, 584, 1561, 2884,
450, 1524, 450, 450, 589, 589, 589, 589, 589, 589,
590, 590, 590, 591, 670, 670, 670, 670, 670, 670,
670, 670, 670, 670, 670, 670, 670, 670, 670, 670,
@@ -2382,7 +2382,7 @@ static yyconst flex_int16_t yy_nxt[27006] =
589, 589, 590, 590, 590, 591, 603, 603, 603, 603,
603, 603, 604, 604, 604, 605, 609, 609, 609, 609,
609, 609, 610, 610, 610, 611, 615, 615, 615, 615,
- 615, 615, 616, 616, 616, 617, 450, 1491, 1567, 2880,
+ 615, 615, 616, 616, 616, 617, 450, 1491, 1567, 2883,
450, 1492, 1448, 450, 676, 676, 676, 676, 676, 676,
676, 676, 676, 676, 676, 676, 676, 676, 676, 676,
676, 676, 676, 621, 621, 621, 621, 621, 621, 622,
@@ -2408,41 +2408,41 @@ static yyconst flex_int16_t yy_nxt[27006] =
690, 690, 691, 691, 691, 692, 696, 696, 696, 696,
696, 696, 697, 697, 697, 698, 702, 702, 702, 702,
702, 702, 703, 703, 703, 704, 708, 708, 708, 708,
- 708, 708, 709, 709, 709, 710, 1023, 1559, 1357, 2132,
- 2877, 2133, 1357, 450, 695, 695, 695, 695, 695, 695,
+ 708, 708, 709, 709, 709, 710, 1023, 1559, 1357, 2134,
+ 2880, 2135, 1357, 450, 695, 695, 695, 695, 695, 695,
695, 695, 695, 695, 695, 695, 695, 695, 695, 695,
695, 695, 695, 727, 728, 729, 730, 731, 732, 733,
734, 735, 736, 695, 695, 701, 754, 754, 754, 754,
754, 754, 755, 755, 755, 756, 718, 1125, 1262, 1237,
- 1126, 1025, 879, 583, 2147, 2147, 879, 450, 450, 450,
+ 1126, 1025, 879, 583, 2149, 2149, 879, 450, 450, 450,
450, 719, 719, 719, 719, 719, 719, 720, 720, 720,
- 721, 718, 1215, 453, 588, 451, 1030, 1359, 2875, 452,
+ 721, 718, 1215, 453, 588, 451, 1030, 1359, 2878, 452,
450, 450, 738, 1042, 701, 701, 701, 701, 701, 701,
701, 701, 701, 701, 701, 701, 701, 701, 701, 701,
701, 701, 701, 739, 739, 739, 739, 739, 739, 740,
740, 740, 741, 701, 701, 707, 1026, 1026, 1026, 1026,
1026, 1026, 1027, 1027, 1027, 1028, 718, 1226, 450, 450,
- 1242, 1035, 1044, 1220, 2874, 450, 738, 1267, 1221, 450,
+ 1242, 1035, 1044, 1220, 2877, 450, 738, 1267, 1221, 450,
450, 1031, 1031, 1031, 1031, 1031, 1031, 1032, 1032, 1032,
1033, 1523, 1045, 1045, 1045, 1045, 1045, 1045, 1046, 1046,
- 1046, 1047, 726, 2870, 707, 707, 707, 707, 707, 707,
+ 1046, 1047, 726, 2873, 707, 707, 707, 707, 707, 707,
707, 707, 707, 707, 707, 707, 707, 707, 707, 707,
707, 707, 707, 869, 878, 878, 1049, 870, 879, 879,
- 2773, 2773, 2869, 707, 707, 744, 1036, 1036, 1036, 1036,
+ 2776, 2776, 2872, 707, 707, 744, 1036, 1036, 1036, 1036,
1036, 1036, 1037, 1037, 1037, 1038, 1050, 1050, 1050, 1050,
1050, 1050, 1051, 1051, 1051, 1052, 738, 726, 1099, 450,
- 450, 450, 450, 451, 1091, 450, 2912, 452, 2912, 2912,
- 1396, 2912, 745, 746, 746, 1369, 747, 748, 749, 750,
+ 450, 450, 450, 451, 1091, 450, 2915, 452, 2915, 2915,
+ 1396, 2915, 745, 746, 746, 1369, 747, 748, 749, 750,
750, 750, 750, 750, 750, 750, 750, 750, 750, 750,
750, 750, 750, 750, 750, 750, 750, 750, 750, 750,
- 753, 507, 450, 500, 1817, 450, 1054, 1116, 2912, 907,
- 1270, 2912, 1387, 908, 501, 2912, 1488, 1383, 450, 450,
- 2912, 1384, 1818, 2868, 2912, 1489, 1055, 1055, 1055, 1055,
+ 753, 507, 450, 500, 1819, 450, 1054, 1116, 2915, 907,
+ 1270, 2915, 1387, 908, 501, 2915, 1488, 1383, 450, 450,
+ 2915, 1384, 1820, 2871, 2915, 1489, 1055, 1055, 1055, 1055,
1055, 1055, 1056, 1056, 1056, 1057, 760, 760, 760, 760,
- 760, 760, 761, 761, 761, 762, 1143, 2866, 450, 753,
+ 760, 760, 761, 761, 761, 762, 1143, 2869, 450, 753,
753, 753, 753, 753, 753, 753, 753, 753, 753, 753,
753, 753, 753, 753, 753, 753, 753, 753, 766, 766,
@@ -2451,7 +2451,7 @@ static yyconst flex_int16_t yy_nxt[27006] =
774, 780, 780, 780, 780, 780, 780, 781, 781, 781,
782, 788, 788, 788, 788, 788, 788, 789, 789, 789,
790, 796, 796, 796, 796, 796, 796, 797, 797, 797,
- 798, 450, 450, 2775, 2775, 2777, 2777, 1585, 1519, 759,
+ 798, 450, 450, 2778, 2778, 2780, 2780, 1585, 1519, 759,
759, 759, 759, 759, 759, 759, 759, 759, 759, 759,
759, 759, 759, 759, 759, 759, 759, 759, 802, 802,
802, 802, 802, 802, 803, 803, 803, 804, 759, 759,
@@ -2460,7 +2460,7 @@ static yyconst flex_int16_t yy_nxt[27006] =
810, 818, 818, 818, 818, 818, 818, 819, 819, 819,
820, 824, 824, 824, 824, 824, 824, 825, 825, 825,
826, 830, 830, 830, 830, 830, 830, 831, 831, 831,
- 832, 450, 1600, 2864, 2863, 2862, 2858, 1560, 450, 765,
+ 832, 450, 1600, 2867, 2866, 2865, 2861, 1560, 450, 765,
765, 765, 765, 765, 765, 765, 765, 765, 765, 765,
765, 765, 765, 765, 765, 765, 765, 765, 839, 839,
839, 839, 839, 839, 840, 840, 840, 841, 765, 765,
@@ -2468,150 +2468,150 @@ static yyconst flex_int16_t yy_nxt[27006] =
847, 851, 851, 851, 851, 851, 851, 852, 852, 852,
853, 857, 857, 857, 857, 857, 857, 858, 858, 858,
- 859, 450, 1390, 2854, 450, 2853, 450, 450, 450, 1113,
+ 859, 450, 1390, 2857, 450, 2856, 450, 450, 450, 1113,
450, 1389, 1391, 1114, 1431, 1392, 1438, 1439, 1495, 771,
771, 771, 771, 771, 771, 771, 771, 771, 771, 771,
771, 771, 771, 771, 771, 771, 771, 771, 1088, 870,
1100, 1103, 1089, 870, 1101, 1104, 450, 1120, 771, 771,
- 779, 1121, 1115, 2844, 450, 1140, 2842, 1128, 1090, 1141,
+ 779, 1121, 1115, 2847, 450, 1140, 2845, 1128, 1090, 1141,
453, 1129, 925, 1105, 954, 1484, 925, 1427, 954, 1102,
- 1367, 1428, 1497, 2841, 1085, 450, 1122, 450, 450, 1130,
+ 1367, 1428, 1497, 2844, 1085, 450, 1122, 450, 450, 1130,
1092, 1498, 450, 453, 450, 453, 1131, 450, 1142, 912,
- 1182, 1363, 450, 913, 450, 1562, 2840, 1526, 450, 779,
+ 1182, 1363, 450, 913, 450, 1562, 2843, 1526, 450, 779,
779, 779, 779, 779, 779, 779, 779, 779, 779, 779,
779, 779, 779, 779, 779, 779, 779, 779, 908, 907,
- 1146, 1149, 908, 908, 1147, 1150, 1152, 2839, 779, 779,
+ 1146, 1149, 908, 908, 1147, 1150, 1152, 2842, 779, 779,
787, 913, 912, 1156, 917, 913, 913, 1157, 918, 453,
- 1178, 2838, 1113, 1151, 1179, 1148, 1114, 1154, 1145, 450,
- 1493, 1494, 453, 1504, 1505, 1144, 450, 450, 450, 2837,
+ 1178, 2841, 1113, 1151, 1179, 1148, 1114, 1154, 1145, 450,
+ 1493, 1494, 453, 1504, 1505, 1144, 450, 450, 450, 2840,
450, 531, 1383, 1216, 1506, 450, 1384, 1217, 1153, 450,
- 450, 1176, 1553, 450, 924, 2831, 1603, 1180, 925, 787,
+ 450, 1176, 1553, 450, 924, 2834, 1603, 1180, 925, 787,
787, 787, 787, 787, 787, 787, 787, 787, 787, 787,
787, 787, 787, 787, 787, 787, 787, 787, 917, 924,
- 450, 1200, 918, 925, 2830, 1201, 1183, 1218, 787, 787,
+ 450, 1200, 918, 925, 2833, 1201, 1183, 1218, 787, 787,
795, 450, 1203, 450, 1206, 944, 1204, 1222, 1207, 945,
1209, 1223, 450, 1616, 1210, 1158, 1159, 1181, 1691, 1202,
- 529, 2829, 1691, 1160, 2828, 1161, 450, 1162, 450, 1163,
+ 529, 2832, 1691, 1160, 2831, 1161, 450, 1162, 450, 1163,
1164, 1205, 1165, 1208, 1166, 1211, 1234, 1224, 1227, 450,
- 1235, 450, 1228, 1229, 450, 2827, 2823, 450, 2822, 795,
+ 1235, 450, 1228, 1229, 450, 2830, 2826, 450, 2825, 795,
795, 795, 795, 795, 795, 795, 795, 795, 795, 795,
- 795, 795, 795, 795, 795, 795, 795, 795, 918, 2803,
- 1239, 1692, 918, 450, 1239, 1693, 2801, 1236, 795, 795,
+ 795, 795, 795, 795, 795, 795, 795, 795, 918, 2806,
+ 1239, 1692, 918, 450, 1239, 1693, 2804, 1236, 795, 795,
801, 930, 1212, 966, 944, 931, 1213, 966, 945, 453,
- 1114, 453, 1241, 2912, 1114, 1167, 1168, 1435, 2799, 2797,
- 2557, 1436, 1214, 1169, 453, 1170, 1186, 1171, 1187, 1172,
- 1173, 453, 1174, 1243, 1175, 2557, 1188, 1244, 450, 450,
- 1189, 450, 1190, 1191, 1192, 1251, 2796, 1233, 450, 801,
+ 1114, 453, 1241, 2915, 1114, 1167, 1168, 1435, 2802, 2800,
+ 2560, 1436, 1214, 1169, 453, 1170, 1186, 1171, 1187, 1172,
+ 1173, 453, 1174, 1243, 1175, 2560, 1188, 1244, 450, 450,
+ 1189, 450, 1190, 1191, 1192, 1251, 2799, 1233, 450, 801,
801, 801, 801, 801, 801, 801, 801, 801, 801, 801,
801, 801, 801, 801, 801, 801, 801, 801, 931, 1238,
- 450, 2129, 931, 1239, 2795, 1245, 2794, 2130, 801, 801,
- 807, 945, 2793, 953, 953, 945, 965, 954, 954, 453,
- 966, 1240, 1383, 1193, 1134, 1194, 1384, 2792, 1134, 2778,
+ 450, 2131, 931, 1239, 2798, 1245, 2797, 2132, 801, 801,
+ 807, 945, 2796, 953, 953, 945, 965, 954, 954, 453,
+ 966, 1240, 1383, 1193, 1134, 1194, 1384, 2795, 1134, 2781,
- 2770, 1204, 453, 1195, 975, 1204, 450, 1196, 975, 1197,
+ 2773, 1204, 453, 1195, 975, 1204, 450, 1196, 975, 1197,
1198, 1199, 965, 1258, 1230, 1136, 966, 1259, 1231, 1232,
- 450, 450, 453, 450, 1261, 978, 2769, 1246, 1250, 807,
+ 450, 450, 453, 450, 1261, 978, 2772, 1246, 1250, 807,
807, 807, 807, 807, 807, 807, 807, 807, 807, 807,
807, 807, 807, 807, 807, 807, 807, 807, 1255, 1253,
- 450, 1260, 1256, 1263, 2768, 2767, 2766, 1264, 807, 807,
- 817, 1271, 988, 989, 2764, 1272, 989, 989, 991, 991,
- 1696, 2759, 992, 992, 1697, 1265, 1213, 1257, 2757, 1217,
- 1213, 450, 1266, 1217, 453, 450, 1023, 1273, 1275, 2756,
- 450, 988, 2753, 1583, 2751, 989, 1543, 453, 450, 450,
+ 450, 1260, 1256, 1263, 2771, 2770, 2769, 1264, 807, 807,
+ 817, 1271, 988, 989, 2767, 1272, 989, 989, 991, 991,
+ 1696, 2762, 992, 992, 1697, 1265, 1213, 1257, 2760, 1217,
+ 1213, 450, 1266, 1217, 453, 450, 1023, 1273, 1275, 2759,
+ 450, 988, 2756, 1583, 2754, 989, 1543, 453, 450, 450,
453, 1274, 1276, 1023, 1084, 1278, 450, 1280, 1085, 817,
817, 817, 817, 817, 817, 817, 817, 817, 817, 817,
817, 817, 817, 817, 817, 817, 817, 817, 450, 992,
- 2749, 2747, 1698, 992, 1277, 450, 1699, 450, 817, 817,
- 823, 451, 1101, 1368, 1121, 452, 1101, 2745, 1121, 1615,
+ 2752, 2750, 1698, 992, 1277, 450, 1699, 450, 817, 817,
+ 823, 451, 1101, 1368, 1121, 452, 1101, 2748, 1121, 1615,
453, 719, 719, 719, 719, 719, 719, 720, 720, 720,
- 721, 2743, 718, 453, 2742, 453, 1279, 1025, 719, 719,
+ 721, 2746, 718, 453, 2745, 453, 1279, 1025, 719, 719,
719, 719, 719, 719, 720, 720, 720, 721, 718, 1399,
- 2741, 1441, 451, 1030, 1688, 2740, 452, 2739, 1042, 823,
+ 2744, 1441, 451, 1030, 1688, 2743, 452, 2742, 1042, 823,
823, 823, 823, 823, 823, 823, 823, 823, 823, 823,
823, 823, 823, 823, 823, 823, 823, 823, 739, 739,
739, 739, 739, 739, 740, 740, 740, 741, 823, 823,
829, 727, 728, 729, 730, 731, 732, 733, 734, 735,
- 736, 2738, 1026, 1026, 1026, 1026, 1026, 1026, 1027, 1027,
- 1027, 1028, 718, 2737, 2736, 2719, 2711, 1035, 1031, 1031,
- 1031, 1031, 1031, 1031, 1032, 1032, 1032, 1033, 718, 2710,
- 1720, 451, 1707, 1025, 1721, 452, 1708, 2709, 1042, 829,
+ 736, 2741, 1026, 1026, 1026, 1026, 1026, 1026, 1027, 1027,
+ 1027, 1028, 718, 2740, 2739, 2722, 2714, 1035, 1031, 1031,
+ 1031, 1031, 1031, 1031, 1032, 1032, 1032, 1033, 718, 2713,
+ 1720, 451, 1707, 1025, 1721, 452, 1708, 2712, 1042, 829,
829, 829, 829, 829, 829, 829, 829, 829, 829, 829,
829, 829, 829, 829, 829, 829, 829, 829, 739, 739,
739, 739, 739, 739, 740, 740, 740, 741, 829, 829,
- 838, 2708, 1084, 1380, 1709, 1157, 1085, 1372, 2707, 1157,
- 2706, 2704, 1036, 1036, 1036, 1036, 1036, 1036, 1037, 1037,
+ 838, 2711, 1084, 1380, 1709, 1157, 1085, 1372, 2710, 1157,
+ 2709, 2707, 1036, 1036, 1036, 1036, 1036, 1036, 1037, 1037,
1037, 1038, 718, 1381, 1375, 1382, 453, 1283, 1026, 1026,
- 1026, 1026, 1026, 1026, 1027, 1027, 1027, 1028, 718, 2699,
+ 1026, 1026, 1026, 1026, 1027, 1027, 1027, 1028, 718, 2702,
1687, 1100, 1449, 1288, 1393, 1101, 1450, 1486, 1394, 838,
838, 838, 838, 838, 838, 838, 838, 838, 838, 838,
- 838, 838, 838, 838, 838, 838, 838, 838, 2697, 1395,
- 1451, 2632, 450, 450, 2695, 450, 2694, 451, 838, 838,
+ 838, 838, 838, 838, 838, 838, 838, 838, 2700, 1395,
+ 1451, 2635, 450, 450, 2698, 450, 2697, 451, 838, 838,
844, 452, 1400, 1408, 1393, 1141, 451, 1104, 1394, 1141,
- 452, 2691, 1284, 1284, 1284, 1284, 1284, 1284, 1285, 1285,
+ 452, 2694, 1284, 1284, 1284, 1284, 1284, 1284, 1285, 1285,
- 1285, 1286, 718, 2689, 1404, 1409, 453, 1293, 1289, 1289,
+ 1285, 1286, 718, 2692, 1404, 1409, 453, 1293, 1289, 1289,
1289, 1289, 1289, 1289, 1290, 1290, 1290, 1291, 718, 1700,
- 878, 1120, 1459, 1298, 879, 1121, 2685, 1712, 2681, 844,
+ 878, 1120, 1459, 1298, 879, 1121, 2688, 1712, 2684, 844,
844, 844, 844, 844, 844, 844, 844, 844, 844, 844,
- 844, 844, 844, 844, 844, 844, 844, 844, 2679, 1129,
- 451, 450, 450, 1129, 1724, 2678, 2677, 1397, 844, 844,
- 850, 451, 1442, 2676, 1150, 452, 1453, 1453, 1150, 2675,
+ 844, 844, 844, 844, 844, 844, 844, 844, 2682, 1129,
+ 451, 450, 450, 1129, 1724, 2681, 2680, 1397, 844, 844,
+ 850, 451, 1442, 2679, 1150, 452, 1453, 1453, 1150, 2678,
453, 1454, 1294, 1294, 1294, 1294, 1294, 1294, 1295, 1295,
- 1295, 1296, 718, 2674, 2673, 1468, 1472, 1304, 1299, 1299,
- 1299, 1299, 1299, 1299, 1300, 1300, 1300, 1301, 718, 2672,
+ 1295, 1296, 718, 2677, 2676, 1468, 1472, 1304, 1299, 1299,
+ 1299, 1299, 1299, 1299, 1300, 1300, 1300, 1301, 718, 2675,
- 1100, 1455, 907, 1309, 1101, 1456, 908, 2656, 2654, 850,
+ 1100, 1455, 907, 1309, 1101, 1456, 908, 2659, 2657, 850,
850, 850, 850, 850, 850, 850, 850, 850, 850, 850,
- 850, 850, 850, 850, 850, 850, 850, 850, 2648, 2647,
- 1147, 450, 450, 450, 1147, 2646, 2645, 1398, 850, 850,
- 856, 1551, 1179, 1210, 1207, 1219, 1179, 1210, 1207, 2644,
- 2643, 453, 1305, 1305, 1305, 1305, 1305, 1305, 1306, 1306,
- 1306, 1307, 1552, 453, 453, 453, 2639, 1464, 1310, 1310,
+ 850, 850, 850, 850, 850, 850, 850, 850, 2651, 2650,
+ 1147, 450, 450, 450, 1147, 2649, 2648, 1398, 850, 850,
+ 856, 1551, 1179, 1210, 1207, 1219, 1179, 1210, 1207, 2647,
+ 2646, 453, 1305, 1305, 1305, 1305, 1305, 1305, 1306, 1306,
+ 1306, 1307, 1552, 453, 453, 453, 2642, 1464, 1310, 1310,
1310, 1310, 1310, 1310, 1311, 1311, 1311, 1312, 718, 1515,
- 1120, 1538, 1713, 1314, 1121, 1541, 1714, 2637, 738, 856,
+ 1120, 1538, 1713, 1314, 1121, 1541, 1714, 2640, 738, 856,
856, 856, 856, 856, 856, 856, 856, 856, 856, 856,
856, 856, 856, 856, 856, 856, 856, 856, 1445, 1128,
- 1689, 450, 1446, 1129, 1690, 2635, 2633, 1440, 856, 856,
- 451, 1715, 2632, 2631, 452, 451, 1452, 1452, 1447, 452,
- 1370, 1130, 450, 1100, 2630, 2627, 738, 1101, 1044, 450,
- 450, 453, 1140, 1146, 2625, 2621, 1141, 1147, 1315, 1315,
+ 1689, 450, 1446, 1129, 1690, 2638, 2636, 1440, 856, 856,
+ 451, 1715, 2635, 2634, 452, 451, 1452, 1452, 1447, 452,
+ 1370, 1130, 450, 1100, 2633, 2630, 738, 1101, 1044, 450,
+ 450, 453, 1140, 1146, 2628, 2624, 1141, 1147, 1315, 1315,
1315, 1315, 1315, 1315, 1316, 1316, 1316, 1317, 1045, 1045,
1045, 1045, 1045, 1045, 1046, 1046, 1046, 1047, 738, 1719,
1694, 924, 1200, 450, 450, 925, 1201, 1726, 738, 1458,
- 1463, 1727, 2614, 1223, 1393, 912, 1049, 1223, 1394, 913,
- 2612, 454, 454, 454, 454, 454, 454, 455, 455, 455,
+ 1463, 1727, 2617, 1223, 1393, 912, 1049, 1223, 1394, 913,
+ 2615, 454, 454, 454, 454, 454, 454, 455, 455, 455,
456, 451, 450, 450, 453, 452, 1050, 1050, 1050, 1050,
- 1050, 1050, 1051, 1051, 1051, 1052, 450, 2611, 1054, 738,
- 1557, 1710, 453, 2610, 1483, 451, 2609, 1146, 1044, 452,
- 1178, 1147, 1203, 738, 1179, 2608, 1204, 2607, 1055, 1055,
+ 1050, 1050, 1051, 1051, 1051, 1052, 450, 2614, 1054, 738,
+ 1557, 1710, 453, 2613, 1483, 451, 2612, 1146, 1044, 452,
+ 1178, 1147, 1203, 738, 1179, 2611, 1204, 2610, 1055, 1055,
1055, 1055, 1055, 1055, 1056, 1056, 1056, 1057, 1045, 1045,
- 1045, 1045, 1045, 1045, 1046, 1046, 1046, 1047, 450, 2606,
- 2605, 450, 1725, 450, 1235, 2588, 2584, 1514, 1235, 1320,
- 1465, 2577, 454, 454, 454, 454, 454, 454, 455, 455,
- 455, 456, 2912, 1325, 2576, 453, 450, 2566, 450, 1321,
- 1321, 1321, 1321, 1321, 1321, 1322, 1322, 1322, 1323, 2562,
-
- 2558, 2556, 738, 1326, 1326, 1326, 1326, 1326, 1326, 1327,
- 1327, 1327, 1328, 2555, 2552, 451, 1752, 1120, 451, 452,
+ 1045, 1045, 1045, 1045, 1046, 1046, 1046, 1047, 450, 2609,
+ 2608, 450, 1725, 450, 1235, 2591, 2587, 1514, 1235, 1320,
+ 1465, 2580, 454, 454, 454, 454, 454, 454, 455, 455,
+ 455, 456, 2915, 1325, 2579, 453, 450, 2569, 450, 1321,
+ 1321, 1321, 1321, 1321, 1321, 1322, 1322, 1322, 1323, 2565,
+
+ 2561, 2559, 738, 1326, 1326, 1326, 1326, 1326, 1326, 1327,
+ 1327, 1327, 1328, 2558, 2555, 451, 1752, 1120, 451, 452,
1753, 1121, 452, 1156, 917, 1206, 1756, 1157, 918, 1207,
- 1757, 2545, 2543, 450, 450, 450, 450, 450, 450, 450,
+ 1757, 2548, 2546, 450, 450, 450, 450, 450, 450, 450,
450, 450, 450, 450, 450, 450, 450, 450, 450, 450,
450, 450, 1330, 1178, 450, 450, 450, 1179, 1728, 1731,
- 1751, 2541, 450, 450, 457, 1485, 1539, 1509, 458, 2530,
- 2529, 2528, 1331, 1331, 1331, 1331, 1331, 1331, 1332, 1332,
- 1332, 1333, 1516, 1704, 450, 2527, 2526, 1705, 2525, 459,
- 460, 461, 462, 463, 464, 2524, 465, 466, 2523, 467,
+ 1751, 2544, 450, 450, 457, 1485, 1539, 1509, 458, 2533,
+ 2532, 2531, 1331, 1331, 1331, 1331, 1331, 1331, 1332, 1332,
+ 1332, 1333, 1516, 1704, 450, 2530, 2529, 1705, 2528, 459,
+ 460, 461, 462, 463, 464, 2527, 465, 466, 2526, 467,
468, 469, 470, 471, 472, 473, 474, 475, 476, 477,
- 478, 2500, 1706, 479, 973, 974, 973, 973, 973, 975,
+ 478, 2503, 1706, 479, 973, 974, 973, 973, 973, 975,
976, 973, 976, 973, 973, 973, 973, 973, 973, 973,
973, 973, 973, 973, 973, 977, 978, 973, 973, 973,
973, 973, 973, 973, 973, 973, 973, 973, 973, 973,
@@ -2630,11 +2630,11 @@ static yyconst flex_int16_t yy_nxt[27006] =
562, 562, 562, 562, 562, 562, 562, 562, 562, 562,
565, 565, 565, 565, 565, 565, 566, 566, 566, 567,
562, 562, 563, 1510, 1520, 1206, 1212, 1511, 1521, 1207,
- 1213, 1239, 1765, 2499, 2496, 1239, 1766, 1256, 452, 2495,
+ 1213, 1239, 1765, 2502, 2499, 1239, 1766, 1256, 452, 2498,
- 2486, 1256, 1262, 1512, 1522, 1767, 1214, 1455, 2485, 1767,
- 2484, 1456, 453, 1241, 450, 450, 450, 450, 453, 453,
- 2482, 1513, 1537, 560, 560, 560, 560, 560, 560, 560,
+ 2489, 1256, 1262, 1512, 1522, 1767, 1214, 1455, 2488, 1767,
+ 2487, 1456, 453, 1241, 450, 450, 450, 450, 453, 453,
+ 2485, 1513, 1537, 560, 560, 560, 560, 560, 560, 560,
560, 560, 560, 560, 560, 560, 560, 560, 560, 560,
560, 560, 568, 568, 568, 568, 568, 568, 568, 568,
568, 568, 568, 568, 568, 568, 568, 568, 568, 568,
@@ -2644,18 +2644,18 @@ static yyconst flex_int16_t yy_nxt[27006] =
999, 999, 999, 999, 999, 999, 999, 999, 999, 570,
1128, 1200, 1544, 1209, 1129, 1201, 1545, 1210, 1216, 1445,
- 2481, 2477, 1217, 1446, 1367, 1554, 1613, 738, 1085, 1555,
- 1606, 1790, 1130, 451, 2476, 1791, 1186, 452, 1187, 1447,
+ 2484, 2480, 1217, 1446, 1367, 1554, 1613, 738, 1085, 1555,
+ 1606, 1792, 1130, 451, 2479, 1793, 1186, 452, 1187, 1447,
738, 450, 450, 450, 450, 1363, 1188, 1608, 1614, 450,
1189, 1457, 1190, 1191, 1192, 1540, 450, 738, 570, 570,
570, 570, 570, 570, 570, 570, 570, 570, 570, 570,
- 570, 570, 570, 570, 570, 570, 570, 1335, 1203, 451,
- 451, 1701, 1204, 452, 452, 1702, 2473, 570, 570, 576,
- 1341, 1222, 1546, 2466, 1768, 1223, 1547, 1336, 1336, 1336,
+ 570, 570, 570, 570, 570, 570, 570, 1335, 1203, 1768,
+ 451, 1701, 1204, 1769, 452, 1702, 2476, 570, 570, 576,
+ 1341, 1222, 1546, 2469, 1770, 1223, 1547, 1336, 1336, 1336,
1336, 1336, 1336, 1337, 1337, 1337, 1338, 1346, 1703, 450,
1342, 1342, 1342, 1342, 1342, 1342, 1343, 1343, 1343, 1344,
- 1548, 1558, 450, 450, 2465, 1536, 1803, 1347, 1347, 1347,
+ 1548, 1558, 450, 450, 2468, 1536, 1805, 1347, 1347, 1347,
1347, 1347, 1347, 1348, 1348, 1348, 1349, 1042, 576, 576,
576, 576, 576, 576, 576, 576, 576, 576, 576, 576,
576, 576, 576, 576, 576, 576, 576, 739, 739, 739,
@@ -2670,355 +2670,355 @@ static yyconst flex_int16_t yy_nxt[27006] =
583, 583, 583, 583, 583, 583, 583, 583, 583, 583,
583, 585, 585, 585, 585, 585, 585, 586, 586, 586,
587, 583, 583, 582, 1201, 1209, 944, 1222, 1201, 1210,
- 945, 1223, 1259, 451, 1384, 2464, 1259, 452, 1384, 1234,
- 1238, 2463, 588, 1235, 1239, 453, 917, 2460, 1823, 1193,
- 918, 1194, 1824, 453, 738, 453, 450, 450, 450, 1195,
- 1729, 2453, 1240, 1196, 1556, 1197, 1198, 1199, 1542, 1589,
+ 945, 1223, 1259, 451, 1384, 2467, 1259, 452, 1384, 1234,
+ 1238, 2466, 588, 1235, 1239, 453, 917, 2463, 1825, 1193,
+ 918, 1194, 1826, 453, 738, 453, 450, 450, 450, 1195,
+ 1729, 2456, 1240, 1196, 1556, 1197, 1198, 1199, 1542, 1589,
450, 450, 582, 582, 582, 582, 582, 582, 582, 582,
582, 582, 582, 582, 582, 582, 582, 582, 582, 582,
582, 582, 582, 582, 582, 582, 582, 582, 582, 582,
582, 582, 582, 588, 1351, 1568, 1243, 1234, 1578, 1569,
1244, 1235, 1579, 953, 965, 1580, 912, 954, 966, 1581,
- 913, 2450, 2448, 2444, 1352, 1352, 1352, 1352, 1352, 1352,
+ 913, 2453, 2451, 2447, 1352, 1352, 1352, 1352, 1352, 1352,
1353, 1353, 1353, 1354, 1566, 1582, 450, 450, 450, 450,
- 2440, 2435, 1577, 1243, 450, 450, 450, 1244, 2434, 1576,
- 2433, 1795, 588, 588, 588, 588, 588, 588, 588, 588,
+ 2443, 2438, 1577, 1243, 450, 450, 450, 1244, 2437, 1576,
+ 2436, 1797, 588, 588, 588, 588, 588, 588, 588, 588,
588, 588, 588, 588, 588, 588, 588, 588, 588, 588,
- 588, 1572, 1244, 1255, 450, 1255, 1244, 1256, 2432, 1256,
+ 588, 1572, 1244, 1255, 450, 1255, 1244, 1256, 2435, 1256,
- 1258, 588, 588, 596, 1259, 1573, 2431, 1258, 2430, 2429,
+ 1258, 588, 588, 596, 1259, 1573, 2434, 1258, 2433, 2432,
1263, 1259, 1595, 453, 1264, 1598, 1596, 1601, 1264, 1599,
- 1574, 1602, 1264, 2428, 450, 1587, 450, 1592, 1592, 1772,
- 2427, 450, 1265, 1773, 1575, 1593, 1593, 1588, 450, 453,
- 1594, 450, 2411, 450, 1590, 988, 450, 451, 450, 989,
+ 1574, 1602, 1264, 2431, 450, 1587, 450, 1592, 1592, 1774,
+ 2430, 450, 1265, 1775, 1575, 1593, 1593, 1588, 450, 453,
+ 1594, 450, 2414, 450, 1590, 988, 450, 451, 450, 989,
1023, 452, 596, 596, 596, 596, 596, 596, 596, 596,
596, 596, 596, 596, 596, 596, 596, 596, 596, 596,
- 596, 1263, 1774, 991, 1619, 1264, 450, 992, 2409, 2403,
- 1771, 596, 596, 602, 2402, 878, 1023, 1380, 1622, 879,
- 2399, 1372, 1394, 1265, 718, 2397, 1394, 451, 2395, 1025,
+ 596, 1263, 1776, 991, 1619, 1264, 450, 992, 2412, 2406,
+ 1773, 596, 596, 602, 2405, 878, 1023, 1380, 1622, 879,
+ 2402, 1372, 1394, 1265, 718, 2400, 1394, 451, 2398, 1025,
- 2383, 452, 450, 2382, 450, 718, 2381, 1381, 1375, 1382,
- 1030, 2379, 1597, 453, 1716, 719, 719, 719, 719, 719,
- 719, 720, 720, 720, 721, 718, 1717, 1178, 2378, 1711,
+ 2386, 452, 450, 2385, 450, 718, 2384, 1381, 1375, 1382,
+ 1030, 2380, 1597, 453, 1716, 719, 719, 719, 719, 719,
+ 719, 720, 720, 720, 721, 718, 1717, 1178, 2379, 1711,
1035, 1179, 602, 602, 602, 602, 602, 602, 602, 602,
602, 602, 602, 602, 602, 602, 602, 602, 602, 602,
602, 719, 719, 719, 719, 719, 719, 720, 720, 720,
721, 602, 602, 608, 1026, 1026, 1026, 1026, 1026, 1026,
- 1027, 1027, 1027, 1028, 2377, 1031, 1031, 1031, 1031, 1031,
- 1031, 1032, 1032, 1032, 1033, 718, 2372, 1380, 1832, 1203,
- 1025, 1372, 1833, 1204, 2368, 1036, 1036, 1036, 1036, 1036,
+ 1027, 1027, 1027, 1028, 2374, 1031, 1031, 1031, 1031, 1031,
+ 1031, 1032, 1032, 1032, 1033, 718, 2370, 1380, 1834, 1203,
+ 1025, 1372, 1835, 1204, 2369, 1036, 1036, 1036, 1036, 1036,
- 1036, 1037, 1037, 1037, 1038, 718, 2367, 1381, 1375, 1382,
+ 1036, 1037, 1037, 1037, 1038, 718, 2366, 1381, 1375, 1382,
1025, 1042, 608, 608, 608, 608, 608, 608, 608, 608,
608, 608, 608, 608, 608, 608, 608, 608, 608, 608,
608, 739, 739, 739, 739, 739, 739, 740, 740, 740,
- 741, 608, 608, 614, 2364, 1408, 2362, 1722, 452, 1104,
+ 741, 608, 608, 614, 2364, 1408, 2361, 1722, 452, 1104,
451, 1412, 452, 2359, 452, 1026, 1026, 1026, 1026, 1026,
1026, 1027, 1027, 1027, 1028, 718, 1404, 1409, 1414, 453,
- 1283, 2357, 2356, 2350, 2345, 1026, 1026, 1026, 1026, 1026,
+ 1283, 2358, 2352, 2347, 2344, 1026, 1026, 1026, 1026, 1026,
1026, 1027, 1027, 1027, 1028, 718, 1730, 451, 2342, 2340,
1288, 452, 614, 614, 614, 614, 614, 614, 614, 614,
614, 614, 614, 614, 614, 614, 614, 614, 614, 614,
- 614, 1420, 451, 1446, 1851, 1420, 452, 1446, 1851, 2338,
- 1745, 614, 614, 620, 1746, 2333, 451, 2331, 2329, 1744,
+ 614, 1420, 451, 1446, 1853, 1420, 452, 1446, 1853, 2335,
+ 1745, 614, 614, 620, 1746, 2333, 451, 2331, 2328, 1744,
452, 1723, 1423, 1755, 453, 1284, 1284, 1284, 1284, 1284,
- 1284, 1285, 1285, 1285, 1286, 718, 2326, 1544, 1456, 1747,
+ 1284, 1285, 1285, 1285, 1286, 718, 2327, 1544, 1456, 1747,
1293, 1545, 1456, 1758, 1748, 1289, 1289, 1289, 1289, 1289,
- 1289, 1290, 1290, 1290, 1291, 718, 2325, 451, 1758, 453,
+ 1289, 1290, 1290, 1290, 1291, 718, 2324, 451, 1758, 453,
1298, 452, 620, 620, 620, 620, 620, 620, 620, 620,
620, 620, 620, 620, 620, 620, 620, 620, 620, 620,
- 620, 451, 2322, 1792, 451, 452, 1749, 1475, 452, 2317,
-
- 451, 620, 620, 626, 452, 1759, 1760, 2311, 2309, 1760,
- 1760, 1452, 1452, 2308, 1477, 1294, 1294, 1294, 1294, 1294,
- 1294, 1295, 1295, 1295, 1296, 718, 2307, 453, 2306, 1769,
- 1298, 2305, 1754, 1761, 1763, 1299, 1299, 1299, 1299, 1299,
- 1299, 1300, 1300, 1300, 1301, 718, 2304, 1762, 1764, 2303,
- 1304, 2302, 626, 626, 626, 626, 626, 626, 626, 626,
+ 620, 451, 2319, 1794, 451, 452, 1749, 1475, 452, 2313,
+
+ 451, 620, 620, 626, 452, 1759, 1760, 2311, 2310, 1760,
+ 1760, 1452, 1452, 2309, 1477, 1294, 1294, 1294, 1294, 1294,
+ 1294, 1295, 1295, 1295, 1296, 718, 2308, 453, 2307, 1771,
+ 1298, 2306, 1754, 1761, 1763, 1299, 1299, 1299, 1299, 1299,
+ 1299, 1300, 1300, 1300, 1301, 718, 2305, 1762, 1764, 2304,
+ 1304, 2303, 626, 626, 626, 626, 626, 626, 626, 626,
626, 626, 626, 626, 626, 626, 626, 626, 626, 626,
- 626, 1775, 1780, 451, 451, 1775, 1781, 452, 452, 2301,
- 451, 626, 626, 632, 452, 1783, 1783, 451, 451, 1783,
- 1783, 452, 452, 2278, 1782, 1299, 1299, 1299, 1299, 1299,
+ 626, 1777, 1782, 451, 451, 1777, 1783, 452, 452, 2280,
+ 451, 626, 626, 632, 452, 1785, 1785, 451, 451, 1785,
+ 1785, 452, 452, 2279, 1784, 1299, 1299, 1299, 1299, 1299,
- 1299, 1300, 1300, 1300, 1301, 718, 2277, 1784, 1784, 1777,
- 1309, 1805, 1801, 1807, 1776, 1305, 1305, 1305, 1305, 1305,
- 1305, 1306, 1306, 1306, 1307, 718, 1836, 1146, 2274, 2912,
+ 1299, 1300, 1300, 1300, 1301, 718, 2276, 1786, 1786, 1779,
+ 1309, 1807, 1803, 1809, 1778, 1305, 1305, 1305, 1305, 1305,
+ 1305, 1306, 1306, 1306, 1307, 718, 1838, 1146, 2915, 2915,
1314, 1147, 632, 632, 632, 632, 632, 632, 632, 632,
632, 632, 632, 632, 632, 632, 632, 632, 632, 632,
- 632, 1785, 2912, 1856, 1510, 1785, 1779, 1857, 1511, 2262,
- 2261, 632, 632, 638, 1786, 1788, 451, 1796, 1786, 1788,
- 452, 1797, 2258, 1782, 1512, 1310, 1310, 1310, 1310, 1310,
- 1310, 1311, 1311, 1311, 1312, 718, 1787, 1789, 451, 1798,
- 1025, 2255, 452, 2250, 2236, 1315, 1315, 1315, 1315, 1315,
+ 632, 1787, 2264, 1858, 1510, 1787, 1781, 1859, 1511, 2263,
+ 2260, 632, 632, 638, 1788, 1790, 451, 1798, 1788, 1790,
+ 452, 1799, 2257, 1784, 1512, 1310, 1310, 1310, 1310, 1310,
+ 1310, 1311, 1311, 1311, 1312, 718, 1789, 1791, 451, 1800,
+ 1025, 2252, 452, 2238, 2236, 1315, 1315, 1315, 1315, 1315,
- 1315, 1316, 1316, 1316, 1317, 718, 2234, 451, 2233, 1793,
+ 1315, 1316, 1316, 1316, 1317, 718, 2235, 451, 2234, 1795,
1283, 452, 638, 638, 638, 638, 638, 638, 638, 638,
638, 638, 638, 638, 638, 638, 638, 638, 638, 638,
- 638, 1799, 451, 2232, 1520, 451, 452, 1802, 1521, 452,
- 2231, 638, 638, 644, 451, 451, 1511, 451, 452, 452,
- 1511, 452, 2230, 2227, 1522, 1026, 1026, 1026, 1026, 1026,
- 1026, 1027, 1027, 1027, 1028, 718, 1822, 453, 1804, 451,
- 1288, 1830, 1835, 452, 2226, 1284, 1284, 1284, 1284, 1284,
- 1284, 1285, 1285, 1285, 1286, 718, 2220, 882, 911, 2212,
- 1293, 2211, 644, 644, 644, 644, 644, 644, 644, 644,
+ 638, 1801, 451, 2233, 1520, 451, 452, 1804, 1521, 452,
+ 2232, 638, 638, 644, 451, 451, 1511, 451, 452, 452,
+ 1511, 452, 2229, 2228, 1522, 1026, 1026, 1026, 1026, 1026,
+ 1026, 1027, 1027, 1027, 1028, 718, 1824, 453, 1806, 451,
+ 1288, 1832, 1837, 452, 2222, 1284, 1284, 1284, 1284, 1284,
+ 1284, 1285, 1285, 1285, 1286, 718, 2214, 882, 911, 2213,
+ 1293, 2208, 644, 644, 644, 644, 644, 644, 644, 644,
644, 644, 644, 644, 644, 644, 644, 644, 644, 644,
- 644, 1521, 1834, 1545, 1875, 1521, 2206, 1545, 1876, 2203,
- 451, 644, 644, 650, 452, 451, 1206, 2201, 2200, 452,
- 1207, 1831, 453, 1974, 453, 1289, 1289, 1289, 1289, 1289,
- 1289, 1290, 1290, 1290, 1291, 718, 2189, 2186, 451, 451,
- 1025, 1806, 452, 452, 1826, 1294, 1294, 1294, 1294, 1294,
- 1294, 1295, 1295, 1295, 1296, 718, 2184, 451, 1847, 943,
+ 644, 1521, 1836, 1545, 1877, 1521, 2205, 1545, 1878, 2203,
+ 451, 644, 644, 650, 452, 451, 1206, 2202, 1976, 452,
+ 1207, 1833, 453, 2191, 453, 1289, 1289, 1289, 1289, 1289,
+ 1289, 1290, 1290, 1290, 1291, 718, 2188, 2186, 451, 451,
+ 1025, 1808, 452, 452, 1828, 1294, 1294, 1294, 1294, 1294,
+ 1294, 1295, 1295, 1295, 1296, 718, 2183, 451, 1849, 943,
1025, 452, 650, 650, 650, 650, 650, 650, 650, 650,
650, 650, 650, 650, 650, 650, 650, 650, 650, 650,
- 650, 2181, 1848, 2179, 451, 901, 1203, 1828, 452, 2168,
+ 650, 2181, 1850, 2170, 451, 901, 1203, 1830, 452, 2168,
1204, 650, 650, 656, 451, 451, 2166, 1551, 452, 452,
- 451, 1860, 2164, 2161, 452, 1026, 1026, 1026, 1026, 1026,
- 1026, 1027, 1027, 1027, 1028, 718, 2160, 933, 1552, 451,
- 1283, 1858, 2159, 452, 1845, 1026, 1026, 1026, 1026, 1026,
- 1026, 1027, 1027, 1027, 1028, 718, 2154, 2144, 2140, 1877,
+ 451, 1862, 2163, 2162, 452, 1026, 1026, 1026, 1026, 1026,
+ 1026, 1027, 1027, 1027, 1028, 718, 2161, 933, 1552, 451,
+ 1283, 1860, 2156, 452, 1847, 1026, 1026, 1026, 1026, 1026,
+ 1026, 1027, 1027, 1027, 1028, 718, 2146, 2142, 2138, 1879,
1288, 957, 656, 656, 656, 656, 656, 656, 656, 656,
656, 656, 656, 656, 656, 656, 656, 656, 656, 656,
- 656, 2136, 1878, 1568, 1570, 1841, 2131, 1569, 1570, 1841,
- 1209, 656, 656, 664, 1210, 1546, 1547, 1907, 451, 1547,
- 1547, 1908, 452, 2128, 2127, 1284, 1284, 1284, 1284, 1284,
-
- 1284, 1285, 1285, 1285, 1286, 718, 1842, 453, 1870, 1850,
- 1293, 2126, 1555, 1852, 1854, 1289, 1289, 1289, 1289, 1289,
- 1289, 1290, 1290, 1290, 1291, 718, 2125, 1853, 1855, 1865,
- 1283, 1879, 664, 664, 664, 664, 664, 664, 664, 664,
+ 656, 2133, 1880, 1568, 1570, 1843, 2130, 1569, 1570, 1843,
+ 1209, 656, 656, 664, 1210, 1546, 1547, 1909, 451, 1547,
+ 1547, 1910, 452, 2129, 2128, 1284, 1284, 1284, 1284, 1284,
+
+ 1284, 1285, 1285, 1285, 1286, 718, 1844, 453, 1872, 1852,
+ 1293, 2127, 1555, 1854, 1856, 1289, 1289, 1289, 1289, 1289,
+ 1289, 1290, 1290, 1290, 1291, 718, 2116, 1855, 1857, 1867,
+ 1283, 1881, 664, 664, 664, 664, 664, 664, 664, 664,
664, 664, 664, 664, 664, 664, 664, 664, 664, 664,
- 664, 1222, 1569, 1871, 1243, 1223, 1569, 2114, 1244, 2113,
- 451, 664, 664, 670, 452, 451, 2108, 451, 1580, 452,
+ 664, 1222, 1569, 1873, 1243, 1223, 1569, 2115, 1244, 2110,
+ 451, 664, 664, 670, 452, 451, 2109, 451, 1580, 452,
953, 452, 1581, 453, 954, 1294, 1294, 1294, 1294, 1294,
- 1294, 1295, 1295, 1295, 1296, 718, 1874, 1890, 1582, 1861,
- 1624, 2107, 1909, 2101, 1872, 1284, 1284, 1284, 1284, 1284,
+ 1294, 1295, 1295, 1295, 1296, 718, 1876, 1892, 1582, 1863,
+ 1624, 2103, 1911, 1861, 1874, 1284, 1284, 1284, 1284, 1284,
- 1284, 1285, 1285, 1285, 1286, 718, 1891, 1234, 1859, 2085,
+ 1284, 1285, 1285, 1285, 1286, 718, 1893, 1234, 2087, 2086,
1629, 1235, 670, 670, 670, 670, 670, 670, 670, 670,
670, 670, 670, 670, 670, 670, 670, 670, 670, 670,
- 670, 451, 2084, 1581, 451, 452, 1883, 1581, 452, 2081,
- 451, 670, 670, 676, 452, 451, 1885, 1258, 1915, 452,
- 2078, 1259, 1915, 1906, 453, 1625, 1625, 1625, 1625, 1625,
- 1625, 1626, 1626, 1626, 1627, 718, 2076, 451, 1884, 1595,
- 1634, 452, 2075, 1596, 1886, 1630, 1630, 1630, 1630, 1630,
- 1630, 1631, 1631, 1631, 1632, 718, 2073, 451, 2072, 2071,
+ 670, 451, 2083, 1581, 451, 452, 1885, 1581, 452, 2080,
+ 451, 670, 670, 676, 452, 451, 1887, 1258, 1917, 452,
+ 2078, 1259, 1917, 1908, 453, 1625, 1625, 1625, 1625, 1625,
+ 1625, 1626, 1626, 1626, 1627, 718, 2077, 451, 1886, 1595,
+ 1634, 452, 2075, 1596, 1888, 1630, 1630, 1630, 1630, 1630,
+ 1630, 1631, 1631, 1631, 1632, 718, 2074, 451, 2073, 2059,
1304, 452, 676, 676, 676, 676, 676, 676, 676, 676,
676, 676, 676, 676, 676, 676, 676, 676, 676, 676,
- 676, 2057, 2052, 1255, 451, 1887, 1913, 1256, 452, 2049,
- 451, 676, 676, 682, 452, 2048, 1596, 2047, 2046, 1598,
- 1596, 1592, 1592, 1599, 2045, 1635, 1635, 1635, 1635, 1635,
- 1635, 1636, 1636, 1636, 1637, 718, 1912, 453, 1911, 1922,
- 1639, 2044, 2042, 1923, 2038, 1305, 1305, 1305, 1305, 1305,
- 1305, 1306, 1306, 1306, 1307, 718, 1919, 451, 2037, 1370,
+ 676, 2054, 2051, 1255, 451, 1889, 1915, 1256, 452, 2050,
+ 451, 676, 676, 682, 452, 2049, 1596, 2048, 2047, 1598,
+ 1596, 1592, 1592, 1599, 2046, 1635, 1635, 1635, 1635, 1635,
+ 1635, 1636, 1636, 1636, 1637, 718, 1914, 453, 1913, 1924,
+ 1639, 2044, 2040, 1925, 2039, 1305, 1305, 1305, 1305, 1305,
+ 1305, 1306, 1306, 1306, 1307, 718, 1921, 451, 2032, 1370,
1644, 452, 682, 682, 682, 682, 682, 682, 682, 682,
682, 682, 682, 682, 682, 682, 682, 682, 682, 682,
- 682, 1916, 1599, 451, 988, 1917, 1599, 452, 989, 2030,
+ 682, 1918, 1599, 451, 988, 1919, 1599, 452, 989, 2009,
- 2007, 682, 682, 689, 1921, 1924, 1918, 451, 451, 1923,
- 1925, 452, 452, 453, 1926, 1640, 1640, 1640, 1640, 1640,
- 1640, 1641, 1641, 1641, 1642, 718, 1375, 1382, 2004, 1920,
- 1649, 1998, 1927, 1997, 1988, 1645, 1645, 1645, 1645, 1645,
- 1645, 1646, 1646, 1646, 1647, 451, 1932, 1691, 1982, 452,
+ 2006, 682, 682, 689, 1923, 1926, 1920, 451, 451, 1925,
+ 1927, 452, 452, 453, 1928, 1640, 1640, 1640, 1640, 1640,
+ 1640, 1641, 1641, 1641, 1642, 718, 1375, 1382, 2000, 1922,
+ 1649, 1999, 1929, 1990, 1984, 1645, 1645, 1645, 1645, 1645,
+ 1645, 1646, 1646, 1646, 1647, 451, 1934, 1691, 1983, 452,
1269, 1691, 689, 689, 689, 689, 689, 689, 689, 689,
689, 689, 689, 689, 689, 689, 689, 689, 689, 689,
- 689, 1741, 1613, 991, 1692, 1436, 1606, 992, 1693, 1981,
- 1692, 689, 689, 695, 1693, 1720, 1928, 1510, 1935, 1721,
- 1929, 1511, 1735, 1608, 1614, 1650, 1650, 1650, 1650, 1650,
+ 689, 1741, 1613, 991, 1692, 1436, 1606, 992, 1693, 1976,
+ 1692, 689, 689, 695, 1693, 1720, 1930, 1510, 1937, 1721,
+ 1931, 1511, 1735, 1608, 1614, 1650, 1650, 1650, 1650, 1650,
1650, 1651, 1651, 1651, 1652, 738, 1742, 1512, 450, 1613,
- 1720, 1974, 1937, 1606, 1721, 738, 1993, 1743, 1825, 1767,
- 1993, 1775, 1775, 1767, 2016, 1775, 1775, 1972, 2017, 1965,
+ 1720, 1974, 1939, 1606, 1721, 738, 1995, 1743, 1827, 1767,
+ 1995, 1777, 1777, 1767, 2018, 1777, 1777, 1967, 2019, 1951,
1608, 1614, 695, 695, 695, 695, 695, 695, 695, 695,
695, 695, 695, 695, 695, 695, 695, 695, 695, 695,
- 695, 1949, 1930, 1790, 1790, 1044, 1931, 1791, 1791, 1936,
- 2053, 695, 695, 701, 2053, 1049, 1023, 1578, 2061, 2064,
- 1934, 1579, 2061, 2064, 2912, 1045, 1045, 1045, 1045, 1045,
+ 695, 1938, 1932, 1792, 1792, 1044, 1933, 1793, 1793, 1936,
+ 2055, 695, 695, 701, 2055, 1049, 1023, 1578, 2063, 2066,
+ 1935, 1579, 2063, 2066, 2915, 1045, 1045, 1045, 1045, 1045,
1045, 1046, 1046, 1046, 1047, 1050, 1050, 1050, 1050, 1050,
- 1050, 1051, 1051, 1051, 1052, 738, 1892, 1893, 1933, 1841,
+ 1050, 1051, 1051, 1051, 1052, 738, 1894, 1895, 1591, 1843,
- 1894, 2062, 1895, 1841, 1896, 738, 1851, 1591, 1914, 1897,
- 1851, 1898, 701, 701, 701, 701, 701, 701, 701, 701,
+ 1896, 2064, 1897, 1843, 1898, 738, 1853, 1916, 1912, 1899,
+ 1853, 1900, 701, 701, 701, 701, 701, 701, 701, 701,
701, 701, 701, 701, 701, 701, 701, 701, 701, 701,
701, 719, 719, 719, 719, 719, 719, 720, 720, 720,
- 721, 701, 701, 707, 2069, 1054, 2134, 1915, 2069, 1930,
- 2134, 1915, 1910, 1931, 1930, 1044, 1841, 1889, 1931, 2153,
- 1841, 738, 1888, 2153, 2070, 1055, 1055, 1055, 1055, 1055,
+ 721, 701, 701, 707, 2071, 1054, 2136, 1917, 2071, 1932,
+ 2136, 1917, 1891, 1933, 1932, 1044, 1843, 1890, 1933, 2155,
+ 1843, 738, 1884, 2155, 2072, 1055, 1055, 1055, 1055, 1055,
1055, 1056, 1056, 1056, 1057, 1045, 1045, 1045, 1045, 1045,
- 1045, 1046, 1046, 1046, 1047, 2074, 1993, 2098, 1882, 1881,
- 1993, 2098, 707, 707, 707, 707, 707, 707, 707, 707,
+ 1045, 1046, 1046, 1046, 1047, 2076, 1995, 2100, 1883, 1882,
+ 1995, 2100, 707, 707, 707, 707, 707, 707, 707, 707,
707, 707, 707, 707, 707, 707, 707, 707, 707, 707,
- 707, 1044, 2099, 2016, 1084, 2016, 2218, 2017, 1085, 2017,
- 2218, 707, 707, 744, 1947, 1722, 1705, 1880, 1690, 1412,
+ 707, 1044, 2101, 2018, 1084, 2018, 2220, 2019, 1085, 2019,
+ 2220, 707, 707, 744, 1949, 1722, 1705, 1875, 1690, 1412,
1705, 1045, 1045, 1045, 1045, 1045, 1045, 1046, 1046, 1046,
- 1047, 738, 450, 1873, 2053, 1375, 1414, 453, 2053, 2061,
- 450, 738, 1859, 2061, 1859, 2061, 1945, 1849, 450, 2061,
- 745, 746, 746, 1959, 747, 748, 749, 750, 750, 750,
+ 1047, 738, 450, 1861, 2055, 1375, 1414, 453, 2055, 2063,
+ 450, 738, 1861, 2063, 1851, 2063, 1947, 1848, 450, 2063,
+ 745, 746, 746, 1961, 747, 748, 749, 750, 750, 750,
750, 750, 750, 750, 750, 750, 750, 750, 750, 750,
750, 750, 750, 750, 750, 750, 750, 750, 753, 1697,
- 2064, 1320, 2243, 1697, 2064, 1691, 2265, 1846, 2271, 1691,
+ 2066, 1320, 2245, 1697, 2066, 1691, 2267, 1846, 2273, 1691,
- 2265, 1325, 2098, 2134, 2318, 2218, 738, 2134, 2318, 2218,
+ 2267, 1325, 2100, 2136, 2320, 2220, 738, 2136, 2320, 2220,
453, 1321, 1321, 1321, 1321, 1321, 1321, 1322, 1322, 1322,
1323, 1326, 1326, 1326, 1326, 1326, 1326, 1327, 1327, 1327,
- 1328, 1951, 1844, 1843, 1840, 1839, 738, 753, 753, 753,
+ 1328, 1953, 1845, 1842, 1841, 1840, 738, 753, 753, 753,
753, 753, 753, 753, 753, 753, 753, 753, 753, 753,
- 753, 753, 753, 753, 753, 753, 1330, 1841, 2268, 2098,
- 2384, 1841, 2096, 2098, 2384, 1838, 753, 753, 759, 1042,
- 2053, 1837, 1829, 1827, 2053, 2385, 1331, 1331, 1331, 1331,
- 1331, 1331, 1332, 1332, 1332, 1333, 1335, 738, 1821, 739,
+ 753, 753, 753, 753, 753, 753, 1330, 1843, 2270, 2100,
+ 2381, 1843, 2098, 2100, 2381, 1839, 753, 753, 759, 1042,
+ 2055, 1831, 1829, 1823, 2055, 2382, 1331, 1331, 1331, 1331,
+ 1331, 1331, 1332, 1332, 1332, 1333, 1335, 738, 1822, 739,
739, 739, 739, 739, 739, 740, 740, 740, 741, 738,
- 2272, 1841, 1820, 1862, 2254, 1841, 1336, 1336, 1336, 1336,
+ 2274, 1843, 1821, 1864, 2256, 1843, 1336, 1336, 1336, 1336,
1336, 1336, 1337, 1337, 1337, 1338, 1023, 759, 759, 759,
759, 759, 759, 759, 759, 759, 759, 759, 759, 759,
- 759, 759, 759, 759, 759, 759, 2069, 1335, 2265, 2061,
- 2069, 2271, 2265, 2061, 1819, 2098, 759, 759, 765, 1341,
- 2268, 2287, 1816, 1815, 2096, 2287, 2070, 1336, 1336, 1336,
- 1336, 1336, 1336, 1337, 1337, 1337, 1338, 738, 2388, 1342,
+ 759, 759, 759, 759, 759, 759, 2071, 1335, 2387, 2063,
+ 2071, 2267, 2387, 2063, 1818, 2267, 759, 759, 765, 1341,
+ 2270, 2273, 1817, 2388, 2098, 2100, 2072, 1336, 1336, 1336,
+ 1336, 1336, 1336, 1337, 1337, 1337, 1338, 738, 2391, 1342,
1342, 1342, 1342, 1342, 1342, 1343, 1343, 1343, 1344, 738,
- 1814, 719, 719, 719, 719, 719, 719, 720, 720, 720,
- 721, 718, 1813, 1812, 1811, 1862, 1025, 765, 765, 765,
+ 1816, 719, 719, 719, 719, 719, 719, 720, 720, 720,
+ 721, 718, 1815, 1814, 1813, 1864, 1025, 765, 765, 765,
765, 765, 765, 765, 765, 765, 765, 765, 765, 765,
- 765, 765, 765, 765, 765, 765, 2251, 1346, 2289, 2318,
- 2251, 2445, 2289, 2318, 1810, 2445, 765, 765, 771, 1351,
- 2451, 2471, 1809, 1808, 2451, 2471, 1755, 1347, 1347, 1347,
- 1347, 1347, 1347, 1348, 1348, 1348, 1349, 738, 1800, 1352,
+ 765, 765, 765, 765, 765, 765, 2253, 1346, 2289, 2291,
+ 2253, 2320, 2289, 2291, 1812, 2320, 765, 765, 771, 1351,
+ 2448, 2454, 1811, 1810, 2448, 2454, 1755, 1347, 1347, 1347,
+ 1347, 1347, 1347, 1348, 1348, 1348, 1349, 738, 1802, 1352,
1352, 1352, 1352, 1352, 1352, 1353, 1353, 1353, 1354, 738,
- 1794, 1026, 1026, 1026, 1026, 1026, 1026, 1027, 1027, 1027,
- 1028, 718, 1778, 1770, 1750, 1718, 1030, 771, 771, 771,
+ 1796, 1026, 1026, 1026, 1026, 1026, 1026, 1027, 1027, 1027,
+ 1028, 718, 1780, 1772, 1750, 1718, 1030, 771, 771, 771,
771, 771, 771, 771, 771, 771, 771, 771, 771, 771,
- 771, 771, 771, 771, 771, 771, 2256, 1044, 2507, 2287,
+ 771, 771, 771, 771, 771, 771, 2258, 1044, 2474, 2510,
- 2256, 2535, 2507, 2287, 1695, 2535, 771, 771, 779, 1320,
- 2445, 2451, 1683, 2912, 2445, 2451, 1906, 1045, 1045, 1045,
- 1045, 1045, 1045, 1046, 1046, 1046, 1047, 738, 2512, 1321,
+ 2258, 2538, 2474, 2510, 1695, 2538, 771, 771, 779, 1320,
+ 2448, 2289, 1683, 2915, 2448, 2289, 1908, 1045, 1045, 1045,
+ 1045, 1045, 1045, 1046, 1046, 1046, 1047, 738, 2915, 1321,
1321, 1321, 1321, 1321, 1321, 1322, 1322, 1322, 1323, 738,
- 2912, 1031, 1031, 1031, 1031, 1031, 1031, 1032, 1032, 1032,
- 1033, 718, 2912, 2912, 2912, 2912, 1035, 779, 779, 779,
+ 2515, 1031, 1031, 1031, 1031, 1031, 1031, 1032, 1032, 1032,
+ 1033, 718, 2915, 2915, 2915, 2915, 1035, 779, 779, 779,
779, 779, 779, 779, 779, 779, 779, 779, 779, 779,
- 779, 779, 779, 779, 779, 779, 2327, 1325, 2471, 2567,
- 2327, 2572, 2471, 2567, 2912, 2573, 779, 779, 787, 1330,
- 2574, 2061, 2575, 2912, 1621, 2061, 1261, 1326, 1326, 1326,
+ 779, 779, 779, 779, 779, 779, 2329, 1325, 2454, 2474,
+ 2329, 2570, 2454, 2474, 2915, 2570, 779, 779, 787, 1330,
+ 2063, 2510, 2915, 1621, 2063, 2510, 1261, 1326, 1326, 1326,
1326, 1326, 1326, 1327, 1327, 1327, 1328, 738, 1618, 1331,
1331, 1331, 1331, 1331, 1331, 1332, 1332, 1332, 1333, 738,
1617, 1036, 1036, 1036, 1036, 1036, 1036, 1037, 1037, 1037,
1038, 718, 1591, 1584, 450, 1565, 1025, 787, 787, 787,
787, 787, 787, 787, 787, 787, 787, 787, 787, 787,
- 787, 787, 787, 787, 787, 787, 2335, 1044, 2507, 2289,
- 2336, 2567, 2507, 2289, 1564, 2567, 787, 787, 795, 1044,
- 2287, 2702, 1563, 450, 2287, 2702, 2169, 1045, 1045, 1045,
+ 787, 787, 787, 787, 787, 787, 2337, 1044, 2291, 2570,
+ 2338, 2575, 2291, 2570, 1564, 2576, 787, 787, 795, 1044,
+ 2577, 2289, 2578, 1563, 450, 2289, 2171, 1045, 1045, 1045,
1045, 1045, 1045, 1046, 1046, 1046, 1047, 738, 1535, 1045,
1045, 1045, 1045, 1045, 1045, 1046, 1046, 1046, 1047, 738,
1534, 1026, 1026, 1026, 1026, 1026, 1026, 1027, 1027, 1027,
1028, 718, 1533, 1532, 1531, 1530, 1025, 795, 795, 795,
795, 795, 795, 795, 795, 795, 795, 795, 795, 795,
- 795, 795, 795, 795, 795, 795, 2251, 1320, 2722, 2098,
- 2251, 2762, 2722, 2098, 1529, 2762, 795, 795, 801, 1325,
- 2098, 2785, 1518, 1508, 2098, 2785, 1755, 1321, 1321, 1321,
- 1321, 1321, 1321, 1322, 1322, 1322, 1323, 738, 2726, 1326,
+ 795, 795, 795, 795, 795, 795, 2253, 1320, 2705, 2725,
+ 2253, 2765, 2705, 2725, 1529, 2765, 795, 795, 801, 1325,
+ 2100, 2100, 1518, 1508, 2100, 2100, 1755, 1321, 1321, 1321,
+ 1321, 1321, 1321, 1322, 1322, 1322, 1323, 738, 1507, 1326,
1326, 1326, 1326, 1326, 1326, 1327, 1327, 1327, 1328, 738,
- 1507, 1026, 1026, 1026, 1026, 1026, 1026, 1027, 1027, 1027,
- 1028, 718, 2784, 1503, 1502, 1499, 1283, 801, 801, 801,
+ 2729, 1026, 1026, 1026, 1026, 1026, 1026, 1027, 1027, 1027,
+ 1028, 718, 2787, 1503, 1502, 1499, 1283, 801, 801, 801,
801, 801, 801, 801, 801, 801, 801, 801, 801, 801,
- 801, 801, 801, 801, 801, 801, 2252, 1330, 2807, 2826,
- 2252, 2762, 2807, 2826, 1496, 2762, 801, 801, 807, 1320,
- 2785, 2847, 1482, 1462, 2785, 2847, 1822, 1331, 1331, 1331,
+ 801, 801, 801, 801, 801, 801, 2254, 1330, 2788, 2829,
+ 2254, 2765, 2788, 2829, 1496, 2765, 801, 801, 807, 1320,
+ 2810, 2788, 1482, 1462, 2810, 2788, 1824, 1331, 1331, 1331,
1331, 1331, 1331, 1332, 1332, 1332, 1333, 738, 450, 1321,
1321, 1321, 1321, 1321, 1321, 1322, 1322, 1322, 1323, 738,
- 2808, 1284, 1284, 1284, 1284, 1284, 1284, 1285, 1285, 1285,
- 1286, 718, 450, 1386, 1355, 2912, 1288, 807, 807, 807,
+ 450, 1284, 1284, 1284, 1284, 1284, 1284, 1285, 1285, 1285,
+ 1286, 718, 2811, 1386, 1355, 2915, 1288, 807, 807, 807,
807, 807, 807, 807, 807, 807, 807, 807, 807, 807,
- 807, 807, 807, 807, 807, 807, 2256, 1654, 2807, 2850,
+ 807, 807, 807, 807, 807, 807, 2258, 1654, 2850, 2810,
- 2256, 2826, 2807, 2850, 2912, 2826, 807, 807, 817, 1659,
- 2855, 2856, 450, 450, 2855, 2856, 1906, 1655, 1655, 1655,
+ 2258, 2853, 2850, 2810, 2915, 2853, 807, 807, 817, 1659,
+ 2829, 2858, 450, 450, 2829, 2858, 1908, 1655, 1655, 1655,
1655, 1655, 1655, 1656, 1656, 1656, 1657, 738, 450, 1660,
1660, 1660, 1660, 1660, 1660, 1661, 1661, 1661, 1662, 738,
450, 1289, 1289, 1289, 1289, 1289, 1289, 1290, 1290, 1290,
1291, 718, 450, 450, 450, 450, 1293, 817, 817, 817,
817, 817, 817, 817, 817, 817, 817, 817, 817, 817,
- 817, 817, 817, 817, 817, 817, 2327, 1664, 2857, 2847,
- 2327, 2850, 2857, 2847, 450, 2850, 817, 817, 823, 1341,
- 2879, 2879, 450, 1112, 2879, 2879, 1261, 1665, 1665, 1665,
+ 817, 817, 817, 817, 817, 817, 2329, 1664, 2859, 2860,
+ 2329, 2850, 2859, 2860, 450, 2850, 817, 817, 823, 1341,
+ 2853, 2882, 450, 1112, 2853, 2882, 1261, 1665, 1665, 1665,
1665, 1665, 1665, 1666, 1666, 1666, 1667, 738, 450, 1342,
1342, 1342, 1342, 1342, 1342, 1343, 1343, 1343, 1344, 738,
1106, 1294, 1294, 1294, 1294, 1294, 1294, 1295, 1295, 1295,
1296, 718, 450, 816, 1068, 1066, 1298, 823, 823, 823,
823, 823, 823, 823, 823, 823, 823, 823, 823, 823,
- 823, 823, 823, 823, 823, 823, 2335, 1669, 2897, 2897,
- 2336, 2289, 2897, 2897, 1041, 2289, 823, 823, 829, 1674,
- 563, 563, 987, 986, 983, 970, 2169, 1670, 1670, 1670,
- 1670, 1670, 1670, 1671, 1671, 1671, 1672, 738, 964, 1675,
- 1675, 1675, 1675, 1675, 1675, 1676, 1676, 1676, 1677, 2417,
-
- 961, 1299, 1299, 1299, 1299, 1299, 1299, 1300, 1300, 1300,
- 1301, 2287, 2289, 960, 952, 2287, 2289, 829, 829, 829,
+ 823, 823, 823, 823, 823, 823, 2337, 1669, 2882, 2900,
+ 2338, 2291, 2882, 2900, 1041, 2291, 823, 823, 829, 1674,
+ 2900, 563, 563, 987, 2900, 986, 2171, 1670, 1670, 1670,
+ 1670, 1670, 1670, 1671, 1671, 1671, 1672, 738, 983, 1675,
+ 1675, 1675, 1675, 1675, 1675, 1676, 1676, 1676, 1677, 2420,
+
+ 970, 1299, 1299, 1299, 1299, 1299, 1299, 1300, 1300, 1300,
+ 1301, 2289, 2291, 964, 961, 2289, 2291, 829, 829, 829,
829, 829, 829, 829, 829, 829, 829, 829, 829, 829,
- 829, 829, 829, 829, 829, 829, 2098, 1679, 2290, 2287,
- 2098, 951, 948, 2287, 2288, 937, 829, 829, 838, 1579,
- 1699, 2098, 1702, 1579, 1699, 2098, 1702, 1680, 1680, 1680,
- 1680, 1680, 1680, 1681, 1681, 1681, 1682, 718, 936, 2407,
- 453, 453, 1298, 453, 935, 927, 2416, 2593, 1899, 1900,
- 718, 916, 1901, 915, 1902, 1304, 1903, 906, 905, 902,
- 718, 1904, 1953, 1905, 1956, 1309, 1042, 838, 838, 838,
+ 829, 829, 829, 829, 829, 829, 2100, 1679, 2292, 2289,
+ 2100, 960, 952, 2289, 2290, 951, 829, 829, 838, 1579,
+ 1699, 2100, 1702, 1579, 1699, 2100, 1702, 1680, 1680, 1680,
+ 1680, 1680, 1680, 1681, 1681, 1681, 1682, 718, 948, 2410,
+ 453, 453, 1298, 453, 937, 936, 2419, 2596, 1901, 1902,
+ 718, 935, 1903, 927, 1904, 1304, 1905, 916, 915, 906,
+ 718, 1906, 1955, 1907, 1958, 1309, 1042, 838, 838, 838,
838, 838, 838, 838, 838, 838, 838, 838, 838, 838,
838, 838, 838, 838, 838, 838, 739, 739, 739, 739,
- 739, 739, 740, 740, 740, 741, 838, 838, 844, 2252,
- 2384, 899, 896, 2252, 2384, 895, 894, 1299, 1299, 1299,
- 1299, 1299, 1299, 1300, 1300, 1300, 1301, 1593, 1593, 1822,
+ 739, 739, 740, 740, 740, 741, 838, 838, 844, 2254,
+ 2381, 905, 902, 2254, 2381, 899, 896, 1299, 1299, 1299,
+ 1299, 1299, 1299, 1300, 1300, 1300, 1301, 1453, 1453, 1824,
1305, 1305, 1305, 1305, 1305, 1305, 1306, 1306, 1306, 1307,
1310, 1310, 1310, 1310, 1310, 1310, 1311, 1311, 1311, 1312,
- 718, 507, 892, 886, 885, 1314, 884, 844, 844, 844,
+ 718, 895, 894, 507, 892, 1314, 886, 844, 844, 844,
844, 844, 844, 844, 844, 844, 844, 844, 844, 844,
- 844, 844, 844, 844, 844, 844, 1708, 452, 877, 1714,
+ 844, 844, 844, 844, 844, 844, 1708, 452, 885, 1714,
- 1708, 1724, 874, 1714, 871, 1727, 844, 844, 850, 1727,
- 1757, 1760, 2098, 868, 1757, 1760, 2098, 453, 453, 718,
- 453, 2289, 867, 2061, 1025, 2289, 453, 2061, 864, 718,
- 549, 453, 453, 1962, 1025, 544, 1968, 2504, 519, 1763,
+ 1708, 1724, 884, 1714, 877, 1727, 844, 844, 850, 1727,
+ 1757, 1760, 2100, 874, 1757, 1760, 2100, 453, 453, 718,
+ 453, 2291, 871, 2063, 1025, 2291, 453, 2063, 868, 718,
+ 867, 453, 453, 1964, 1025, 864, 1970, 2507, 549, 1763,
1315, 1315, 1315, 1315, 1315, 1315, 1316, 1316, 1316, 1317,
- 718, 2489, 513, 1764, 498, 1283, 2513, 850, 850, 850,
+ 718, 2492, 544, 1764, 519, 1283, 2516, 850, 850, 850,
850, 850, 850, 850, 850, 850, 850, 850, 850, 850,
- 850, 850, 850, 850, 850, 850, 487, 816, 794, 786,
- 737, 563, 563, 556, 549, 544, 850, 850, 856, 1026,
+ 850, 850, 850, 850, 850, 850, 513, 498, 487, 816,
+ 794, 786, 737, 563, 563, 556, 850, 850, 856, 1026,
1026, 1026, 1026, 1026, 1026, 1027, 1027, 1027, 1028, 1026,
1026, 1026, 1026, 1026, 1026, 1027, 1027, 1027, 1028, 718,
- 519, 2000, 1773, 1781, 1288, 2000, 1773, 1781, 513, 498,
+ 549, 2002, 1775, 1769, 1288, 2002, 1775, 1769, 544, 519,
1284, 1284, 1284, 1284, 1284, 1284, 1285, 1285, 1285, 1286,
- 487, 2912, 453, 453, 453, 1784, 2912, 856, 856, 856,
+ 1453, 1453, 453, 453, 453, 513, 498, 856, 856, 856,
856, 856, 856, 856, 856, 856, 856, 856, 856, 856,
- 856, 856, 856, 856, 856, 856, 718, 2912, 2098, 2855,
- 2912, 1293, 2098, 2855, 2912, 2912, 856, 856, 451, 1696,
- 2912, 1746, 452, 1697, 451, 1746, 2773, 2773, 452, 1289,
+ 856, 856, 856, 856, 856, 856, 718, 487, 2100, 2387,
+ 2915, 1293, 2100, 2387, 2915, 2915, 856, 856, 451, 1696,
+ 2915, 1746, 452, 1697, 451, 1746, 1593, 1593, 452, 1289,
1289, 1289, 1289, 1289, 1289, 1290, 1290, 1290, 1291, 453,
- 718, 2912, 453, 2660, 1783, 1283, 2912, 450, 1783, 1753,
+ 718, 2915, 453, 2663, 1783, 1283, 2915, 450, 1783, 1753,
- 1792, 718, 450, 1753, 1475, 450, 1283, 1081, 1985, 2912,
- 450, 1950, 1948, 450, 2912, 1785, 1784, 718, 450, 1785,
- 453, 1477, 1624, 2912, 2912, 2912, 1294, 1294, 1294, 1294,
- 1294, 1294, 1295, 1295, 1295, 1296, 1990, 1782, 2912, 454,
+ 1794, 718, 450, 1753, 1475, 450, 1283, 1081, 1987, 2915,
+ 450, 1952, 1950, 450, 2915, 453, 1786, 718, 450, 2915,
+ 453, 1477, 1624, 2915, 2915, 2915, 1294, 1294, 1294, 1294,
+ 1294, 1294, 1295, 1295, 1295, 1296, 1992, 2915, 2915, 454,
454, 454, 454, 454, 454, 455, 455, 455, 456, 451,
- 2912, 2912, 2912, 452, 2287, 2912, 2912, 2912, 2287, 2912,
+ 2915, 2915, 2915, 452, 2289, 2915, 2915, 2915, 2289, 2915,
1284, 1284, 1284, 1284, 1284, 1284, 1285, 1285, 1285, 1286,
453, 1284, 1284, 1284, 1284, 1284, 1284, 1285, 1285, 1285,
- 1286, 2020, 2596, 2912, 2912, 2020, 928, 1625, 1625, 1625,
- 1625, 1625, 1625, 1626, 1626, 1626, 1627, 2023, 2011, 2912,
+ 1286, 1785, 2599, 2915, 2915, 1785, 928, 1625, 1625, 1625,
+ 1625, 1625, 1625, 1626, 1626, 1626, 1627, 1787, 2541, 2915,
- 2856, 2024, 2912, 2021, 2856, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2775, 2775, 994,
+ 2915, 1787, 2541, 1786, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 1723, 1784,
454, 454, 454, 454, 454, 454, 455, 455, 455, 456,
1132, 1133, 1132, 1132, 1132, 1134, 1135, 1132, 1135, 1132,
1132, 1132, 1132, 1132, 1132, 1132, 1132, 1132, 1132, 1132,
@@ -3037,16 +3037,16 @@ static yyconst flex_int16_t yy_nxt[27006] =
973, 973, 973, 973, 973, 973, 973, 976, 976, 976,
976, 976, 976, 976, 976, 976, 976, 976, 976, 976,
976, 976, 976, 976, 976, 976, 980, 980, 980, 980,
- 980, 980, 981, 981, 981, 982, 976, 976, 2912, 1686,
+ 980, 980, 981, 981, 981, 982, 976, 976, 2915, 1686,
- 2912, 451, 450, 1686, 450, 452, 2039, 2912, 2043, 2912,
- 2039, 718, 2043, 1824, 2912, 2857, 1629, 1824, 1797, 2857,
- 453, 718, 1797, 2912, 2912, 2912, 1634, 1942, 1409, 450,
- 1472, 718, 2777, 2777, 453, 1943, 1304, 450, 2912, 453,
- 2026, 718, 2912, 1944, 2912, 450, 1304, 2912, 1954, 450,
+ 2915, 451, 450, 1686, 450, 452, 2022, 2915, 2025, 2013,
+ 2022, 718, 2026, 1826, 2915, 2858, 1629, 1826, 1799, 2858,
+ 453, 718, 1799, 2915, 2915, 2915, 1634, 1944, 2023, 450,
+ 994, 718, 2776, 2776, 453, 1945, 1304, 450, 2915, 453,
+ 2028, 718, 2915, 1946, 2915, 450, 1304, 2915, 1956, 450,
450, 450, 450, 450, 450, 450, 450, 450, 450, 450,
- 450, 450, 450, 450, 450, 450, 450, 450, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 450, 450,
+ 450, 450, 450, 450, 450, 450, 450, 450, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 450, 450,
563, 1630, 1630, 1630, 1630, 1630, 1630, 1631, 1631, 1631,
1632, 1635, 1635, 1635, 1635, 1635, 1635, 1636, 1636, 1636,
@@ -3054,23 +3054,23 @@ static yyconst flex_int16_t yy_nxt[27006] =
1307, 1305, 1305, 1305, 1305, 1305, 1305, 1306, 1306, 1306,
1307, 560, 560, 560, 560, 560, 560, 560, 560, 560,
560, 560, 560, 560, 560, 560, 560, 560, 560, 560,
- 451, 451, 1698, 2040, 452, 452, 1699, 2040, 2912, 2912,
- 1833, 2050, 2912, 2912, 1833, 2050, 1946, 718, 2912, 1701,
- 2912, 453, 1639, 1702, 1701, 1454, 2912, 2912, 1702, 450,
- 450, 453, 2041, 1594, 2912, 2087, 2912, 450, 450, 2087,
- 2051, 2912, 2912, 718, 1952, 450, 450, 450, 1644, 2912,
- 2538, 2912, 450, 718, 2538, 450, 453, 2089, 1649, 2912,
-
- 450, 1955, 1741, 450, 2912, 2092, 1436, 1957, 450, 1860,
- 1723, 454, 454, 454, 454, 454, 454, 455, 455, 455,
- 456, 451, 2912, 1735, 2912, 452, 2093, 1640, 1640, 1640,
- 1640, 1640, 1640, 1641, 1641, 1641, 1642, 1742, 2912, 2912,
- 2912, 2912, 453, 2912, 2912, 1704, 2912, 1704, 1743, 1705,
- 2912, 1705, 1358, 1645, 1645, 1645, 1645, 1645, 1645, 1646,
+ 451, 451, 1698, 2041, 452, 452, 1699, 2041, 2915, 2045,
+ 2042, 2052, 2915, 2045, 2042, 2052, 1948, 718, 2915, 1701,
+ 2915, 453, 1639, 1702, 1701, 1409, 2915, 2915, 1702, 450,
+ 450, 1472, 1454, 1594, 2915, 2915, 2915, 450, 450, 2043,
+ 2053, 2915, 2915, 718, 1954, 450, 450, 450, 1644, 2915,
+ 2586, 2915, 450, 718, 2586, 450, 2915, 2915, 1649, 2915,
+
+ 450, 1957, 1741, 450, 2915, 1835, 1436, 1959, 450, 1835,
+ 2336, 454, 454, 454, 454, 454, 454, 455, 455, 455,
+ 456, 451, 2915, 1735, 2915, 452, 453, 1640, 1640, 1640,
+ 1640, 1640, 1640, 1641, 1641, 1641, 1642, 1742, 2915, 2915,
+ 2915, 2915, 453, 2915, 2915, 1704, 2915, 1704, 1743, 1705,
+ 2915, 1705, 1358, 1645, 1645, 1645, 1645, 1645, 1645, 1646,
1646, 1646, 1647, 1650, 1650, 1650, 1650, 1650, 1650, 1651,
- 1651, 1651, 1652, 450, 2912, 450, 2912, 2912, 2912, 2912,
- 2912, 450, 1958, 450, 2912, 2912, 2912, 2912, 2912, 450,
- 1960, 450, 454, 454, 454, 454, 454, 454, 455, 455,
+ 1651, 1651, 1652, 450, 2915, 450, 2915, 2915, 2915, 2915,
+ 2915, 450, 1960, 450, 2915, 2915, 2915, 2915, 2915, 450,
+ 1962, 450, 454, 454, 454, 454, 454, 454, 455, 455,
455, 456, 1360, 1361, 1360, 1360, 1360, 1085, 1362, 1360,
1362, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360,
@@ -3118,16 +3118,16 @@ static yyconst flex_int16_t yy_nxt[27006] =
1421, 1421, 1421, 1421, 1421, 1421, 1421, 1421, 1421, 1421,
1424, 1424, 1424, 1424, 1424, 1424, 1425, 1425, 1425, 1426,
- 1421, 1421, 2912, 1707, 2912, 1707, 450, 1708, 450, 1708,
- 2912, 1876, 2912, 2912, 1908, 1876, 1922, 2583, 1908, 2912,
- 1923, 2583, 2912, 1924, 2912, 1917, 2912, 1923, 2912, 1917,
- 2912, 450, 453, 450, 1963, 453, 2912, 2334, 1376, 450,
- 1961, 450, 1593, 1593, 1375, 1382, 453, 450, 2104, 450,
- 2912, 2912, 2912, 450, 450, 450, 450, 450, 450, 450,
+ 1421, 1421, 2915, 1707, 2915, 1707, 450, 1708, 450, 1708,
+ 2915, 2094, 2915, 2915, 2089, 1862, 1910, 2153, 2089, 2915,
+ 1910, 1931, 1924, 2915, 1949, 1919, 1925, 2915, 1690, 1919,
+ 2915, 450, 2095, 450, 1965, 453, 2091, 453, 1608, 450,
+ 1963, 450, 1593, 1593, 1376, 1375, 453, 450, 2915, 450,
+ 2915, 2915, 2915, 450, 450, 450, 450, 450, 450, 450,
450, 450, 450, 450, 450, 450, 450, 450, 1437, 450,
- 450, 450, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
+ 450, 450, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
- 2912, 2912, 450, 450, 1132, 1133, 1132, 1132, 1132, 1134,
+ 2915, 2915, 450, 450, 1132, 1133, 1132, 1132, 1132, 1134,
1135, 1132, 1135, 1132, 1132, 1132, 1132, 1132, 1132, 1132,
1132, 1132, 1132, 1132, 1132, 1132, 1136, 1132, 1132, 1132,
1132, 1132, 1132, 1132, 1132, 1132, 1132, 1132, 1132, 1132,
@@ -3164,23 +3164,23 @@ static yyconst flex_int16_t yy_nxt[27006] =
1607, 1607, 1607, 1607, 1607, 1607, 1607, 1607, 1607, 1607,
1607, 1607, 1607, 1607, 1607, 1607, 1607, 1607, 1610, 1610,
1610, 1610, 1610, 1610, 1611, 1611, 1611, 1612, 1607, 1607,
- 451, 451, 2912, 1393, 452, 452, 1713, 1394, 2538, 2912,
- 1714, 2912, 2538, 1713, 2151, 1870, 738, 1714, 1929, 1555,
- 878, 453, 2912, 1726, 879, 2912, 2912, 1727, 1723, 450,
- 1964, 450, 1966, 738, 450, 1608, 1865, 450, 2912, 450,
- 2912, 450, 450, 1967, 1684, 450, 738, 450, 1970, 450,
- 450, 450, 2912, 2912, 1969, 2912, 450, 450, 2912, 450,
-
- 1871, 1947, 2912, 738, 450, 1690, 1044, 450, 2912, 2912,
- 2912, 454, 454, 454, 454, 454, 454, 455, 455, 455,
- 456, 1685, 1375, 1049, 2912, 1686, 1045, 1045, 1045, 1045,
- 1045, 1045, 1046, 1046, 1046, 1047, 1054, 2912, 2912, 2912,
- 2912, 2912, 453, 1050, 1050, 1050, 1050, 1050, 1050, 1051,
- 1051, 1051, 1052, 1044, 2912, 2912, 1055, 1055, 1055, 1055,
- 1055, 1055, 1056, 1056, 1056, 1057, 1924, 2912, 1926, 2912,
- 1923, 2912, 1926, 1045, 1045, 1045, 1045, 1045, 1045, 1046,
- 1046, 1046, 1047, 2912, 2912, 2147, 2147, 1375, 1382, 453,
- 2148, 2912, 454, 454, 454, 454, 454, 454, 455, 455,
+ 451, 451, 2915, 1393, 452, 452, 1713, 1394, 2541, 2915,
+ 1714, 2915, 2541, 1713, 2002, 1872, 738, 1714, 2002, 1555,
+ 878, 453, 2915, 1726, 879, 2915, 2915, 1727, 1723, 450,
+ 1966, 450, 1968, 738, 450, 453, 1867, 450, 2915, 450,
+ 2915, 450, 450, 1969, 1684, 450, 738, 450, 1972, 450,
+ 450, 450, 2915, 2915, 1971, 2915, 450, 450, 2915, 450,
+
+ 1873, 1878, 2915, 738, 450, 1878, 1044, 450, 2915, 2915,
+ 2915, 454, 454, 454, 454, 454, 454, 455, 455, 455,
+ 456, 1685, 453, 1049, 2915, 1686, 1045, 1045, 1045, 1045,
+ 1045, 1045, 1046, 1046, 1046, 1047, 1054, 2859, 2106, 2915,
+ 2915, 2859, 453, 1050, 1050, 1050, 1050, 1050, 1050, 1051,
+ 1051, 1051, 1052, 1044, 2778, 2778, 1055, 1055, 1055, 1055,
+ 1055, 1055, 1056, 1056, 1056, 1057, 1926, 2860, 1926, 2915,
+ 1925, 2860, 1925, 1045, 1045, 1045, 1045, 1045, 1045, 1046,
+ 1046, 1046, 1047, 2915, 2780, 2780, 2915, 1375, 1382, 1375,
+ 1382, 2915, 454, 454, 454, 454, 454, 454, 455, 455,
455, 456, 1360, 1361, 1360, 1360, 1360, 1085, 1362, 1360,
1362, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360,
@@ -3190,16 +3190,16 @@ static yyconst flex_int16_t yy_nxt[27006] =
1360, 1360, 1360, 1360, 1360, 1362, 1362, 1362, 1362, 1362,
1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362,
1362, 1362, 1362, 1362, 1364, 1364, 1364, 1364, 1364, 1364,
- 1365, 1365, 1365, 1366, 1362, 1362, 2912, 451, 2912, 1100,
- 450, 452, 450, 1101, 2912, 2000, 2912, 1977, 1992, 2000,
+ 1365, 1365, 1365, 1366, 1362, 1362, 2915, 451, 2915, 1100,
+ 450, 452, 450, 1101, 2915, 2218, 2915, 1979, 1994, 2012,
- 2216, 1977, 1992, 2912, 2010, 1785, 2912, 2028, 2059, 1785,
- 2228, 2028, 2059, 2912, 2039, 450, 453, 450, 453, 453,
- 2912, 2012, 1973, 450, 2194, 450, 2912, 1782, 453, 453,
- 1971, 450, 1409, 450, 2208, 2912, 2912, 450, 450, 450,
+ 2915, 1979, 1994, 2915, 1787, 2915, 2030, 2230, 1787, 2915,
+ 2030, 2041, 2915, 2915, 2915, 450, 2014, 450, 453, 453,
+ 2915, 2915, 1975, 450, 2196, 450, 1784, 453, 2915, 1409,
+ 1973, 450, 2915, 450, 2210, 2915, 2915, 450, 450, 450,
450, 450, 450, 450, 450, 450, 450, 450, 450, 450,
- 450, 450, 450, 450, 450, 450, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 450, 450, 1370, 1371,
+ 450, 450, 450, 450, 450, 450, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 450, 450, 1370, 1371,
1370, 1370, 1370, 1372, 1373, 1370, 1373, 1370, 1370, 1370,
1370, 1370, 1370, 1370, 1370, 1370, 1370, 1370, 1370, 1374,
1375, 1376, 1370, 1370, 1370, 1370, 1370, 1370, 1370, 1370,
@@ -3255,16 +3255,16 @@ static yyconst flex_int16_t yy_nxt[27006] =
1473, 1476, 1476, 1476, 1476, 1476, 1476, 1476, 1476, 1476,
1476, 1476, 1476, 1476, 1476, 1476, 1476, 1476, 1476, 1476,
1478, 1478, 1478, 1478, 1478, 1478, 1479, 1479, 1479, 1480,
- 1476, 1476, 1862, 1863, 1862, 1862, 1862, 1555, 1864, 1862,
- 1864, 1862, 1862, 1862, 1862, 1862, 1862, 1862, 1862, 1862,
- 1862, 1862, 1862, 1862, 1865, 1862, 1862, 1862, 1862, 1862,
+ 1476, 1476, 1864, 1865, 1864, 1864, 1864, 1555, 1866, 1864,
+ 1866, 1864, 1864, 1864, 1864, 1864, 1864, 1864, 1864, 1864,
+ 1864, 1864, 1864, 1864, 1867, 1864, 1864, 1864, 1864, 1864,
- 1862, 1862, 1862, 1862, 1862, 1862, 1862, 1862, 1862, 1862,
- 1862, 1862, 1862, 1862, 1862, 1862, 1862, 1862, 1866, 1862,
- 1862, 1862, 1862, 1862, 1862, 1864, 1864, 1864, 1864, 1864,
1864, 1864, 1864, 1864, 1864, 1864, 1864, 1864, 1864, 1864,
- 1864, 1864, 1864, 1864, 1867, 1867, 1867, 1867, 1867, 1867,
- 1868, 1868, 1868, 1869, 1864, 1864, 1604, 1605, 1604, 1604,
+ 1864, 1864, 1864, 1864, 1864, 1864, 1864, 1864, 1868, 1864,
+ 1864, 1864, 1864, 1864, 1864, 1866, 1866, 1866, 1866, 1866,
+ 1866, 1866, 1866, 1866, 1866, 1866, 1866, 1866, 1866, 1866,
+ 1866, 1866, 1866, 1866, 1869, 1869, 1869, 1869, 1869, 1869,
+ 1870, 1870, 1870, 1871, 1866, 1866, 1604, 1605, 1604, 1604,
1604, 1606, 1607, 1604, 1607, 1604, 1604, 1604, 1604, 1604,
1604, 1604, 1604, 1604, 1604, 1604, 1604, 1604, 1608, 1609,
1604, 1604, 1604, 1604, 1604, 1604, 1604, 1604, 1604, 1604,
@@ -3274,87 +3274,87 @@ static yyconst flex_int16_t yy_nxt[27006] =
1607, 1607, 1607, 1607, 1607, 1607, 1607, 1607, 1607, 1607,
1607, 1607, 1607, 1607, 1607, 1607, 1607, 1607, 1610, 1610,
1610, 1610, 1610, 1610, 1611, 1611, 1611, 1612, 1607, 1607,
- 738, 451, 1976, 2583, 2912, 452, 1977, 2583, 451, 2912,
- 738, 451, 452, 451, 2092, 452, 2912, 452, 452, 2912,
- 451, 2040, 2912, 2334, 452, 2040, 1975, 2912, 2912, 450,
- 450, 2912, 2912, 2912, 1979, 2093, 450, 450, 450, 450,
- 738, 450, 2912, 1454, 450, 450, 450, 450, 450, 450,
- 1044, 738, 450, 1980, 1978, 450, 450, 450, 2912, 2912,
-
- 1320, 2912, 2229, 2912, 450, 2912, 2912, 2912, 2912, 2912,
+ 738, 451, 1978, 2586, 2915, 452, 1979, 2586, 451, 2915,
+ 738, 451, 452, 451, 2061, 452, 2915, 452, 2061, 2915,
+ 451, 2042, 2915, 2336, 452, 2042, 1977, 2915, 2915, 450,
+ 450, 2915, 2915, 2915, 1981, 453, 450, 450, 450, 450,
+ 738, 450, 2915, 1454, 450, 450, 450, 450, 450, 450,
+ 1044, 738, 450, 1982, 1980, 450, 450, 450, 2915, 2915,
+
+ 1320, 2915, 2231, 2915, 450, 2915, 2915, 2915, 2915, 2915,
1045, 1045, 1045, 1045, 1045, 1045, 1046, 1046, 1046, 1047,
1321, 1321, 1321, 1321, 1321, 1321, 1322, 1322, 1322, 1323,
- 1325, 738, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2832, 1330, 738, 451, 2832, 2912, 2912, 452, 2912, 2912,
+ 1325, 738, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2835, 1330, 738, 451, 2835, 2915, 2915, 452, 2915, 2915,
1326, 1326, 1326, 1326, 1326, 1326, 1327, 1327, 1327, 1328,
- 2727, 1331, 1331, 1331, 1331, 1331, 1331, 1332, 1332, 1332,
- 1333, 450, 738, 2040, 2043, 2912, 507, 2040, 2043, 450,
- 2832, 1335, 738, 2912, 2832, 2912, 2912, 450, 2912, 2050,
- 1453, 1453, 1335, 2050, 2912, 1454, 1472, 2912, 2912, 2912,
+ 2730, 1331, 1331, 1331, 1331, 1331, 1331, 1332, 1332, 1332,
+ 1333, 450, 738, 1928, 2045, 2915, 507, 1928, 2045, 450,
+ 2835, 1335, 738, 2915, 2835, 2915, 2915, 450, 2915, 2052,
+ 2149, 2149, 1335, 2052, 453, 2150, 1472, 2915, 2915, 2915,
- 2727, 1336, 1336, 1336, 1336, 1336, 1336, 1337, 1337, 1337,
+ 2730, 1336, 1336, 1336, 1336, 1336, 1336, 1337, 1337, 1337,
1338, 1594, 1336, 1336, 1336, 1336, 1336, 1336, 1337, 1337,
- 1337, 1338, 1341, 738, 2912, 2912, 2912, 2912, 2912, 2912,
- 2235, 2912, 1346, 738, 451, 2912, 1745, 2912, 452, 2912,
- 1746, 2912, 1342, 1342, 1342, 1342, 1342, 1342, 1343, 1343,
+ 1337, 1338, 1341, 738, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2237, 2915, 1346, 738, 451, 2915, 1745, 2915, 452, 2915,
+ 1746, 2915, 1342, 1342, 1342, 1342, 1342, 1342, 1343, 1343,
1343, 1344, 1347, 1347, 1347, 1347, 1347, 1347, 1348, 1348,
- 1348, 1349, 450, 738, 450, 2269, 2912, 2050, 2912, 2096,
- 450, 2050, 450, 1351, 738, 2912, 2912, 1983, 450, 2912,
- 450, 2912, 2912, 1044, 1593, 1593, 1865, 451, 2912, 1594,
- 2912, 452, 2912, 1352, 1352, 1352, 1352, 1352, 1352, 1353,
+ 1348, 1349, 450, 738, 450, 2094, 2915, 2042, 2915, 452,
+ 450, 2042, 450, 1351, 738, 2915, 2915, 1985, 450, 2915,
+ 450, 2915, 2915, 1044, 1453, 1453, 2095, 451, 2915, 1454,
+ 2915, 452, 2915, 1352, 1352, 1352, 1352, 1352, 1352, 1353,
1353, 1353, 1354, 1045, 1045, 1045, 1045, 1045, 1045, 1046,
- 1046, 1046, 1047, 1044, 738, 450, 1986, 2912, 2912, 2912,
- 2912, 2912, 2912, 450, 1320, 738, 1745, 2912, 2912, 2912,
- 1746, 450, 2912, 1045, 1045, 1045, 1045, 1045, 1045, 1046,
- 1046, 1046, 1047, 2912, 1321, 1321, 1321, 1321, 1321, 1321,
- 1322, 1322, 1322, 1323, 450, 738, 2237, 2320, 2912, 2912,
- 2238, 2321, 450, 1984, 1325, 738, 2912, 2912, 2912, 2912,
- 450, 2912, 2912, 2150, 2912, 1330, 2239, 2150, 1609, 1382,
- 2912, 2912, 2912, 2912, 1326, 1326, 1326, 1326, 1326, 1326,
- 1327, 1327, 1327, 1328, 453, 1331, 1331, 1331, 1331, 1331,
-
- 1331, 1332, 1332, 1332, 1333, 1320, 738, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 1320, 738, 451, 2912, 1120,
- 2912, 452, 2912, 1121, 2912, 1321, 1321, 1321, 1321, 1321,
+ 1046, 1046, 1047, 1044, 738, 450, 1988, 2915, 2915, 2915,
+ 2915, 2915, 2915, 450, 1320, 738, 1745, 2915, 2915, 2915,
+ 1746, 450, 2915, 1045, 1045, 1045, 1045, 1045, 1045, 1046,
+ 1046, 1046, 1047, 2915, 1321, 1321, 1321, 1321, 1321, 1321,
+ 1322, 1322, 1322, 1323, 450, 738, 2052, 2239, 2915, 2915,
+ 2052, 2240, 450, 1986, 1325, 738, 2915, 2915, 2915, 2915,
+ 450, 2915, 2322, 1593, 1593, 1330, 2323, 2241, 1594, 1609,
+ 2915, 2915, 2915, 2915, 1326, 1326, 1326, 1326, 1326, 1326,
+ 1327, 1327, 1327, 1328, 1382, 1331, 1331, 1331, 1331, 1331,
+
+ 1331, 1332, 1332, 1332, 1333, 1320, 738, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 1320, 738, 451, 2915, 1120,
+ 2915, 452, 2915, 1121, 2915, 1321, 1321, 1321, 1321, 1321,
1321, 1322, 1322, 1322, 1323, 1321, 1321, 1321, 1321, 1321,
- 1321, 1322, 1322, 1322, 1323, 450, 738, 450, 2066, 2912,
- 2087, 2912, 2066, 450, 2087, 450, 1654, 738, 2912, 2912,
- 1987, 450, 2912, 450, 2912, 2912, 1659, 2912, 2070, 453,
- 1991, 453, 2089, 2912, 1992, 2912, 1655, 1655, 1655, 1655,
+ 1321, 1322, 1322, 1322, 1323, 450, 738, 450, 2068, 2915,
+ 2089, 2915, 2068, 450, 2089, 450, 1654, 738, 2915, 2915,
+ 1989, 450, 2915, 450, 2915, 2915, 1659, 2915, 2072, 453,
+ 1993, 453, 2091, 2915, 1994, 2915, 1655, 1655, 1655, 1655,
1655, 1655, 1656, 1656, 1656, 1657, 1660, 1660, 1660, 1660,
- 1660, 1660, 1661, 1661, 1661, 1662, 1664, 738, 450, 2912,
-
- 2912, 2912, 2912, 2912, 2912, 2912, 450, 1341, 738, 1752,
- 2912, 2912, 2912, 1753, 450, 2912, 1665, 1665, 1665, 1665,
- 1665, 1665, 1666, 1666, 1666, 1667, 2912, 1342, 1342, 1342,
- 1342, 1342, 1342, 1343, 1343, 1343, 1344, 450, 738, 2252,
- 2228, 2912, 2912, 2252, 2039, 450, 1989, 1341, 738, 2912,
- 2912, 2912, 2912, 450, 2912, 2912, 2912, 2912, 1669, 1822,
- 2912, 1756, 1409, 2912, 2912, 1757, 2912, 1342, 1342, 1342,
- 1342, 1342, 1342, 1343, 1343, 1343, 1344, 2253, 1670, 1670,
+ 1660, 1660, 1661, 1661, 1661, 1662, 1664, 738, 450, 2915,
+
+ 2915, 2915, 2915, 2915, 2915, 2915, 450, 1341, 738, 1752,
+ 2915, 2915, 2915, 1753, 450, 2915, 1665, 1665, 1665, 1665,
+ 1665, 1665, 1666, 1666, 1666, 1667, 2915, 1342, 1342, 1342,
+ 1342, 1342, 1342, 1343, 1343, 1343, 1344, 450, 738, 2254,
+ 2230, 2915, 2915, 2254, 2041, 450, 1991, 1341, 738, 2915,
+ 2915, 2915, 2915, 450, 2915, 2915, 2915, 2915, 1669, 1824,
+ 2915, 1756, 1409, 2915, 2915, 1757, 2915, 1342, 1342, 1342,
+ 1342, 1342, 1342, 1343, 1343, 1343, 1344, 2255, 1670, 1670,
1670, 1670, 1670, 1670, 1671, 1671, 1671, 1672, 1674, 450,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 450, 1679, 2912,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 450, 1679, 2915,
- 2912, 2912, 451, 2912, 2912, 450, 452, 2912, 1675, 1675,
+ 2915, 2915, 451, 2915, 2915, 450, 452, 2915, 1675, 1675,
1675, 1675, 1675, 1675, 1676, 1676, 1676, 1677, 1680, 1680,
1680, 1680, 1680, 1680, 1681, 1681, 1681, 1682, 451, 451,
- 450, 1994, 452, 452, 2912, 1759, 2912, 2151, 450, 1760,
- 451, 1929, 2912, 2912, 452, 2912, 450, 451, 2912, 453,
- 2912, 452, 1999, 2912, 894, 2912, 2000, 450, 1608, 2912,
- 2198, 2912, 2912, 1761, 2198, 450, 1938, 2912, 450, 1995,
- 2912, 450, 2912, 450, 2171, 450, 450, 1762, 2171, 450,
- 450, 453, 2912, 450, 450, 2912, 2912, 2912, 450, 1128,
- 1996, 450, 2912, 1129, 2334, 2174, 450, 2912, 2912, 454,
+ 450, 1996, 452, 452, 2915, 1759, 2915, 2271, 450, 1760,
+ 451, 2098, 2915, 2915, 452, 2915, 450, 451, 2915, 453,
+ 2915, 452, 2001, 2915, 894, 2915, 2002, 450, 1867, 2915,
+ 2152, 2915, 2915, 1761, 2152, 450, 1940, 2915, 450, 1997,
+ 2915, 450, 2915, 450, 2153, 450, 450, 1762, 1931, 450,
+ 450, 453, 2915, 450, 450, 2915, 2915, 2915, 450, 1128,
+ 1998, 450, 2915, 1129, 2915, 1608, 450, 2915, 2915, 454,
454, 454, 454, 454, 454, 455, 455, 455, 456, 1685,
- 451, 1130, 451, 1686, 452, 2216, 452, 450, 2912, 2010,
- 2912, 2912, 1772, 2264, 1772, 450, 1773, 2264, 1773, 2912,
- 453, 2001, 2912, 450, 2912, 2002, 2012, 1939, 450, 2912,
- 450, 2912, 2912, 2240, 453, 1940, 450, 2238, 450, 2912,
- 450, 2912, 450, 1941, 450, 2003, 450, 2380, 450, 2005,
- 450, 2380, 2912, 2241, 1608, 1614, 450, 2912, 450, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2026,
+ 1768, 1130, 451, 1686, 1769, 451, 452, 450, 2915, 452,
+ 2200, 2915, 1774, 2915, 2200, 450, 1775, 1452, 1452, 2915,
+ 453, 2003, 1774, 450, 2915, 2004, 1775, 1941, 450, 2915,
+ 450, 453, 2915, 450, 2915, 1942, 450, 2915, 450, 2915,
+ 450, 450, 2915, 1943, 450, 2005, 450, 2242, 450, 450,
+ 450, 2240, 2915, 2915, 2007, 2915, 450, 2915, 450, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 450, 2243, 1608, 1614,
454, 454, 454, 454, 454, 454, 455, 455, 455, 456,
1370, 1689, 1370, 1370, 1370, 1690, 1373, 1370, 1373, 1370,
@@ -3374,114 +3374,114 @@ static yyconst flex_int16_t yy_nxt[27006] =
1732, 1737, 1732, 1732, 1732, 1732, 1732, 1734, 1734, 1734,
1734, 1734, 1734, 1734, 1734, 1734, 1734, 1734, 1734, 1734,
1734, 1734, 1734, 1734, 1734, 1734, 1738, 1738, 1738, 1738,
- 1738, 1738, 1739, 1739, 1739, 1740, 1734, 1734, 2912, 451,
- 2912, 1146, 450, 452, 450, 1147, 2386, 2912, 2912, 2912,
- 2386, 2237, 2912, 2269, 2912, 2238, 2912, 2096, 2240, 2912,
- 2282, 2912, 2238, 2912, 2282, 2912, 2240, 450, 2148, 450,
- 2238, 2239, 2912, 1609, 1865, 450, 2912, 450, 2241, 1608,
+ 1738, 1738, 1739, 1739, 1739, 1740, 1734, 1734, 2915, 451,
+ 2915, 1146, 450, 452, 450, 1147, 2915, 2218, 2915, 2915,
+ 2173, 2012, 2383, 2915, 2173, 2266, 2383, 2389, 2239, 2266,
+ 2271, 2389, 2240, 2915, 2098, 2915, 2915, 450, 2014, 450,
+ 2336, 2176, 2915, 2915, 2028, 450, 453, 450, 2241, 2150,
- 1614, 453, 2006, 450, 2912, 450, 2241, 1608, 1614, 450,
+ 1609, 1867, 2008, 450, 2915, 450, 2915, 2915, 2915, 450,
450, 450, 450, 450, 450, 450, 450, 450, 450, 450,
- 450, 450, 450, 450, 450, 450, 450, 450, 1780, 2252,
- 2285, 2320, 1781, 2252, 2285, 2321, 2320, 2912, 450, 450,
- 2321, 2336, 2470, 2380, 2589, 2336, 2470, 2380, 2589, 1822,
- 1782, 453, 2912, 1382, 2912, 2912, 450, 2912, 1382, 2912,
- 2396, 2912, 2174, 453, 450, 2026, 1241, 2912, 2912, 2912,
- 2912, 2912, 450, 2008, 2009, 2008, 2008, 2008, 2010, 2011,
- 2008, 2011, 2008, 2008, 2008, 2008, 2008, 2008, 2008, 2008,
- 2008, 2008, 2008, 2008, 2008, 2012, 450, 2008, 2008, 2008,
-
- 2008, 2008, 2008, 2008, 2008, 2008, 2008, 2008, 2008, 2008,
- 2008, 2008, 2008, 2008, 2008, 2008, 2008, 2008, 2008, 2008,
- 2008, 2008, 2008, 2008, 2008, 2008, 2011, 2011, 2011, 2011,
- 2011, 2011, 2011, 2011, 2011, 2011, 2011, 2011, 2011, 2011,
- 2011, 2011, 2011, 2011, 2011, 2013, 2013, 2013, 2013, 2013,
- 2013, 2014, 2014, 2014, 2015, 2011, 2011, 1780, 2912, 2437,
- 451, 1781, 2458, 2437, 452, 2912, 2458, 2912, 2912, 912,
- 2912, 2912, 1796, 913, 2912, 2912, 1797, 2443, 2912, 1782,
- 453, 2443, 2027, 453, 2912, 450, 2028, 2912, 450, 2532,
- 2912, 451, 2550, 450, 1798, 452, 450, 450, 453, 2912,
-
- 450, 450, 2018, 2011, 450, 450, 2019, 2008, 450, 2008,
- 450, 2386, 2912, 2025, 2912, 2386, 450, 2912, 450, 450,
- 2506, 2537, 2912, 2008, 2506, 2912, 450, 450, 2147, 2147,
- 2912, 2912, 2565, 2148, 2031, 450, 2565, 2912, 2912, 2912,
- 2912, 453, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 453, 2008, 2008, 2008, 2008, 2008, 2008,
- 2008, 2008, 2008, 2008, 2008, 2008, 2008, 2008, 2008, 2008,
- 2008, 2008, 2008, 2008, 2008, 2008, 2008, 2008, 2008, 2008,
- 2008, 2008, 2008, 2008, 2008, 451, 451, 1128, 2912, 452,
- 452, 1129, 2912, 2912, 2912, 2912, 451, 2662, 451, 451,
-
- 452, 2662, 452, 452, 2912, 2912, 2912, 452, 1823, 1130,
- 2029, 452, 1824, 450, 450, 450, 2033, 2727, 453, 2032,
- 2912, 450, 450, 450, 450, 2034, 450, 450, 453, 450,
- 450, 450, 450, 1263, 450, 2035, 450, 1264, 2912, 1220,
- 450, 2912, 450, 450, 450, 2054, 2912, 451, 2912, 2055,
- 1832, 452, 450, 2912, 1833, 1265, 2912, 2912, 2912, 2058,
- 2065, 450, 2036, 2059, 2066, 1512, 2912, 2912, 2912, 450,
- 451, 2912, 451, 450, 452, 450, 452, 450, 450, 2912,
- 1090, 450, 2912, 450, 2912, 2063, 450, 450, 450, 450,
- 2056, 450, 2060, 451, 450, 450, 450, 452, 450, 2912,
-
- 450, 2912, 2912, 450, 450, 451, 450, 1206, 450, 452,
- 2912, 1207, 2912, 2912, 450, 2067, 450, 2912, 1209, 451,
- 2912, 450, 1210, 452, 2912, 2912, 2912, 2912, 2068, 450,
- 451, 2912, 2912, 450, 452, 450, 2079, 450, 2040, 2912,
- 2912, 450, 2040, 450, 2080, 2912, 450, 450, 2077, 450,
- 2912, 450, 451, 2086, 450, 450, 452, 2087, 450, 2082,
- 1454, 2912, 450, 450, 451, 2050, 450, 2912, 452, 2050,
- 2912, 2912, 2040, 451, 450, 2088, 2040, 452, 451, 2912,
- 450, 450, 452, 2912, 2483, 2912, 2912, 1594, 450, 450,
- 2912, 2912, 450, 2912, 1454, 2083, 450, 450, 2618, 2589,
-
- 450, 450, 2619, 2589, 2912, 2912, 450, 2090, 450, 450,
- 2571, 2487, 2912, 2912, 450, 2094, 2912, 450, 2912, 2912,
- 1614, 1241, 450, 1862, 1863, 1862, 1862, 1862, 1555, 1864,
- 1862, 1864, 1862, 1862, 1862, 1862, 1862, 1862, 1862, 1862,
- 1862, 1862, 1862, 1862, 1862, 1865, 1862, 1862, 1862, 1862,
- 1862, 1862, 1862, 1862, 1862, 1862, 1862, 1862, 1862, 1862,
- 1862, 1862, 1862, 1862, 1862, 1862, 1862, 1862, 1862, 1866,
- 1862, 1862, 1862, 1862, 1862, 1862, 1864, 1864, 1864, 1864,
+ 450, 450, 450, 450, 450, 450, 450, 450, 1782, 2254,
+ 2915, 2242, 1783, 2254, 2915, 2240, 2915, 2242, 450, 450,
+ 2284, 2240, 2287, 2322, 2284, 2915, 2287, 2323, 2915, 1824,
+ 1784, 2243, 1608, 1614, 2915, 2915, 450, 2243, 1608, 1614,
+ 2399, 453, 2915, 453, 450, 1382, 2915, 2915, 2915, 2915,
+ 2915, 2915, 450, 2010, 2011, 2010, 2010, 2010, 2012, 2013,
+ 2010, 2013, 2010, 2010, 2010, 2010, 2010, 2010, 2010, 2010,
+ 2010, 2010, 2010, 2010, 2010, 2014, 450, 2010, 2010, 2010,
+
+ 2010, 2010, 2010, 2010, 2010, 2010, 2010, 2010, 2010, 2010,
+ 2010, 2010, 2010, 2010, 2010, 2010, 2010, 2010, 2010, 2010,
+ 2010, 2010, 2010, 2010, 2010, 2010, 2013, 2013, 2013, 2013,
+ 2013, 2013, 2013, 2013, 2013, 2013, 2013, 2013, 2013, 2013,
+ 2013, 2013, 2013, 2013, 2013, 2015, 2015, 2015, 2015, 2015,
+ 2015, 2016, 2016, 2016, 2017, 2013, 2013, 1782, 2322, 2915,
+ 451, 1783, 2323, 2338, 452, 2915, 2915, 2338, 2915, 912,
+ 2383, 2915, 1798, 913, 2383, 2915, 1799, 2471, 2915, 1784,
+ 1382, 2471, 2029, 2915, 2176, 450, 2030, 2915, 450, 2915,
+ 2915, 451, 2028, 450, 1800, 452, 450, 450, 453, 2915,
+
+ 450, 450, 2020, 2013, 450, 450, 2021, 2010, 450, 2010,
+ 450, 2389, 2915, 2027, 2563, 2389, 450, 2440, 450, 450,
+ 2461, 2440, 2915, 2010, 2461, 2915, 450, 450, 2149, 2149,
+ 2915, 2915, 2473, 2150, 2033, 450, 2473, 2915, 453, 2915,
+ 2915, 453, 2915, 2915, 2915, 2915, 2915, 2535, 2915, 2915,
+ 2553, 2915, 2915, 453, 2010, 2010, 2010, 2010, 2010, 2010,
+ 2010, 2010, 2010, 2010, 2010, 2010, 2010, 2010, 2010, 2010,
+ 2010, 2010, 2010, 2010, 2010, 2010, 2010, 2010, 2010, 2010,
+ 2010, 2010, 2010, 2010, 2010, 451, 451, 1128, 2915, 452,
+ 452, 1129, 2915, 2915, 2915, 2915, 451, 2446, 451, 451,
+
+ 452, 2446, 452, 452, 2915, 2915, 2915, 2509, 1825, 1130,
+ 2031, 2509, 1826, 450, 450, 450, 2035, 2915, 453, 2034,
+ 2915, 450, 450, 450, 450, 2036, 450, 450, 453, 450,
+ 450, 450, 450, 1263, 450, 2037, 450, 1264, 2915, 1220,
+ 450, 2540, 450, 450, 450, 2056, 2915, 451, 2915, 2057,
+ 1834, 452, 450, 2915, 1835, 1265, 2915, 2915, 2915, 2060,
+ 2067, 450, 2038, 2061, 2068, 1512, 2915, 2915, 2915, 450,
+ 451, 2915, 451, 450, 452, 450, 452, 450, 450, 2915,
+ 1090, 450, 2915, 450, 2915, 2065, 450, 450, 450, 450,
+ 2058, 450, 2062, 451, 450, 450, 450, 452, 450, 2915,
+
+ 450, 2915, 2915, 450, 450, 451, 450, 1206, 450, 452,
+ 2915, 1207, 2915, 2915, 450, 2069, 450, 2915, 1209, 451,
+ 2915, 450, 1210, 452, 2915, 2915, 2915, 2915, 2070, 450,
+ 451, 2915, 2915, 450, 452, 450, 2081, 450, 2042, 2915,
+ 2915, 450, 2042, 450, 2082, 2915, 450, 450, 2079, 450,
+ 2915, 450, 451, 2088, 450, 450, 452, 2089, 450, 2084,
+ 1454, 2915, 450, 450, 451, 2052, 450, 2915, 452, 2052,
+ 2915, 2915, 2042, 451, 450, 2090, 2042, 452, 451, 2915,
+ 450, 450, 452, 2915, 2486, 2915, 2915, 1594, 450, 450,
+ 2915, 2915, 450, 2915, 1454, 2085, 450, 450, 2592, 2621,
+
+ 450, 450, 2592, 2622, 2915, 2915, 450, 2092, 450, 450,
+ 2574, 2490, 2915, 2915, 450, 2096, 2915, 450, 2915, 2915,
+ 1241, 1614, 450, 1864, 1865, 1864, 1864, 1864, 1555, 1866,
+ 1864, 1866, 1864, 1864, 1864, 1864, 1864, 1864, 1864, 1864,
+ 1864, 1864, 1864, 1864, 1864, 1867, 1864, 1864, 1864, 1864,
1864, 1864, 1864, 1864, 1864, 1864, 1864, 1864, 1864, 1864,
- 1864, 1864, 1864, 1864, 1864, 1867, 1867, 1867, 1867, 1867,
-
- 1867, 1868, 1868, 1868, 1869, 1864, 1864, 2095, 2912, 451,
- 1222, 2096, 1875, 452, 1223, 2912, 1876, 2912, 2912, 451,
- 2040, 2912, 451, 452, 2040, 2912, 452, 2912, 2912, 451,
- 2912, 2912, 2097, 452, 2912, 450, 2912, 450, 450, 2102,
- 450, 2912, 1454, 450, 2912, 450, 450, 450, 450, 2103,
- 450, 450, 2100, 450, 450, 450, 450, 450, 450, 2912,
- 1234, 2642, 2912, 2105, 1235, 450, 450, 451, 2106, 2912,
- 451, 452, 451, 450, 452, 2912, 452, 451, 2912, 1243,
- 2912, 452, 2109, 1244, 2912, 2912, 2912, 2050, 450, 953,
- 451, 2050, 2912, 954, 452, 450, 450, 2050, 450, 2912,
-
- 450, 2050, 2111, 450, 450, 450, 450, 450, 450, 1594,
- 2912, 450, 2110, 2112, 450, 450, 450, 450, 450, 1594,
- 451, 450, 2115, 450, 452, 450, 450, 451, 2649, 2912,
- 2116, 452, 451, 450, 450, 2578, 452, 451, 451, 2912,
- 2587, 452, 452, 2912, 2587, 2912, 2912, 2618, 450, 451,
- 2618, 2619, 2912, 452, 2619, 2118, 450, 2912, 2912, 2912,
- 450, 453, 1241, 450, 2117, 450, 450, 2122, 450, 1614,
- 2119, 450, 1614, 450, 450, 2120, 450, 450, 2912, 1907,
- 451, 2121, 450, 1908, 452, 450, 2123, 2912, 2124, 2912,
- 451, 1255, 2468, 450, 452, 1256, 2468, 451, 2912, 2761,
-
- 2569, 452, 451, 2761, 2569, 2912, 452, 450, 450, 2137,
- 2912, 2912, 2912, 453, 2912, 450, 450, 2135, 450, 450,
- 453, 453, 2912, 450, 450, 450, 450, 450, 2138, 2560,
- 450, 2143, 1916, 450, 450, 450, 1917, 2641, 450, 1263,
- 2139, 450, 2912, 1264, 451, 2912, 450, 2040, 452, 1592,
- 1592, 2040, 2912, 2912, 2912, 2912, 2912, 2912, 2149, 2912,
- 450, 1265, 2150, 2912, 2912, 2912, 2912, 450, 450, 1454,
- 451, 2912, 450, 1925, 452, 450, 450, 1926, 2616, 2912,
- 450, 2141, 2616, 450, 2912, 2142, 450, 2145, 450, 2765,
- 2146, 2146, 2729, 2912, 450, 1927, 2729, 2912, 450, 453,
-
- 2912, 450, 450, 2912, 2912, 2152, 450, 2912, 2912, 450,
- 2912, 2912, 2912, 453, 450, 2683, 2912, 450, 1604, 1928,
- 1604, 1604, 1604, 1929, 1607, 1604, 1607, 1604, 1604, 1604,
+ 1864, 1864, 1864, 1864, 1864, 1864, 1864, 1864, 1864, 1868,
+ 1864, 1864, 1864, 1864, 1864, 1864, 1866, 1866, 1866, 1866,
+ 1866, 1866, 1866, 1866, 1866, 1866, 1866, 1866, 1866, 1866,
+ 1866, 1866, 1866, 1866, 1866, 1869, 1869, 1869, 1869, 1869,
+
+ 1869, 1870, 1870, 1870, 1871, 1866, 1866, 2097, 2915, 451,
+ 1222, 2098, 1877, 452, 1223, 2915, 1878, 2915, 2915, 451,
+ 2568, 452, 451, 452, 2568, 452, 452, 2915, 2915, 451,
+ 2915, 2915, 2099, 452, 2915, 450, 2915, 450, 450, 2104,
+ 450, 453, 453, 450, 2915, 450, 450, 450, 450, 2105,
+ 450, 450, 2102, 450, 450, 450, 450, 450, 450, 2915,
+ 1234, 2915, 2915, 2107, 1235, 450, 450, 451, 2108, 2915,
+ 451, 452, 451, 450, 452, 2915, 452, 451, 2915, 1243,
+ 2915, 452, 2111, 1244, 2915, 2915, 2915, 2042, 450, 953,
+ 451, 2042, 2915, 954, 452, 450, 450, 2052, 450, 2915,
+
+ 450, 2052, 2113, 450, 450, 450, 450, 450, 450, 1454,
+ 2915, 450, 2112, 2114, 450, 450, 450, 450, 450, 1594,
+ 451, 450, 2117, 450, 452, 450, 450, 451, 2645, 2915,
+ 2118, 452, 451, 450, 450, 2581, 452, 451, 451, 2915,
+ 2590, 452, 452, 2915, 2590, 2915, 2915, 2592, 450, 451,
+ 2621, 2592, 2915, 452, 2622, 2120, 450, 2915, 2915, 2915,
+ 450, 453, 1241, 450, 2119, 450, 450, 2124, 450, 1241,
+ 2121, 450, 1614, 450, 450, 2122, 450, 450, 2915, 1909,
+ 451, 2123, 450, 1910, 452, 450, 2125, 2915, 2126, 2915,
+ 451, 1255, 2572, 450, 452, 1256, 2572, 451, 2915, 2764,
+
+ 2619, 452, 451, 2764, 2619, 2915, 452, 450, 450, 2139,
+ 2915, 2915, 2915, 453, 2915, 450, 450, 2137, 450, 450,
+ 453, 453, 2915, 450, 450, 450, 450, 450, 2140, 2644,
+ 450, 2145, 1918, 450, 450, 450, 1919, 2686, 450, 1263,
+ 2141, 450, 2915, 1264, 451, 2915, 450, 2621, 452, 1592,
+ 1592, 2622, 2915, 2915, 2915, 2915, 2915, 2915, 2151, 2915,
+ 450, 1265, 2152, 2915, 2915, 2915, 2915, 450, 450, 1614,
+ 451, 2915, 450, 1927, 452, 450, 450, 1928, 2783, 2915,
+ 450, 2143, 2783, 450, 2915, 2144, 450, 2147, 450, 2784,
+ 2148, 2148, 2665, 2784, 450, 1929, 2665, 2915, 450, 453,
+
+ 2091, 450, 450, 2915, 2915, 2154, 450, 2915, 2915, 450,
+ 2915, 2091, 2730, 453, 450, 2915, 2915, 450, 1604, 1930,
+ 1604, 1604, 1604, 1931, 1607, 1604, 1607, 1604, 1604, 1604,
1604, 1604, 1604, 1604, 1604, 1604, 1604, 1604, 1604, 1604,
1608, 450, 1604, 1604, 1604, 1604, 1604, 1604, 1604, 1604,
1604, 1604, 1604, 1604, 1604, 1604, 1604, 1604, 1604, 1604,
@@ -3490,120 +3490,120 @@ static yyconst flex_int16_t yy_nxt[27006] =
1607, 1607, 1607, 1607, 1607, 1607, 1607, 1607, 1607, 1607,
1610, 1610, 1610, 1610, 1610, 1610, 1611, 1611, 1611, 1612,
- 1607, 1607, 451, 2912, 991, 451, 452, 451, 992, 452,
- 2912, 452, 2781, 2912, 2040, 2912, 2781, 2912, 2040, 451,
- 2912, 2912, 2050, 452, 451, 2912, 2050, 2912, 452, 2912,
- 450, 2912, 450, 450, 2089, 450, 1454, 2912, 450, 2155,
- 450, 450, 2705, 450, 1594, 2157, 450, 450, 450, 2156,
- 1084, 450, 450, 509, 1085, 450, 2912, 451, 2158, 2912,
- 450, 452, 451, 450, 2771, 2912, 452, 2780, 450, 451,
- 2818, 2780, 2912, 452, 2818, 451, 2912, 2912, 450, 452,
- 2040, 2912, 2912, 2912, 2040, 450, 450, 2050, 453, 2089,
- 450, 2050, 2819, 450, 450, 2163, 2761, 450, 450, 2162,
-
- 2761, 450, 1454, 450, 2912, 450, 450, 2912, 2167, 1594,
- 2912, 450, 2165, 450, 2912, 2712, 2912, 453, 2912, 450,
- 2169, 2170, 2169, 2169, 2169, 2171, 2172, 2169, 2172, 2169,
- 2169, 2169, 2169, 2169, 2169, 2169, 2169, 2169, 2169, 2169,
- 2169, 2173, 2174, 2169, 2169, 2169, 2169, 2169, 2169, 2169,
- 2169, 2169, 2169, 2169, 2169, 2169, 2169, 2169, 2169, 2169,
- 2169, 2169, 2169, 2169, 2169, 2169, 2169, 2169, 2169, 2169,
- 2169, 2169, 2169, 2172, 2172, 2172, 2172, 2172, 2172, 2172,
- 2172, 2172, 2172, 2172, 2172, 2172, 2172, 2172, 2172, 2172,
- 2172, 2172, 2175, 2175, 2175, 2175, 2175, 2175, 2176, 2176,
-
- 2176, 2177, 2172, 2172, 451, 1701, 451, 2050, 452, 1702,
- 452, 2050, 2912, 2912, 2781, 1704, 2848, 451, 2781, 1705,
- 2849, 452, 2912, 2912, 1707, 2912, 451, 1393, 1708, 1594,
- 452, 1394, 450, 450, 450, 2180, 2089, 2178, 2809, 2912,
- 450, 450, 450, 450, 2182, 450, 2183, 2912, 450, 450,
- 450, 450, 450, 450, 450, 450, 2912, 2912, 2912, 450,
- 450, 450, 450, 450, 2187, 2912, 2912, 2912, 450, 2185,
- 450, 450, 451, 2912, 1713, 451, 452, 451, 1714, 452,
- 2912, 452, 2912, 2805, 2818, 2912, 1100, 2805, 2818, 451,
- 1101, 2912, 2912, 452, 2825, 2912, 2912, 2912, 2825, 2912,
-
- 450, 2188, 450, 450, 453, 450, 2819, 2912, 450, 2912,
- 450, 450, 2190, 450, 450, 453, 450, 450, 450, 450,
- 1976, 450, 450, 2191, 1977, 450, 2912, 2195, 2912, 2912,
- 450, 2196, 2192, 450, 2197, 2912, 2912, 451, 2198, 451,
- 451, 452, 2912, 452, 452, 2846, 451, 2193, 450, 2846,
- 452, 2912, 2912, 2912, 2912, 450, 450, 2848, 2912, 2912,
- 2912, 2849, 450, 450, 450, 450, 453, 450, 450, 2202,
- 450, 450, 871, 450, 450, 450, 450, 2912, 450, 2809,
- 2912, 450, 450, 450, 450, 451, 2199, 451, 2912, 452,
- 450, 452, 1991, 2912, 451, 2912, 1992, 451, 452, 2912,
-
- 2912, 452, 2912, 2912, 2912, 2849, 1999, 451, 2912, 2849,
- 2000, 452, 2204, 450, 2912, 450, 2205, 2912, 2912, 2912,
- 450, 450, 450, 450, 2912, 450, 2813, 2209, 2207, 450,
- 450, 450, 2210, 450, 450, 450, 450, 2912, 450, 2912,
- 451, 450, 450, 450, 452, 2912, 2912, 2912, 2912, 1128,
- 450, 450, 451, 1129, 451, 2912, 452, 2912, 452, 2912,
- 2912, 451, 2912, 451, 2912, 452, 912, 452, 450, 2912,
- 913, 1130, 2912, 2213, 2912, 2027, 450, 450, 2912, 2028,
- 450, 2912, 450, 2215, 450, 450, 2912, 2214, 450, 450,
- 450, 450, 2912, 450, 450, 2912, 450, 450, 450, 450,
-
- 451, 2912, 450, 450, 452, 450, 2912, 450, 2217, 451,
- 450, 450, 451, 452, 2912, 2912, 452, 2912, 451, 450,
- 2780, 2912, 452, 2912, 2780, 2912, 2912, 2219, 450, 1128,
- 451, 2912, 2912, 1129, 452, 2912, 450, 450, 2912, 1263,
- 450, 453, 2089, 1264, 450, 450, 450, 2912, 450, 2222,
- 2221, 1130, 2912, 450, 450, 2224, 450, 450, 450, 451,
- 2058, 1265, 450, 452, 2059, 450, 450, 450, 2912, 2912,
- 2223, 2912, 2058, 450, 450, 450, 2059, 2196, 451, 2912,
- 2225, 2196, 452, 450, 2912, 2912, 2912, 450, 450, 2912,
- 2065, 2912, 2912, 2912, 2066, 450, 450, 451, 453, 2912,
-
- 450, 452, 2353, 450, 450, 2912, 450, 1510, 450, 2242,
- 1090, 1511, 2912, 2244, 450, 2912, 450, 2245, 450, 451,
- 2912, 2354, 2246, 452, 2247, 450, 450, 1512, 1206, 2912,
- 2912, 2912, 1207, 450, 450, 450, 2912, 2912, 2248, 2912,
- 451, 450, 2912, 450, 452, 2249, 451, 450, 2811, 451,
- 452, 450, 2811, 452, 2912, 450, 450, 2912, 2086, 2912,
- 2912, 2263, 2087, 450, 450, 2264, 2912, 2257, 450, 2813,
- 2819, 2260, 450, 2912, 450, 2912, 450, 450, 2912, 2259,
- 2088, 2912, 450, 451, 450, 450, 450, 452, 2912, 450,
- 450, 2912, 451, 450, 450, 2912, 452, 450, 2912, 2912,
-
- 2912, 2912, 450, 2095, 451, 450, 1222, 2096, 452, 451,
- 1223, 450, 2912, 452, 2912, 2912, 2266, 2912, 451, 450,
- 450, 451, 452, 2912, 2912, 452, 2912, 450, 450, 2912,
- 2912, 450, 450, 2912, 450, 2267, 450, 450, 2273, 450,
- 450, 2912, 450, 2912, 2270, 450, 450, 450, 450, 450,
- 450, 451, 451, 450, 450, 452, 452, 450, 2912, 2912,
- 2912, 2275, 450, 2284, 2281, 2276, 2912, 2285, 2282, 2912,
- 1243, 2912, 2912, 451, 1244, 2912, 2912, 452, 2912, 450,
- 450, 2912, 451, 2912, 2912, 2280, 452, 450, 450, 2286,
- 2912, 450, 450, 2912, 2279, 450, 450, 2283, 450, 450,
-
- 450, 450, 2912, 2912, 2912, 2912, 450, 450, 450, 450,
- 450, 451, 2912, 2912, 2291, 452, 451, 2292, 450, 2293,
- 452, 451, 2912, 2912, 2912, 452, 450, 2912, 451, 2912,
- 2912, 451, 452, 2714, 2912, 452, 2912, 2714, 2912, 450,
- 2294, 451, 451, 2912, 450, 452, 452, 450, 2912, 450,
- 2773, 2773, 450, 2912, 453, 450, 450, 450, 2296, 450,
- 450, 2912, 2295, 2297, 450, 450, 2299, 450, 2912, 450,
- 450, 2298, 450, 2912, 2912, 450, 451, 450, 450, 451,
- 452, 2300, 2912, 452, 2912, 450, 450, 2912, 1255, 2912,
- 451, 1263, 1256, 2912, 452, 1264, 2912, 451, 2912, 2912,
-
- 2912, 452, 2912, 2912, 450, 2912, 2912, 2312, 2912, 2912,
- 2310, 2912, 450, 1265, 2912, 450, 450, 2313, 450, 450,
- 450, 2912, 2912, 450, 450, 450, 450, 450, 2912, 2315,
- 2314, 2912, 450, 450, 450, 450, 451, 2912, 451, 451,
- 452, 450, 452, 452, 2912, 2149, 2912, 2912, 991, 2150,
- 2912, 2912, 992, 2912, 2912, 2912, 2146, 2146, 2912, 451,
- 2912, 2912, 2912, 452, 450, 451, 450, 450, 2912, 452,
- 2912, 2912, 450, 450, 450, 450, 450, 2912, 2912, 2316,
- 450, 450, 450, 450, 450, 2912, 1994, 450, 2912, 450,
- 451, 2912, 2323, 450, 452, 450, 2716, 2319, 2718, 2912,
-
- 2716, 450, 2718, 450, 2912, 2912, 2912, 2324, 2912, 450,
- 2912, 2912, 2912, 2775, 2775, 2777, 2777, 453, 450, 453,
- 2912, 2912, 2912, 2912, 2912, 2912, 450, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 450, 2912, 883, 973, 974, 973,
+ 1607, 1607, 451, 2915, 991, 451, 452, 451, 992, 452,
+ 2915, 452, 2915, 2732, 2042, 2915, 2764, 2732, 2042, 451,
+ 2764, 2915, 2042, 452, 451, 2915, 2042, 2915, 452, 2915,
+ 450, 2915, 450, 450, 453, 450, 1454, 453, 450, 2157,
+ 450, 450, 2708, 450, 1454, 2159, 450, 450, 450, 2158,
+ 1084, 450, 450, 509, 1085, 450, 2915, 451, 2160, 2915,
+ 450, 452, 451, 450, 2768, 2915, 452, 2821, 450, 451,
+ 2042, 2821, 2915, 452, 2042, 451, 2915, 2915, 450, 452,
+ 2052, 2915, 2915, 2915, 2052, 450, 450, 2052, 2915, 2822,
+ 450, 2052, 1454, 450, 450, 2165, 2783, 450, 450, 2164,
+
+ 2783, 450, 1594, 450, 2915, 450, 450, 2915, 2169, 1594,
+ 2915, 450, 2167, 450, 2915, 2715, 2915, 453, 2091, 450,
+ 2171, 2172, 2171, 2171, 2171, 2173, 2174, 2171, 2174, 2171,
+ 2171, 2171, 2171, 2171, 2171, 2171, 2171, 2171, 2171, 2171,
+ 2171, 2175, 2176, 2171, 2171, 2171, 2171, 2171, 2171, 2171,
+ 2171, 2171, 2171, 2171, 2171, 2171, 2171, 2171, 2171, 2171,
+ 2171, 2171, 2171, 2171, 2171, 2171, 2171, 2171, 2171, 2171,
+ 2171, 2171, 2171, 2174, 2174, 2174, 2174, 2174, 2174, 2174,
+ 2174, 2174, 2174, 2174, 2174, 2174, 2174, 2174, 2174, 2174,
+ 2174, 2174, 2177, 2177, 2177, 2177, 2177, 2177, 2178, 2178,
+
+ 2178, 2179, 2174, 2174, 451, 1701, 451, 2784, 452, 1702,
+ 452, 2784, 2915, 2915, 2052, 1704, 2052, 451, 2052, 1705,
+ 2052, 452, 2915, 2915, 1707, 2915, 451, 1393, 1708, 2091,
+ 452, 1394, 450, 450, 450, 2182, 1594, 2180, 1594, 2915,
+ 450, 450, 450, 450, 2184, 450, 2185, 2915, 450, 450,
+ 450, 450, 450, 450, 450, 450, 2774, 2652, 2915, 450,
+ 450, 450, 450, 450, 2189, 2915, 2915, 2915, 450, 2187,
+ 450, 450, 451, 2915, 1713, 451, 452, 451, 1714, 452,
+ 2915, 452, 2915, 2808, 2851, 2915, 1100, 2808, 2852, 451,
+ 1101, 2915, 2915, 452, 2828, 2915, 2915, 2915, 2828, 2915,
+
+ 450, 2190, 450, 450, 453, 450, 2812, 2915, 450, 2915,
+ 450, 450, 2192, 450, 450, 453, 450, 450, 450, 450,
+ 1978, 450, 450, 2193, 1979, 450, 2915, 2197, 2915, 2915,
+ 450, 2198, 2194, 450, 2199, 2915, 2915, 451, 2200, 451,
+ 451, 452, 2915, 452, 452, 2849, 451, 2195, 450, 2849,
+ 452, 2915, 2915, 2915, 2915, 450, 450, 2821, 2915, 2915,
+ 2915, 2821, 450, 450, 450, 450, 453, 450, 450, 2204,
+ 450, 450, 871, 450, 450, 450, 450, 2915, 450, 2822,
+ 2915, 450, 450, 450, 450, 451, 2201, 451, 2915, 452,
+ 450, 452, 1993, 2915, 451, 2851, 1994, 451, 452, 2852,
+
+ 2915, 452, 2915, 2915, 2915, 2852, 2001, 451, 2915, 2852,
+ 2002, 452, 2206, 450, 2915, 450, 2207, 2812, 2915, 2915,
+ 450, 450, 450, 450, 2915, 450, 2816, 2211, 2209, 450,
+ 450, 450, 2212, 450, 450, 450, 450, 2915, 450, 2915,
+ 451, 450, 450, 450, 452, 2915, 2915, 2915, 2915, 1128,
+ 450, 450, 451, 1129, 451, 2915, 452, 2915, 452, 2915,
+ 2915, 451, 2915, 451, 2915, 452, 912, 452, 450, 2915,
+ 913, 1130, 2915, 2215, 2915, 2029, 450, 450, 2915, 2030,
+ 450, 2915, 450, 2217, 450, 450, 2915, 2216, 450, 450,
+ 450, 450, 2915, 450, 450, 2915, 450, 450, 450, 450,
+
+ 451, 2915, 450, 450, 452, 450, 2915, 450, 2219, 451,
+ 450, 450, 451, 452, 2915, 2915, 452, 2915, 451, 450,
+ 2814, 2915, 452, 2915, 2814, 2915, 2915, 2221, 450, 1128,
+ 451, 2915, 2915, 1129, 452, 2915, 450, 450, 2915, 1263,
+ 450, 2816, 2822, 1264, 450, 450, 450, 2915, 450, 2224,
+ 2223, 1130, 2915, 450, 450, 2226, 450, 450, 450, 451,
+ 2060, 1265, 450, 452, 2061, 450, 450, 450, 2915, 2915,
+ 2225, 2915, 2060, 450, 450, 450, 2061, 2198, 451, 2915,
+ 2227, 2198, 452, 450, 2915, 2915, 2915, 450, 450, 2915,
+ 2067, 2915, 2915, 2915, 2068, 450, 450, 451, 453, 2915,
+
+ 450, 452, 2355, 450, 450, 2915, 450, 1510, 450, 2244,
+ 1090, 1511, 2915, 2246, 450, 2915, 450, 2247, 450, 451,
+ 2915, 2356, 2248, 452, 2249, 450, 450, 1512, 1206, 2915,
+ 2915, 2915, 1207, 450, 450, 450, 2915, 2915, 2250, 2915,
+ 451, 450, 2915, 450, 452, 2251, 451, 450, 2915, 451,
+ 452, 450, 2915, 452, 2915, 450, 450, 2915, 2088, 2915,
+ 2915, 2265, 2089, 450, 450, 2266, 2915, 2259, 450, 2915,
+ 2915, 2262, 450, 2915, 450, 2915, 450, 450, 2915, 2261,
+ 2090, 2915, 450, 451, 450, 450, 450, 452, 2915, 450,
+ 450, 2915, 451, 450, 450, 2915, 452, 450, 2915, 2915,
+
+ 2915, 2915, 450, 2097, 451, 450, 1222, 2098, 452, 451,
+ 1223, 450, 2915, 452, 2915, 2915, 2268, 2915, 451, 450,
+ 450, 451, 452, 2915, 2915, 452, 2915, 450, 450, 2915,
+ 2915, 450, 450, 2915, 450, 2269, 450, 450, 2275, 450,
+ 450, 2915, 450, 2915, 2272, 450, 450, 450, 450, 450,
+ 450, 451, 451, 450, 450, 452, 452, 450, 2915, 2915,
+ 2915, 2277, 450, 2286, 2283, 2278, 2915, 2287, 2284, 2915,
+ 1243, 2915, 2915, 451, 1244, 2915, 2915, 452, 2915, 450,
+ 450, 2915, 451, 2915, 2915, 2282, 452, 450, 450, 2288,
+ 2915, 450, 450, 2915, 2281, 450, 450, 2285, 450, 450,
+
+ 450, 450, 2915, 2915, 2915, 2915, 450, 450, 450, 450,
+ 450, 451, 2915, 2915, 2293, 452, 451, 2294, 450, 2295,
+ 452, 451, 2915, 2915, 2915, 452, 450, 2915, 451, 2915,
+ 2915, 451, 452, 2717, 2915, 452, 2915, 2717, 2915, 450,
+ 2296, 451, 451, 2915, 450, 452, 452, 450, 2915, 450,
+ 2776, 2776, 450, 2915, 453, 450, 450, 450, 2298, 450,
+ 450, 2915, 2297, 2299, 450, 450, 2301, 450, 2915, 450,
+ 450, 2300, 450, 2915, 2915, 450, 451, 450, 450, 451,
+ 452, 2302, 2915, 452, 2915, 450, 450, 2915, 1255, 2915,
+ 451, 1263, 1256, 2915, 452, 1264, 2915, 451, 2915, 2915,
+
+ 2915, 452, 2915, 2915, 450, 2915, 2915, 2314, 2915, 2915,
+ 2312, 2915, 450, 1265, 2915, 450, 450, 2315, 450, 450,
+ 450, 2915, 2915, 450, 450, 450, 450, 450, 2915, 2317,
+ 2316, 2915, 450, 450, 450, 450, 451, 2915, 451, 451,
+ 452, 450, 452, 452, 2915, 2151, 2915, 2915, 991, 2152,
+ 2915, 2915, 992, 2915, 2915, 2915, 2148, 2148, 2915, 451,
+ 2915, 2915, 2915, 452, 450, 451, 450, 450, 2915, 452,
+ 2915, 2915, 450, 450, 450, 450, 450, 2915, 2915, 2318,
+ 450, 450, 450, 450, 450, 2915, 1996, 450, 2915, 450,
+ 451, 2915, 2325, 450, 452, 450, 2719, 2321, 2721, 2915,
+
+ 2719, 450, 2721, 450, 2915, 2915, 2915, 2326, 2915, 450,
+ 2915, 2915, 2915, 2778, 2778, 2780, 2780, 453, 450, 453,
+ 2915, 2915, 2915, 2915, 2915, 2915, 450, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 450, 2915, 883, 973, 974, 973,
973, 973, 975, 976, 973, 976, 973, 973, 973, 973,
973, 973, 973, 973, 973, 973, 973, 973, 977, 978,
973, 973, 973, 973, 973, 973, 973, 973, 973, 973,
@@ -3613,341 +3613,341 @@ static yyconst flex_int16_t yy_nxt[27006] =
976, 976, 976, 976, 976, 976, 976, 976, 976, 980,
980, 980, 980, 980, 980, 981, 981, 981, 982, 976,
- 976, 451, 451, 451, 451, 452, 452, 452, 452, 2912,
- 2912, 2912, 2912, 451, 2912, 2912, 2912, 452, 2912, 2912,
- 2912, 2912, 451, 2912, 2912, 2912, 452, 2912, 2330, 450,
- 450, 450, 450, 2912, 2912, 2332, 2912, 450, 450, 450,
- 450, 450, 2912, 871, 2328, 450, 450, 450, 450, 450,
- 450, 451, 1704, 2337, 2912, 452, 1705, 450, 450, 2912,
- 2912, 451, 2912, 451, 2912, 452, 450, 452, 2912, 2912,
- 2912, 2912, 451, 1393, 1713, 2912, 452, 1394, 1714, 450,
-
- 450, 2912, 2912, 2912, 2912, 2912, 1100, 450, 450, 450,
- 1101, 450, 2912, 2912, 2339, 450, 450, 450, 2343, 450,
- 450, 450, 2346, 2912, 2341, 450, 2912, 450, 450, 450,
- 450, 2912, 2912, 2912, 450, 2344, 450, 450, 450, 2347,
- 451, 2912, 450, 451, 452, 2195, 2912, 452, 2912, 2196,
- 450, 2912, 2197, 2912, 451, 2912, 2198, 451, 452, 2912,
- 2912, 452, 2912, 451, 2912, 2912, 2912, 452, 450, 2348,
- 2351, 450, 2912, 450, 451, 2912, 450, 2912, 452, 2349,
- 450, 450, 450, 2355, 450, 450, 2912, 450, 450, 2352,
- 450, 450, 2912, 450, 2912, 2912, 450, 2912, 450, 450,
-
- 2358, 450, 450, 2912, 451, 451, 2360, 450, 452, 452,
- 450, 2912, 2912, 2912, 2912, 451, 451, 2361, 450, 452,
- 452, 2912, 1140, 2912, 2912, 2912, 1141, 2912, 2912, 2363,
- 2912, 2912, 450, 450, 2912, 1128, 2912, 2912, 2365, 1129,
- 450, 450, 2366, 450, 2369, 2912, 2912, 2912, 450, 450,
- 450, 450, 450, 2912, 2912, 2912, 2912, 1130, 450, 450,
- 450, 912, 2912, 450, 451, 913, 450, 2912, 452, 2912,
- 2912, 450, 2912, 451, 451, 2912, 2370, 452, 452, 450,
- 2912, 2912, 2912, 451, 451, 2912, 1823, 452, 452, 450,
- 1824, 2912, 450, 2912, 2912, 2912, 2912, 450, 2371, 2375,
-
- 450, 450, 450, 2374, 2912, 450, 2373, 2912, 450, 450,
- 450, 450, 450, 2912, 450, 2912, 2912, 450, 450, 450,
- 450, 1916, 450, 2912, 2912, 1917, 2912, 450, 450, 2912,
- 450, 2058, 2912, 451, 2912, 2059, 2376, 452, 451, 2912,
- 451, 2912, 452, 2912, 452, 2912, 2912, 2912, 2912, 450,
- 451, 2912, 2912, 2912, 452, 2912, 2912, 450, 2912, 450,
- 2387, 450, 2912, 2390, 2912, 450, 450, 450, 450, 450,
- 2912, 2389, 2912, 2912, 450, 450, 450, 450, 450, 2391,
- 2912, 451, 450, 2912, 450, 452, 450, 1510, 2392, 2912,
- 2912, 1511, 451, 2912, 450, 2912, 452, 2912, 2912, 451,
-
- 2912, 2912, 451, 452, 2912, 2912, 452, 1512, 2912, 450,
- 2912, 2912, 2263, 2912, 2393, 450, 2264, 450, 2394, 2912,
- 450, 2912, 2912, 450, 2912, 450, 2398, 450, 450, 2401,
- 450, 450, 2400, 451, 2912, 450, 450, 452, 450, 2912,
- 450, 2912, 2912, 450, 2912, 451, 450, 2095, 450, 452,
- 451, 2096, 2912, 2912, 452, 2912, 450, 2912, 2404, 451,
- 2405, 450, 451, 452, 2912, 2912, 452, 2912, 451, 450,
- 2912, 2912, 452, 450, 2912, 450, 2912, 450, 450, 2912,
- 2406, 450, 2912, 450, 2912, 2912, 450, 450, 451, 450,
- 450, 450, 452, 2408, 450, 450, 450, 2412, 450, 2912,
-
- 2912, 2912, 2912, 450, 450, 2410, 450, 2281, 2912, 2912,
- 2281, 2282, 450, 2284, 2282, 2912, 450, 2285, 2912, 2912,
- 2912, 2912, 2284, 2912, 450, 1243, 2285, 2912, 2912, 1244,
- 2912, 2912, 450, 2912, 2912, 450, 2912, 2413, 450, 2912,
- 2912, 450, 2912, 450, 451, 2912, 450, 2414, 452, 450,
- 450, 450, 2912, 450, 450, 451, 2912, 450, 450, 452,
- 2415, 450, 2912, 2912, 2912, 2912, 450, 451, 2418, 450,
- 2912, 452, 450, 2912, 2912, 2912, 2912, 2419, 2912, 2912,
- 450, 451, 2420, 450, 2912, 452, 451, 2912, 450, 2912,
- 452, 450, 451, 451, 2912, 450, 452, 452, 451, 450,
-
- 2912, 2423, 452, 450, 2421, 2912, 2912, 2912, 2912, 450,
- 2422, 450, 451, 2912, 450, 451, 452, 450, 2912, 452,
- 450, 450, 450, 2912, 2912, 450, 450, 963, 450, 450,
- 450, 2912, 2912, 2912, 2425, 2436, 450, 2424, 2912, 2437,
- 450, 2426, 450, 450, 2912, 1255, 451, 2912, 450, 1256,
- 452, 450, 2912, 2912, 2912, 2912, 450, 451, 1263, 450,
- 451, 452, 1264, 450, 452, 2912, 2912, 2912, 2912, 2912,
- 2912, 450, 991, 450, 2438, 2912, 992, 2912, 2912, 450,
- 1265, 450, 450, 2442, 2912, 450, 450, 2443, 450, 450,
- 450, 987, 2912, 450, 450, 2912, 450, 2912, 2912, 2439,
+ 976, 451, 451, 451, 451, 452, 452, 452, 452, 2915,
+ 2915, 2915, 2915, 451, 2915, 2915, 2915, 452, 2915, 2915,
+ 2915, 2915, 451, 2915, 2915, 2915, 452, 2915, 2332, 450,
+ 450, 450, 450, 2915, 2915, 2334, 2915, 450, 450, 450,
+ 450, 450, 2915, 871, 2330, 450, 450, 450, 450, 450,
+ 450, 451, 1704, 2339, 2915, 452, 1705, 450, 450, 2915,
+ 2915, 451, 2915, 451, 2915, 452, 450, 452, 2915, 2915,
+ 2915, 2915, 451, 1393, 1713, 2915, 452, 1394, 1714, 450,
+
+ 450, 2915, 2915, 2915, 2915, 2915, 1100, 450, 450, 450,
+ 1101, 450, 2915, 2915, 2341, 450, 450, 450, 2345, 450,
+ 450, 450, 2348, 2915, 2343, 450, 2915, 450, 450, 450,
+ 450, 2915, 2915, 2915, 450, 2346, 450, 450, 450, 2349,
+ 451, 2915, 450, 451, 452, 2197, 2915, 452, 2915, 2198,
+ 450, 2915, 2199, 2915, 451, 2915, 2200, 451, 452, 2915,
+ 2915, 452, 2915, 451, 2915, 2915, 2915, 452, 450, 2350,
+ 2353, 450, 2915, 450, 451, 2915, 450, 2915, 452, 2351,
+ 450, 450, 450, 2357, 450, 450, 2915, 450, 450, 2354,
+ 450, 450, 2915, 450, 2915, 2915, 450, 2915, 450, 450,
+
+ 2360, 450, 450, 2915, 451, 451, 2362, 450, 452, 452,
+ 450, 2915, 2915, 2915, 2915, 451, 451, 2363, 450, 452,
+ 452, 2915, 1140, 2915, 2915, 2915, 1141, 2915, 2915, 2365,
+ 2915, 2915, 450, 450, 2915, 1128, 2915, 2915, 2367, 1129,
+ 450, 450, 2368, 450, 2371, 2915, 2915, 2915, 450, 450,
+ 450, 450, 450, 2915, 2915, 2915, 2915, 1130, 450, 450,
+ 450, 912, 2915, 450, 451, 913, 450, 2915, 452, 2915,
+ 2915, 450, 2915, 451, 451, 2915, 2372, 452, 452, 450,
+ 2915, 2915, 2915, 451, 1768, 2915, 1825, 452, 1769, 450,
+ 1826, 2915, 450, 2915, 2915, 2915, 2915, 450, 2373, 2377,
+
+ 450, 450, 450, 2376, 2915, 450, 2375, 2915, 450, 450,
+ 450, 450, 450, 2915, 450, 2915, 2915, 450, 450, 450,
+ 450, 1918, 450, 2915, 2915, 1919, 2915, 450, 450, 2915,
+ 450, 2060, 2915, 451, 2915, 2061, 2378, 452, 451, 2915,
+ 451, 2915, 452, 2915, 452, 2915, 2915, 2915, 2915, 450,
+ 451, 2915, 2915, 2915, 452, 2915, 2915, 450, 2915, 450,
+ 2390, 450, 2915, 2393, 2915, 450, 450, 450, 450, 450,
+ 2915, 2392, 2915, 2915, 450, 450, 450, 450, 450, 2394,
+ 2915, 451, 450, 2915, 450, 452, 450, 1510, 2395, 2915,
+ 2915, 1511, 451, 2915, 450, 2915, 452, 2915, 2915, 451,
+
+ 2915, 2915, 451, 452, 2915, 2915, 452, 1512, 2915, 450,
+ 2915, 2915, 2265, 2915, 2396, 450, 2266, 450, 2397, 2915,
+ 450, 2915, 2915, 450, 2915, 450, 2401, 450, 450, 2404,
+ 450, 450, 2403, 451, 2915, 450, 450, 452, 450, 2915,
+ 450, 2915, 2915, 450, 2915, 451, 450, 2097, 450, 452,
+ 451, 2098, 2915, 2915, 452, 2915, 450, 2915, 2407, 451,
+ 2408, 450, 451, 452, 2915, 2915, 452, 2915, 451, 450,
+ 2915, 2915, 452, 450, 2915, 450, 2915, 450, 450, 2915,
+ 2409, 450, 2915, 450, 2915, 2915, 450, 450, 451, 450,
+ 450, 450, 452, 2411, 450, 450, 450, 2415, 450, 2915,
+
+ 2915, 2915, 2915, 450, 450, 2413, 450, 2283, 2915, 2915,
+ 2283, 2284, 450, 2286, 2284, 2915, 450, 2287, 2915, 2915,
+ 2915, 2915, 2286, 2915, 450, 1243, 2287, 2915, 2915, 1244,
+ 2915, 2915, 450, 2915, 2915, 450, 2915, 2416, 450, 2915,
+ 2915, 450, 2915, 450, 451, 2915, 450, 2417, 452, 450,
+ 450, 450, 2915, 450, 450, 451, 2915, 450, 450, 452,
+ 2418, 450, 2915, 2915, 2915, 2915, 450, 451, 2421, 450,
+ 2915, 452, 450, 2915, 2915, 2915, 2915, 2422, 2915, 2915,
+ 450, 451, 2423, 450, 2915, 452, 451, 2915, 450, 2915,
+ 452, 450, 451, 451, 2915, 450, 452, 452, 451, 450,
+
+ 2915, 2426, 452, 450, 2424, 2915, 2915, 2915, 2915, 450,
+ 2425, 450, 451, 2915, 450, 451, 452, 450, 2915, 452,
+ 450, 450, 450, 2915, 2915, 450, 450, 963, 450, 450,
+ 450, 2915, 2915, 2915, 2428, 2439, 450, 2427, 2915, 2440,
+ 450, 2429, 450, 450, 2915, 1255, 451, 2915, 450, 1256,
+ 452, 450, 2915, 2915, 2915, 2915, 450, 451, 1263, 450,
+ 451, 452, 1264, 450, 452, 2915, 2915, 2915, 2915, 2915,
+ 2915, 450, 991, 450, 2441, 2915, 992, 2915, 2915, 450,
+ 1265, 450, 450, 2445, 2915, 450, 450, 2446, 450, 450,
+ 450, 987, 2915, 450, 450, 2915, 450, 2915, 2915, 2442,
450, 450, 450, 451, 450, 451, 451, 452, 450, 452,
- 452, 450, 2912, 2912, 2441, 2912, 450, 451, 451, 450,
- 451, 452, 452, 2912, 452, 2912, 2912, 450, 2446, 2912,
- 2912, 450, 2447, 450, 450, 2912, 2912, 2912, 2912, 450,
- 1393, 450, 2449, 2912, 1394, 450, 450, 450, 450, 450,
- 450, 451, 2912, 2452, 450, 452, 450, 2912, 2912, 2912,
- 2912, 450, 450, 1713, 450, 1100, 2457, 1714, 450, 1101,
- 2458, 2912, 2912, 2912, 2912, 2912, 450, 2912, 2912, 450,
- 2912, 2912, 2912, 2454, 450, 2912, 2912, 450, 451, 2912,
- 2912, 450, 452, 450, 450, 450, 2912, 2912, 2912, 450,
-
- 451, 450, 450, 2912, 452, 2912, 2912, 2455, 2912, 450,
- 450, 451, 2912, 2459, 2456, 452, 450, 1113, 451, 2912,
- 2912, 1114, 452, 2467, 450, 2912, 2912, 2468, 450, 2469,
- 2912, 2912, 450, 2470, 2912, 2912, 2461, 2912, 451, 450,
- 2912, 2912, 452, 2912, 450, 450, 450, 450, 2462, 2912,
- 2912, 450, 451, 450, 450, 450, 452, 450, 2912, 450,
- 2912, 450, 450, 2912, 451, 450, 450, 450, 452, 2912,
- 2912, 2912, 2912, 450, 450, 2474, 2912, 451, 2912, 2472,
- 450, 452, 450, 2912, 2912, 2912, 451, 1128, 450, 451,
- 452, 1129, 450, 452, 2912, 2912, 450, 2912, 2475, 451,
-
- 450, 2912, 2912, 452, 2912, 450, 2478, 2912, 450, 1130,
- 2912, 2912, 1128, 450, 450, 2479, 1129, 450, 2912, 2912,
- 2912, 450, 450, 450, 1263, 450, 2912, 1970, 1264, 2912,
- 450, 450, 2912, 450, 1130, 450, 2480, 2058, 451, 2912,
- 450, 2059, 452, 450, 2912, 2912, 1265, 451, 450, 451,
- 451, 452, 450, 452, 452, 2912, 450, 2912, 2002, 451,
- 450, 2912, 2912, 452, 2912, 2488, 450, 2912, 450, 2912,
- 2142, 2912, 2912, 450, 450, 450, 2912, 450, 450, 2912,
- 2491, 450, 450, 450, 2490, 450, 450, 450, 2492, 2912,
- 2912, 450, 1510, 450, 450, 450, 1511, 2494, 451, 2912,
-
- 2493, 451, 452, 450, 451, 452, 2912, 451, 452, 2912,
- 2912, 452, 1512, 2912, 2912, 2912, 2912, 2912, 451, 2095,
- 450, 2505, 452, 2096, 2912, 2506, 450, 2912, 450, 450,
- 2912, 2912, 450, 2912, 450, 450, 450, 450, 2497, 2501,
- 450, 2498, 450, 450, 2503, 450, 450, 450, 450, 450,
- 2912, 450, 451, 2912, 450, 450, 452, 450, 2912, 2912,
- 2502, 2912, 450, 450, 451, 450, 451, 2912, 452, 2281,
- 452, 2284, 2912, 2282, 2912, 2285, 2912, 2912, 2912, 2912,
- 450, 2912, 2912, 2912, 2912, 2912, 451, 2912, 450, 1243,
- 452, 2912, 450, 1244, 450, 2509, 450, 450, 2510, 450,
-
- 450, 2912, 450, 2912, 2912, 450, 2511, 450, 450, 2912,
- 450, 2508, 451, 450, 450, 450, 452, 450, 2514, 2912,
- 2912, 2912, 450, 451, 451, 450, 2912, 452, 452, 2912,
- 450, 2515, 2912, 450, 451, 2912, 2912, 2912, 452, 2912,
- 450, 2912, 2912, 2912, 2912, 451, 2912, 2912, 450, 452,
- 2912, 450, 450, 2912, 451, 2516, 450, 2436, 452, 450,
- 450, 2437, 450, 2912, 2912, 2518, 2517, 450, 450, 2912,
- 450, 2519, 451, 450, 2520, 451, 452, 2912, 450, 452,
- 2521, 450, 450, 2912, 1263, 450, 2912, 2531, 1264, 450,
- 450, 2912, 2912, 450, 2912, 991, 2912, 2912, 450, 992,
-
- 450, 450, 2912, 450, 2533, 2442, 1265, 2912, 450, 2443,
- 451, 450, 2534, 2522, 452, 2912, 450, 451, 2912, 450,
- 450, 452, 451, 450, 2912, 2912, 452, 2912, 450, 2912,
- 2912, 450, 451, 450, 2912, 2912, 452, 2539, 450, 450,
- 2912, 450, 2912, 2912, 1393, 450, 450, 2542, 1394, 2536,
- 450, 2540, 2912, 450, 450, 1713, 2912, 2544, 450, 1714,
- 450, 450, 2912, 2912, 2912, 1100, 450, 2457, 450, 1101,
- 451, 2458, 450, 2546, 452, 2912, 450, 2912, 2912, 2912,
- 450, 2912, 2547, 450, 2912, 2912, 2912, 2912, 450, 2912,
- 2912, 450, 451, 450, 2548, 450, 452, 2549, 450, 450,
-
- 2912, 450, 2912, 450, 451, 2912, 450, 2912, 452, 450,
- 2912, 450, 2912, 2551, 450, 2467, 2912, 2469, 2553, 2468,
- 450, 2470, 451, 2912, 2912, 2912, 452, 451, 450, 2554,
- 2912, 452, 450, 2912, 2564, 2912, 450, 2568, 2565, 2912,
- 450, 2569, 2912, 450, 2912, 450, 2912, 2912, 450, 2912,
- 450, 450, 2559, 450, 2561, 450, 2563, 2912, 450, 450,
- 1128, 450, 450, 450, 1129, 450, 450, 2912, 2912, 2912,
- 450, 450, 451, 450, 2912, 2058, 452, 2912, 450, 2059,
- 2912, 450, 1130, 2912, 2912, 451, 2912, 451, 450, 452,
- 2912, 452, 2912, 2912, 451, 2912, 450, 451, 452, 2912,
-
- 450, 452, 2570, 450, 450, 2912, 451, 2912, 450, 2912,
- 452, 450, 2912, 450, 2579, 450, 450, 2912, 2912, 450,
- 451, 450, 2582, 450, 452, 450, 2581, 2912, 2580, 450,
- 450, 450, 2586, 450, 450, 2585, 2587, 2912, 450, 451,
- 2912, 450, 450, 452, 2912, 2095, 2912, 2912, 450, 2096,
- 450, 2505, 2912, 2590, 1240, 2506, 450, 451, 451, 2912,
- 450, 452, 452, 2912, 450, 2912, 2912, 450, 450, 2912,
- 2281, 2592, 2912, 450, 2282, 450, 450, 2912, 2912, 450,
- 2284, 450, 2591, 450, 2285, 450, 450, 450, 2912, 450,
- 2912, 2912, 1243, 450, 450, 450, 1244, 2912, 2595, 451,
-
- 2594, 450, 450, 452, 2912, 451, 450, 2912, 450, 452,
- 2912, 451, 2912, 2912, 450, 452, 450, 451, 2912, 2912,
- 450, 452, 451, 2912, 450, 2912, 452, 450, 450, 2912,
- 2597, 2912, 2598, 450, 451, 450, 450, 2600, 452, 450,
- 2912, 450, 2912, 450, 2599, 450, 2912, 450, 451, 450,
- 450, 2912, 452, 450, 2912, 450, 2912, 2912, 450, 451,
- 2615, 2601, 450, 452, 2616, 451, 450, 2912, 2912, 452,
- 450, 2602, 2912, 2912, 2604, 2912, 450, 2912, 450, 2912,
- 2912, 2603, 2912, 2912, 450, 1263, 451, 450, 450, 1264,
- 452, 2912, 450, 450, 2912, 450, 450, 2912, 451, 2912,
-
- 2613, 450, 452, 450, 450, 2912, 2912, 1265, 451, 450,
- 2912, 451, 452, 450, 450, 452, 2912, 2912, 2912, 2912,
- 451, 450, 450, 2912, 452, 2620, 450, 2617, 2912, 450,
- 450, 1393, 2622, 1713, 450, 1394, 450, 1714, 2912, 450,
- 2912, 2912, 450, 2912, 450, 1100, 2912, 450, 450, 1101,
- 451, 2912, 450, 2912, 452, 450, 450, 2912, 2912, 450,
- 2912, 450, 451, 2912, 450, 2912, 452, 450, 451, 450,
- 2912, 2912, 452, 450, 2912, 450, 2623, 450, 450, 451,
- 451, 450, 2912, 452, 452, 2624, 450, 2912, 2912, 450,
- 450, 451, 2912, 451, 450, 452, 450, 452, 450, 2912,
-
- 2912, 2628, 2912, 2626, 450, 2912, 450, 450, 450, 2634,
- 2564, 2912, 450, 2568, 2565, 2629, 450, 2569, 2912, 2638,
- 2912, 450, 2912, 450, 450, 2912, 2636, 450, 451, 450,
- 1128, 2912, 452, 2912, 1129, 450, 2912, 450, 450, 451,
- 451, 450, 2912, 452, 452, 2912, 450, 2912, 2912, 450,
- 2640, 451, 1130, 2912, 450, 452, 450, 450, 450, 2912,
- 2912, 451, 2912, 2912, 450, 452, 450, 450, 450, 2650,
- 2912, 2912, 450, 2912, 450, 450, 450, 2912, 2912, 450,
- 2912, 2651, 2653, 450, 450, 2912, 2586, 450, 2912, 2655,
- 2587, 2912, 2652, 2912, 2912, 450, 451, 450, 451, 2661,
-
- 452, 2912, 452, 2662, 2912, 450, 2912, 2912, 1240, 2912,
- 2912, 2912, 2912, 2095, 450, 2912, 2281, 2096, 2912, 2663,
- 2282, 2912, 450, 2912, 450, 2912, 450, 450, 2912, 2912,
- 450, 2912, 450, 451, 450, 450, 2912, 452, 2912, 2658,
- 450, 450, 450, 450, 450, 2657, 451, 2912, 2659, 450,
- 452, 451, 450, 2912, 2912, 452, 2912, 450, 2912, 2912,
- 450, 450, 2912, 451, 2912, 2912, 2912, 452, 2664, 450,
- 2912, 2912, 2912, 2912, 450, 2912, 451, 450, 2912, 450,
- 452, 451, 450, 2912, 2665, 452, 2912, 450, 2912, 2912,
- 450, 450, 2912, 2912, 2666, 450, 2912, 2912, 451, 450,
-
- 2912, 2912, 452, 2912, 450, 2912, 2912, 2667, 2912, 450,
- 451, 2668, 450, 2912, 452, 451, 2912, 450, 2912, 452,
- 450, 2912, 2912, 2912, 2669, 450, 450, 2615, 2912, 2670,
- 2912, 2616, 2912, 2912, 450, 2912, 2912, 451, 450, 2912,
- 2680, 452, 450, 450, 1263, 2912, 450, 2912, 1264, 2686,
- 2912, 450, 2912, 2687, 2671, 450, 1713, 2912, 451, 450,
- 1714, 2912, 452, 450, 2682, 450, 1265, 2912, 2912, 2912,
- 451, 450, 450, 450, 452, 2912, 2684, 450, 2912, 2912,
- 450, 450, 2912, 2688, 450, 450, 450, 451, 450, 2912,
- 2912, 452, 450, 450, 450, 2912, 2912, 2912, 450, 2912,
-
- 450, 451, 450, 2690, 451, 452, 450, 451, 452, 2912,
- 2912, 452, 2912, 2700, 450, 450, 2912, 2701, 2692, 2912,
- 2912, 2912, 2713, 450, 451, 2912, 2714, 2693, 452, 450,
- 2912, 450, 450, 2912, 451, 450, 2912, 450, 452, 2912,
- 450, 450, 2912, 450, 2698, 450, 2912, 2696, 450, 450,
- 450, 450, 450, 2703, 2912, 2715, 2912, 450, 450, 2716,
- 450, 2717, 450, 2912, 2912, 2718, 450, 2720, 450, 2912,
- 450, 2721, 451, 2912, 2912, 2912, 452, 2912, 450, 2912,
- 2912, 451, 2661, 450, 2095, 452, 2662, 2912, 2096, 450,
- 2912, 450, 2912, 2912, 2728, 450, 2912, 450, 2729, 450,
-
- 450, 2723, 2663, 450, 2912, 450, 2912, 2912, 450, 450,
- 450, 450, 450, 2725, 2724, 451, 450, 450, 450, 452,
- 450, 2912, 450, 2912, 2912, 450, 450, 451, 450, 451,
- 450, 452, 451, 452, 2912, 2912, 452, 2912, 450, 2912,
- 2912, 2912, 2912, 450, 2730, 2912, 451, 2912, 2912, 2912,
- 452, 450, 451, 2912, 2912, 450, 452, 450, 451, 450,
- 450, 2912, 452, 450, 2731, 450, 2912, 2912, 450, 2732,
- 2912, 450, 2912, 450, 450, 451, 450, 960, 2912, 452,
- 450, 962, 450, 2912, 2912, 2733, 450, 451, 450, 2912,
- 450, 452, 2912, 451, 450, 2912, 450, 452, 2912, 2912,
-
- 2912, 2912, 2734, 450, 451, 2912, 451, 2912, 452, 2912,
- 452, 450, 2735, 451, 2912, 450, 2912, 452, 2744, 450,
- 2912, 450, 2746, 450, 451, 2912, 451, 2912, 452, 450,
- 452, 450, 450, 2748, 450, 2912, 2912, 450, 2912, 451,
- 450, 450, 450, 452, 2750, 2912, 2912, 2912, 450, 450,
- 450, 451, 450, 2912, 450, 452, 2912, 450, 2912, 2912,
- 450, 451, 450, 2912, 2912, 452, 2912, 450, 450, 2912,
- 2754, 2752, 2755, 2760, 2713, 450, 451, 2761, 2714, 450,
- 452, 2912, 2912, 450, 2912, 2912, 2912, 450, 2912, 450,
- 2758, 2772, 2772, 2912, 2779, 450, 2715, 450, 2780, 1216,
-
- 2716, 450, 450, 1217, 450, 450, 2912, 2912, 2912, 450,
- 450, 2717, 450, 2774, 2774, 2718, 2088, 450, 450, 2763,
- 450, 451, 450, 2912, 450, 452, 2912, 450, 2776, 2776,
- 450, 2912, 450, 2095, 2912, 450, 2912, 2096, 450, 450,
- 450, 2912, 2728, 450, 451, 2912, 2729, 450, 452, 450,
- 2912, 2912, 2912, 2912, 2912, 450, 2912, 450, 2782, 451,
- 2912, 450, 2912, 452, 451, 450, 2912, 2912, 452, 450,
- 450, 2912, 450, 2912, 2912, 2783, 2912, 450, 450, 451,
- 450, 2786, 2912, 452, 2787, 2912, 450, 450, 450, 451,
- 2912, 2912, 450, 452, 2912, 450, 2912, 2788, 451, 2912,
-
- 450, 451, 452, 450, 2789, 452, 2912, 450, 450, 2912,
- 451, 2912, 2912, 2912, 452, 450, 2798, 450, 2912, 2912,
- 451, 2912, 2912, 450, 452, 450, 450, 2790, 2912, 450,
- 2912, 2791, 2804, 450, 450, 2912, 2805, 450, 450, 2912,
- 2912, 2912, 450, 2912, 2912, 450, 450, 2912, 450, 2912,
- 2912, 2912, 2912, 2800, 450, 2912, 450, 2912, 2912, 2912,
- 450, 2802, 2912, 2912, 450, 2912, 2912, 2912, 450, 2912,
- 2912, 2912, 2912, 2912, 2806, 2912, 450, 2809, 2810, 2809,
- 2809, 2809, 2811, 2812, 2809, 2812, 2809, 2809, 2809, 2809,
- 2809, 2809, 2809, 2809, 2809, 2809, 2809, 2809, 2809, 2813,
-
- 2814, 2809, 2809, 2809, 2809, 2809, 2809, 2809, 2809, 2809,
- 2809, 2809, 2809, 2809, 2809, 2809, 2809, 2809, 2809, 2809,
- 2809, 2809, 2809, 2809, 2809, 2809, 2809, 2809, 2809, 2809,
+ 452, 450, 2915, 2915, 2444, 2915, 450, 451, 451, 450,
+ 451, 452, 452, 2915, 452, 2915, 2915, 450, 2449, 2915,
+ 2915, 450, 2450, 450, 450, 2915, 2915, 2915, 2915, 450,
+ 1393, 450, 2452, 2915, 1394, 450, 450, 450, 450, 450,
+ 450, 451, 2915, 2455, 450, 452, 450, 2915, 2915, 2915,
+ 2915, 450, 450, 1713, 450, 1100, 2460, 1714, 450, 1101,
+ 2461, 2915, 2915, 2915, 2915, 2915, 450, 2915, 2915, 450,
+ 2915, 2915, 2915, 2457, 450, 2915, 2915, 450, 451, 2915,
+ 2915, 450, 452, 450, 450, 450, 2915, 2915, 2915, 450,
+
+ 451, 450, 450, 2915, 452, 2915, 2915, 2458, 2915, 450,
+ 450, 451, 2915, 2462, 2459, 452, 450, 1113, 451, 2915,
+ 2915, 1114, 452, 2470, 450, 2915, 2915, 2471, 450, 2472,
+ 2915, 2915, 450, 2473, 2915, 2915, 2464, 2915, 451, 450,
+ 2915, 2915, 452, 2915, 450, 450, 450, 450, 2465, 2915,
+ 2915, 450, 451, 450, 450, 450, 452, 450, 2915, 450,
+ 2915, 450, 450, 2915, 451, 450, 450, 450, 452, 2915,
+ 2915, 2915, 2915, 450, 450, 2477, 2915, 451, 2915, 2475,
+ 450, 452, 450, 2915, 2915, 2915, 451, 1128, 450, 451,
+ 452, 1129, 450, 452, 2915, 2915, 450, 2915, 2478, 451,
+
+ 450, 2915, 2915, 452, 2915, 450, 2481, 2915, 450, 1130,
+ 2915, 2915, 1128, 450, 450, 2482, 1129, 450, 2915, 2915,
+ 2915, 450, 450, 450, 1263, 450, 2915, 1972, 1264, 2915,
+ 450, 450, 2915, 450, 1130, 450, 2483, 2060, 451, 2915,
+ 450, 2061, 452, 450, 2915, 2915, 1265, 451, 450, 451,
+ 451, 452, 450, 452, 452, 2915, 450, 2915, 2004, 451,
+ 450, 2915, 2915, 452, 2915, 2491, 450, 2915, 450, 2915,
+ 2144, 2915, 2915, 450, 450, 450, 2915, 450, 450, 2915,
+ 2494, 450, 450, 450, 2493, 450, 450, 450, 2495, 2915,
+ 2915, 450, 1510, 450, 450, 450, 1511, 2497, 451, 2915,
+
+ 2496, 451, 452, 450, 451, 452, 2915, 451, 452, 2915,
+ 2915, 452, 1512, 2915, 2915, 2915, 2915, 2915, 451, 2097,
+ 450, 2508, 452, 2098, 2915, 2509, 450, 2915, 450, 450,
+ 2915, 2915, 450, 2915, 450, 450, 450, 450, 2500, 2504,
+ 450, 2501, 450, 450, 2506, 450, 450, 450, 450, 450,
+ 2915, 450, 451, 2915, 450, 450, 452, 450, 2915, 2915,
+ 2505, 2915, 450, 450, 451, 450, 451, 2915, 452, 2283,
+ 452, 2286, 2915, 2284, 2915, 2287, 2915, 2915, 2915, 2915,
+ 450, 2915, 2915, 2915, 2915, 2915, 451, 2915, 450, 1243,
+ 452, 2915, 450, 1244, 450, 2512, 450, 450, 2513, 450,
+
+ 450, 2915, 450, 2915, 2915, 450, 2514, 450, 450, 2915,
+ 450, 2511, 451, 450, 450, 450, 452, 450, 2517, 2915,
+ 2915, 2915, 450, 451, 451, 450, 2915, 452, 452, 2915,
+ 450, 2518, 2915, 450, 451, 2915, 2915, 2915, 452, 2915,
+ 450, 2915, 2915, 2915, 2915, 451, 2915, 2915, 450, 452,
+ 2915, 450, 450, 2915, 451, 2519, 450, 2439, 452, 450,
+ 450, 2440, 450, 2915, 2915, 2521, 2520, 450, 450, 2915,
+ 450, 2522, 451, 450, 2523, 451, 452, 2915, 450, 452,
+ 2524, 450, 450, 2915, 1263, 450, 2915, 2534, 1264, 450,
+ 450, 2915, 2915, 450, 2915, 991, 2915, 2915, 450, 992,
+
+ 450, 450, 2915, 450, 2536, 2445, 1265, 2915, 450, 2446,
+ 451, 450, 2537, 2525, 452, 2915, 450, 451, 2915, 450,
+ 450, 452, 451, 450, 2915, 2915, 452, 2915, 450, 2915,
+ 2915, 450, 451, 450, 2915, 2915, 452, 2542, 450, 450,
+ 2915, 450, 2915, 2915, 1393, 450, 450, 2545, 1394, 2539,
+ 450, 2543, 2915, 450, 450, 1713, 2915, 2547, 450, 1714,
+ 450, 450, 2915, 2915, 2915, 1100, 450, 2460, 450, 1101,
+ 451, 2461, 450, 2549, 452, 2915, 450, 2915, 2915, 2915,
+ 450, 2915, 2550, 450, 2915, 2915, 2915, 2915, 450, 2915,
+ 2915, 450, 451, 450, 2551, 450, 452, 2552, 450, 450,
+
+ 2915, 450, 2915, 450, 451, 2915, 450, 2915, 452, 450,
+ 2915, 450, 2915, 2554, 450, 2470, 2915, 2472, 2556, 2471,
+ 450, 2473, 451, 2915, 2915, 2915, 452, 451, 450, 2557,
+ 2915, 452, 450, 2915, 2567, 2915, 450, 2571, 2568, 2915,
+ 450, 2572, 2915, 450, 2915, 450, 2915, 2915, 450, 2915,
+ 450, 450, 2562, 450, 2564, 450, 2566, 2915, 450, 450,
+ 1128, 450, 450, 450, 1129, 450, 450, 2915, 2915, 2915,
+ 450, 450, 451, 450, 2915, 2060, 452, 2915, 450, 2061,
+ 2915, 450, 1130, 2915, 2915, 451, 2915, 451, 450, 452,
+ 2915, 452, 2915, 2915, 451, 2915, 450, 451, 452, 2915,
+
+ 450, 452, 2573, 450, 450, 2915, 451, 2915, 450, 2915,
+ 452, 450, 2915, 450, 2582, 450, 450, 2915, 2915, 450,
+ 451, 450, 2585, 450, 452, 450, 2584, 2915, 2583, 450,
+ 450, 450, 2589, 450, 450, 2588, 2590, 2915, 450, 451,
+ 2915, 450, 450, 452, 2915, 2097, 2915, 2915, 450, 2098,
+ 450, 2508, 2915, 2593, 1240, 2509, 450, 451, 451, 2915,
+ 450, 452, 452, 2915, 450, 2915, 2915, 450, 450, 2915,
+ 2283, 2595, 2915, 450, 2284, 450, 450, 2915, 2915, 450,
+ 2286, 450, 2594, 450, 2287, 450, 450, 450, 2915, 450,
+ 2915, 2915, 1243, 450, 450, 450, 1244, 2915, 2598, 451,
+
+ 2597, 450, 450, 452, 2915, 451, 450, 2915, 450, 452,
+ 2915, 451, 2915, 2915, 450, 452, 450, 451, 2915, 2915,
+ 450, 452, 451, 2915, 450, 2915, 452, 450, 450, 2915,
+ 2600, 2915, 2601, 450, 451, 450, 450, 2603, 452, 450,
+ 2915, 450, 2915, 450, 2602, 450, 2915, 450, 451, 450,
+ 450, 2915, 452, 450, 2915, 450, 2915, 2915, 450, 451,
+ 2618, 2604, 450, 452, 2619, 451, 450, 2915, 2915, 452,
+ 450, 2605, 2915, 2915, 2607, 2915, 450, 2915, 450, 2915,
+ 2915, 2606, 2915, 2915, 450, 1263, 451, 450, 450, 1264,
+ 452, 2915, 450, 450, 2915, 450, 450, 2915, 451, 2915,
+
+ 2616, 450, 452, 450, 450, 2915, 2915, 1265, 451, 450,
+ 2915, 451, 452, 450, 450, 452, 2915, 2915, 2915, 2915,
+ 451, 450, 450, 2915, 452, 2623, 450, 2620, 2915, 450,
+ 450, 1393, 2625, 1713, 450, 1394, 450, 1714, 2915, 450,
+ 2915, 2915, 450, 2915, 450, 1100, 2915, 450, 450, 1101,
+ 451, 2915, 450, 2915, 452, 450, 450, 2915, 2915, 450,
+ 2915, 450, 451, 2915, 450, 2915, 452, 450, 451, 450,
+ 2915, 2915, 452, 450, 2915, 450, 2626, 450, 450, 451,
+ 451, 450, 2915, 452, 452, 2627, 450, 2915, 2915, 450,
+ 450, 451, 2915, 451, 450, 452, 450, 452, 450, 2915,
+
+ 2915, 2631, 2915, 2629, 450, 2915, 450, 450, 450, 2637,
+ 2567, 2915, 450, 2571, 2568, 2632, 450, 2572, 2915, 2641,
+ 2915, 450, 2915, 450, 450, 2915, 2639, 450, 451, 450,
+ 1128, 2915, 452, 2915, 1129, 450, 2915, 450, 450, 451,
+ 451, 450, 2915, 452, 452, 2915, 450, 2915, 2915, 450,
+ 2643, 451, 1130, 2915, 450, 452, 450, 450, 450, 2915,
+ 2915, 451, 2915, 2915, 450, 452, 450, 450, 450, 2653,
+ 2915, 2915, 450, 2915, 450, 450, 450, 2915, 2915, 450,
+ 2915, 2654, 2656, 450, 450, 2915, 2589, 450, 2915, 2658,
+ 2590, 2915, 2655, 2915, 2915, 450, 451, 450, 451, 2664,
+
+ 452, 2915, 452, 2665, 2915, 450, 2915, 2915, 1240, 2915,
+ 2915, 2915, 2915, 2097, 450, 2915, 2283, 2098, 2915, 2666,
+ 2284, 2915, 450, 2915, 450, 2915, 450, 450, 2915, 2915,
+ 450, 2915, 450, 451, 450, 450, 2915, 452, 2915, 2661,
+ 450, 450, 450, 450, 450, 2660, 451, 2915, 2662, 450,
+ 452, 451, 450, 2915, 2915, 452, 2915, 450, 2915, 2915,
+ 450, 450, 2915, 451, 2915, 2915, 2915, 452, 2667, 450,
+ 2915, 2915, 2915, 2915, 450, 2915, 451, 450, 2915, 450,
+ 452, 451, 450, 2915, 2668, 452, 2915, 450, 2915, 2915,
+ 450, 450, 2915, 2915, 2669, 450, 2915, 2915, 451, 450,
+
+ 2915, 2915, 452, 2915, 450, 2915, 2915, 2670, 2915, 450,
+ 451, 2671, 450, 2915, 452, 451, 2915, 450, 2915, 452,
+ 450, 2915, 2915, 2915, 2672, 450, 450, 2618, 2915, 2673,
+ 2915, 2619, 2915, 2915, 450, 2915, 2915, 451, 450, 2915,
+ 2683, 452, 450, 450, 1263, 2915, 450, 2915, 1264, 2689,
+ 2915, 450, 2915, 2690, 2674, 450, 1713, 2915, 451, 450,
+ 1714, 2915, 452, 450, 2685, 450, 1265, 2915, 2915, 2915,
+ 451, 450, 450, 450, 452, 2915, 2687, 450, 2915, 2915,
+ 450, 450, 2915, 2691, 450, 450, 450, 451, 450, 2915,
+ 2915, 452, 450, 450, 450, 2915, 2915, 2915, 450, 2915,
+
+ 450, 451, 450, 2693, 451, 452, 450, 451, 452, 2915,
+ 2915, 452, 2915, 2703, 450, 450, 2915, 2704, 2695, 2915,
+ 2915, 2915, 2716, 450, 451, 2915, 2717, 2696, 452, 450,
+ 2915, 450, 450, 2915, 451, 450, 2915, 450, 452, 2915,
+ 450, 450, 2915, 450, 2701, 450, 2915, 2699, 450, 450,
+ 450, 450, 450, 2706, 2915, 2718, 2915, 450, 450, 2719,
+ 450, 2720, 450, 2915, 2915, 2721, 450, 2723, 450, 2915,
+ 450, 2724, 451, 2915, 2915, 2915, 452, 2915, 450, 2915,
+ 2915, 451, 2664, 450, 2097, 452, 2665, 2915, 2098, 450,
+ 2915, 450, 2915, 2915, 2731, 450, 2915, 450, 2732, 450,
+
+ 450, 2726, 2666, 450, 2915, 450, 2915, 2915, 450, 450,
+ 450, 450, 450, 2728, 2727, 451, 450, 450, 450, 452,
+ 450, 2915, 450, 2915, 2915, 450, 450, 451, 450, 451,
+ 450, 452, 451, 452, 2915, 2915, 452, 2915, 450, 2915,
+ 2915, 2915, 2915, 450, 2733, 2915, 451, 2915, 2915, 2915,
+ 452, 450, 451, 2915, 2915, 450, 452, 450, 451, 450,
+ 450, 2915, 452, 450, 2734, 450, 2915, 2915, 450, 2735,
+ 2915, 450, 2915, 450, 450, 451, 450, 960, 2915, 452,
+ 450, 962, 450, 2915, 2915, 2736, 450, 451, 450, 2915,
+ 450, 452, 2915, 451, 450, 2915, 450, 452, 2915, 2915,
+
+ 2915, 2915, 2737, 450, 451, 2915, 451, 2915, 452, 2915,
+ 452, 450, 2738, 451, 2915, 450, 2915, 452, 2747, 450,
+ 2915, 450, 2749, 450, 451, 2915, 451, 2915, 452, 450,
+ 452, 450, 450, 2751, 450, 2915, 2915, 450, 2915, 451,
+ 450, 450, 450, 452, 2753, 2915, 2915, 2915, 450, 450,
+ 450, 451, 450, 2915, 450, 452, 2915, 450, 2915, 2915,
+ 450, 451, 450, 2915, 2915, 452, 2915, 450, 450, 2915,
+ 2757, 2755, 2758, 2763, 2716, 450, 451, 2764, 2717, 450,
+ 452, 2915, 2915, 450, 2915, 2915, 2915, 450, 2915, 450,
+ 2761, 2775, 2775, 2915, 2782, 450, 2718, 450, 2783, 1216,
+
+ 2719, 450, 450, 1217, 450, 450, 2915, 2915, 2915, 450,
+ 450, 2720, 450, 2777, 2777, 2721, 2090, 450, 450, 2766,
+ 450, 451, 450, 2915, 450, 452, 2915, 450, 2779, 2779,
+ 450, 2915, 450, 2097, 2915, 450, 2915, 2098, 450, 450,
+ 450, 2915, 2731, 450, 451, 2915, 2732, 450, 452, 450,
+ 2915, 2915, 2915, 2915, 2915, 450, 2915, 450, 2785, 451,
+ 2915, 450, 2915, 452, 451, 450, 2915, 2915, 452, 450,
+ 450, 2915, 450, 2915, 2915, 2786, 2915, 450, 450, 451,
+ 450, 2789, 2915, 452, 2790, 2915, 450, 450, 450, 451,
+ 2915, 2915, 450, 452, 2915, 450, 2915, 2791, 451, 2915,
+
+ 450, 451, 452, 450, 2792, 452, 2915, 450, 450, 2915,
+ 451, 2915, 2915, 2915, 452, 450, 2801, 450, 2915, 2915,
+ 451, 2915, 2915, 450, 452, 450, 450, 2793, 2915, 450,
+ 2915, 2794, 2807, 450, 450, 2915, 2808, 450, 450, 2915,
+ 2915, 2915, 450, 2915, 2915, 450, 450, 2915, 450, 2915,
+ 2915, 2915, 2915, 2803, 450, 2915, 450, 2915, 2915, 2915,
+ 450, 2805, 2915, 2915, 450, 2915, 2915, 2915, 450, 2915,
+ 2915, 2915, 2915, 2915, 2809, 2915, 450, 2812, 2813, 2812,
+ 2812, 2812, 2814, 2815, 2812, 2815, 2812, 2812, 2812, 2812,
+ 2812, 2812, 2812, 2812, 2812, 2812, 2812, 2812, 2812, 2816,
+
+ 2817, 2812, 2812, 2812, 2812, 2812, 2812, 2812, 2812, 2812,
2812, 2812, 2812, 2812, 2812, 2812, 2812, 2812, 2812, 2812,
- 2812, 2812, 2812, 2812, 2812, 2812, 2812, 2812, 2812, 2815,
- 2815, 2815, 2815, 2815, 2815, 2816, 2816, 2816, 2817, 2812,
- 2812, 451, 451, 2912, 2824, 452, 452, 2760, 2825, 2912,
- 2912, 2761, 2912, 451, 2912, 2912, 2912, 452, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 450,
- 450, 2912, 450, 2821, 2912, 450, 2912, 450, 450, 2820,
-
- 450, 450, 2912, 450, 2912, 450, 450, 2912, 450, 450,
- 451, 450, 451, 2912, 452, 451, 452, 450, 2912, 452,
- 2912, 451, 2779, 2912, 2912, 452, 2780, 2772, 2772, 2774,
- 2774, 2912, 2776, 2776, 2912, 451, 2912, 2912, 450, 452,
- 450, 451, 2912, 450, 2088, 452, 450, 2912, 450, 450,
- 450, 450, 2833, 2912, 450, 2912, 450, 450, 450, 450,
- 451, 2912, 2912, 450, 452, 450, 450, 2912, 2912, 450,
- 451, 450, 451, 2912, 452, 2912, 452, 450, 2912, 450,
- 2912, 2912, 451, 451, 886, 450, 452, 452, 450, 451,
- 2912, 2912, 2912, 452, 2912, 959, 450, 2912, 450, 2912,
-
- 450, 451, 2912, 2912, 450, 452, 450, 1249, 450, 2912,
- 450, 450, 2912, 2912, 450, 2834, 450, 2843, 450, 450,
- 2845, 2912, 2912, 2912, 2846, 450, 2835, 450, 2836, 450,
- 2804, 451, 451, 450, 2805, 452, 452, 450, 2912, 2912,
- 2912, 2912, 451, 451, 2912, 450, 452, 452, 450, 2824,
- 2912, 2912, 2912, 2825, 2912, 2912, 450, 2912, 450, 450,
- 450, 2912, 2912, 2912, 450, 2912, 450, 450, 450, 451,
- 450, 450, 2912, 452, 450, 450, 450, 450, 450, 450,
- 2912, 2912, 2912, 2912, 2859, 450, 2851, 2852, 451, 2912,
- 2912, 451, 452, 450, 451, 452, 2912, 450, 452, 2912,
-
- 2912, 2912, 2912, 451, 2912, 450, 2845, 452, 2912, 2912,
- 2846, 2912, 2912, 450, 2912, 2912, 450, 2860, 2912, 450,
- 2912, 451, 450, 2861, 450, 452, 2912, 450, 2912, 2912,
- 450, 450, 450, 1585, 450, 450, 2865, 2912, 450, 450,
- 451, 451, 450, 2912, 452, 452, 2912, 450, 2912, 450,
- 450, 451, 451, 2912, 2912, 452, 452, 450, 2912, 2912,
- 2912, 2912, 451, 1430, 2912, 450, 452, 2912, 450, 450,
- 2912, 2912, 2912, 2912, 2912, 2912, 450, 450, 2867, 450,
- 450, 2912, 451, 2871, 450, 450, 452, 450, 450, 451,
- 450, 2912, 2872, 452, 2873, 450, 450, 2912, 450, 451,
-
- 2912, 2912, 2912, 452, 451, 2912, 2878, 2876, 452, 451,
- 450, 2912, 2912, 452, 2912, 2912, 451, 450, 450, 451,
- 452, 2912, 2882, 452, 2912, 450, 450, 450, 2912, 2884,
- 2912, 2912, 450, 450, 451, 450, 2887, 450, 452, 2912,
- 450, 2883, 2912, 450, 450, 450, 451, 450, 450, 2912,
- 452, 2912, 450, 450, 2912, 450, 2912, 451, 1979, 2892,
- 450, 452, 450, 450, 2912, 2912, 2912, 451, 2891, 451,
- 450, 452, 2912, 452, 450, 2912, 2912, 2912, 450, 451,
- 2912, 969, 450, 452, 2912, 450, 2912, 2912, 451, 2912,
- 450, 2912, 452, 450, 2912, 450, 923, 450, 2895, 2912,
-
- 2912, 450, 451, 450, 961, 450, 452, 450, 2912, 2912,
- 2912, 450, 2900, 450, 451, 450, 450, 451, 452, 2912,
- 2912, 452, 2912, 450, 450, 2912, 2912, 2912, 2912, 2912,
- 450, 2903, 450, 2912, 2912, 2906, 2912, 2912, 450, 2912,
- 2912, 2909, 450, 2912, 2912, 450, 450, 2912, 2912, 2912,
- 450, 2912, 2912, 450, 2912, 2912, 2912, 2912, 450, 958,
- 2912, 450, 124, 124, 124, 124, 124, 124, 124, 124,
+ 2812, 2812, 2812, 2812, 2812, 2812, 2812, 2812, 2812, 2812,
+ 2815, 2815, 2815, 2815, 2815, 2815, 2815, 2815, 2815, 2815,
+ 2815, 2815, 2815, 2815, 2815, 2815, 2815, 2815, 2815, 2818,
+ 2818, 2818, 2818, 2818, 2818, 2819, 2819, 2819, 2820, 2815,
+ 2815, 451, 451, 2915, 2827, 452, 452, 2763, 2828, 2915,
+ 2915, 2764, 2915, 451, 2915, 2915, 2915, 452, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 450,
+ 450, 2915, 450, 2824, 2915, 450, 2915, 450, 450, 2823,
+
+ 450, 450, 2915, 450, 2915, 450, 450, 2915, 450, 450,
+ 451, 450, 451, 2915, 452, 451, 452, 450, 2915, 452,
+ 2915, 451, 2782, 2915, 2915, 452, 2783, 2775, 2775, 2777,
+ 2777, 2915, 2779, 2779, 2915, 451, 2915, 2915, 450, 452,
+ 450, 451, 2915, 450, 2090, 452, 450, 2915, 450, 450,
+ 450, 450, 2836, 2915, 450, 2915, 450, 450, 450, 450,
+ 451, 2915, 2915, 450, 452, 450, 450, 2915, 2915, 450,
+ 451, 450, 451, 2915, 452, 2915, 452, 450, 2915, 450,
+ 2915, 2915, 451, 451, 886, 450, 452, 452, 450, 451,
+ 2915, 2915, 2915, 452, 2915, 959, 450, 2915, 450, 2915,
+
+ 450, 451, 2915, 2915, 450, 452, 450, 1249, 450, 2915,
+ 450, 450, 2915, 2915, 450, 2837, 450, 2846, 450, 450,
+ 2848, 2915, 2915, 2915, 2849, 450, 2838, 450, 2839, 450,
+ 2807, 451, 451, 450, 2808, 452, 452, 450, 2915, 2915,
+ 2915, 2915, 451, 451, 2915, 450, 452, 452, 450, 2827,
+ 2915, 2915, 2915, 2828, 2915, 2915, 450, 2915, 450, 450,
+ 450, 2915, 2915, 2915, 450, 2915, 450, 450, 450, 451,
+ 450, 450, 2915, 452, 450, 450, 450, 450, 450, 450,
+ 2915, 2915, 2915, 2915, 2862, 450, 2854, 2855, 451, 2915,
+ 2915, 451, 452, 450, 451, 452, 2915, 450, 452, 2915,
+
+ 2915, 2915, 2915, 451, 2915, 450, 2848, 452, 2915, 2915,
+ 2849, 2915, 2915, 450, 2915, 2915, 450, 2863, 2915, 450,
+ 2915, 451, 450, 2864, 450, 452, 2915, 450, 2915, 2915,
+ 450, 450, 450, 1585, 450, 450, 2868, 2915, 450, 450,
+ 451, 451, 450, 2915, 452, 452, 2915, 450, 2915, 450,
+ 450, 451, 451, 2915, 2915, 452, 452, 450, 2915, 2915,
+ 2915, 2915, 451, 1430, 2915, 450, 452, 2915, 450, 450,
+ 2915, 2915, 2915, 2915, 2915, 2915, 450, 450, 2870, 450,
+ 450, 2915, 451, 2874, 450, 450, 452, 450, 450, 451,
+ 450, 2915, 2875, 452, 2876, 450, 450, 2915, 450, 451,
+
+ 2915, 2915, 2915, 452, 451, 2915, 2881, 2879, 452, 451,
+ 450, 2915, 2915, 452, 2915, 2915, 451, 450, 450, 451,
+ 452, 2915, 2885, 452, 2915, 450, 450, 450, 2915, 2887,
+ 2915, 2915, 450, 450, 451, 450, 2890, 450, 452, 2915,
+ 450, 2886, 2915, 450, 450, 450, 451, 450, 450, 2915,
+ 452, 2915, 450, 450, 2915, 450, 2915, 451, 1981, 2895,
+ 450, 452, 450, 450, 2915, 2915, 2915, 451, 2894, 451,
+ 450, 452, 2915, 452, 450, 2915, 2915, 2915, 450, 451,
+ 2915, 969, 450, 452, 2915, 450, 2915, 2915, 451, 2915,
+ 450, 2915, 452, 450, 2915, 450, 923, 450, 2898, 2915,
+
+ 2915, 450, 451, 450, 961, 450, 452, 450, 2915, 2915,
+ 2915, 450, 2903, 450, 451, 450, 450, 451, 452, 2915,
+ 2915, 452, 2915, 450, 450, 2915, 2915, 2915, 2915, 2915,
+ 450, 2906, 450, 2915, 2915, 2909, 2915, 2915, 450, 2915,
+ 2915, 2912, 450, 2915, 2915, 450, 450, 2915, 2915, 2915,
+ 450, 2915, 2915, 450, 2915, 2915, 2915, 2915, 450, 958,
+ 2915, 450, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 169, 169, 169, 169, 169,
169, 169, 169, 169, 169, 169, 169, 169, 257, 257,
257, 257, 257, 257, 257, 257, 257, 257, 257, 257,
@@ -3966,345 +3966,345 @@ static yyconst flex_int16_t yy_nxt[27006] =
363, 363, 363, 363, 363, 373, 373, 373, 373, 373,
373, 373, 373, 373, 373, 373, 373, 373, 382, 382,
382, 382, 382, 382, 382, 382, 382, 382, 382, 382,
- 382, 449, 449, 2912, 449, 449, 449, 449, 449, 449,
- 449, 449, 449, 449, 450, 450, 450, 450, 450, 2912,
- 450, 450, 450, 450, 550, 550, 2912, 550, 550, 550,
- 550, 550, 550, 550, 550, 550, 550, 551, 551, 2912,
+ 382, 449, 449, 2915, 449, 449, 449, 449, 449, 449,
+ 449, 449, 449, 449, 450, 450, 450, 450, 450, 2915,
+ 450, 450, 450, 450, 550, 550, 2915, 550, 550, 550,
+ 550, 550, 550, 550, 550, 550, 550, 551, 551, 2915,
551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
- 557, 557, 2912, 557, 557, 557, 557, 557, 557, 557,
+ 557, 557, 2915, 557, 557, 557, 557, 557, 557, 557,
- 557, 557, 557, 558, 558, 2912, 558, 558, 558, 558,
- 558, 558, 558, 558, 558, 558, 559, 559, 2912, 559,
+ 557, 557, 557, 558, 558, 2915, 558, 558, 558, 558,
+ 558, 558, 558, 558, 558, 558, 559, 559, 2915, 559,
559, 559, 559, 559, 559, 559, 559, 559, 559, 562,
- 2912, 562, 562, 2912, 562, 562, 562, 562, 562, 562,
- 562, 562, 570, 570, 2912, 2912, 570, 570, 570, 570,
- 570, 570, 574, 574, 2912, 574, 574, 574, 574, 574,
- 574, 574, 574, 574, 574, 575, 575, 2912, 575, 575,
+ 2915, 562, 562, 2915, 562, 562, 562, 562, 562, 562,
+ 562, 562, 570, 570, 2915, 2915, 570, 570, 570, 570,
+ 570, 570, 574, 574, 2915, 574, 574, 574, 574, 574,
+ 574, 574, 574, 574, 574, 575, 575, 2915, 575, 575,
575, 575, 575, 575, 575, 575, 575, 575, 576, 576,
- 2912, 2912, 576, 576, 576, 576, 576, 576, 580, 580,
- 2912, 580, 580, 580, 580, 580, 580, 580, 580, 580,
+ 2915, 2915, 576, 576, 576, 576, 576, 576, 580, 580,
+ 2915, 580, 580, 580, 580, 580, 580, 580, 580, 580,
- 580, 581, 581, 2912, 581, 581, 581, 581, 581, 581,
+ 580, 581, 581, 2915, 581, 581, 581, 581, 581, 581,
581, 581, 581, 581, 583, 583, 583, 583, 583, 583,
- 583, 583, 2912, 583, 583, 583, 583, 588, 588, 2912,
- 2912, 588, 588, 588, 588, 588, 588, 592, 592, 592,
+ 583, 583, 2915, 583, 583, 583, 583, 588, 588, 2915,
+ 2915, 588, 588, 588, 588, 588, 588, 592, 592, 592,
592, 592, 592, 592, 592, 592, 592, 592, 592, 592,
594, 594, 594, 594, 594, 594, 594, 594, 594, 594,
- 594, 594, 594, 596, 596, 2912, 2912, 596, 596, 596,
- 596, 596, 596, 600, 600, 2912, 600, 600, 600, 600,
- 600, 600, 600, 600, 600, 600, 601, 601, 2912, 601,
+ 594, 594, 594, 596, 596, 2915, 2915, 596, 596, 596,
+ 596, 596, 596, 600, 600, 2915, 600, 600, 600, 600,
+ 600, 600, 600, 600, 600, 600, 601, 601, 2915, 601,
601, 601, 601, 601, 601, 601, 601, 601, 601, 602,
- 602, 2912, 2912, 602, 602, 602, 602, 602, 602, 606,
- 606, 2912, 606, 606, 606, 606, 606, 606, 606, 606,
- 606, 606, 607, 607, 2912, 607, 607, 607, 607, 607,
- 607, 607, 607, 607, 607, 608, 608, 2912, 2912, 608,
- 608, 608, 608, 608, 608, 612, 612, 2912, 612, 612,
+ 602, 2915, 2915, 602, 602, 602, 602, 602, 602, 606,
+ 606, 2915, 606, 606, 606, 606, 606, 606, 606, 606,
+ 606, 606, 607, 607, 2915, 607, 607, 607, 607, 607,
+ 607, 607, 607, 607, 607, 608, 608, 2915, 2915, 608,
+ 608, 608, 608, 608, 608, 612, 612, 2915, 612, 612,
612, 612, 612, 612, 612, 612, 612, 612, 613, 613,
- 2912, 613, 613, 613, 613, 613, 613, 613, 613, 613,
- 613, 614, 614, 2912, 2912, 614, 614, 614, 614, 614,
- 614, 618, 618, 2912, 618, 618, 618, 618, 618, 618,
- 618, 618, 618, 618, 619, 619, 2912, 619, 619, 619,
+ 2915, 613, 613, 613, 613, 613, 613, 613, 613, 613,
+ 613, 614, 614, 2915, 2915, 614, 614, 614, 614, 614,
+ 614, 618, 618, 2915, 618, 618, 618, 618, 618, 618,
+ 618, 618, 618, 618, 619, 619, 2915, 619, 619, 619,
- 619, 619, 619, 619, 619, 619, 619, 620, 620, 2912,
- 2912, 620, 620, 620, 620, 620, 620, 624, 624, 2912,
+ 619, 619, 619, 619, 619, 619, 619, 620, 620, 2915,
+ 2915, 620, 620, 620, 620, 620, 620, 624, 624, 2915,
624, 624, 624, 624, 624, 624, 624, 624, 624, 624,
- 625, 625, 2912, 625, 625, 625, 625, 625, 625, 625,
- 625, 625, 625, 626, 626, 2912, 2912, 626, 626, 626,
- 626, 626, 626, 630, 630, 2912, 630, 630, 630, 630,
- 630, 630, 630, 630, 630, 630, 631, 631, 2912, 631,
+ 625, 625, 2915, 625, 625, 625, 625, 625, 625, 625,
+ 625, 625, 625, 626, 626, 2915, 2915, 626, 626, 626,
+ 626, 626, 626, 630, 630, 2915, 630, 630, 630, 630,
+ 630, 630, 630, 630, 630, 630, 631, 631, 2915, 631,
631, 631, 631, 631, 631, 631, 631, 631, 631, 632,
- 632, 2912, 2912, 632, 632, 632, 632, 632, 632, 636,
- 636, 2912, 636, 636, 636, 636, 636, 636, 636, 636,
+ 632, 2915, 2915, 632, 632, 632, 632, 632, 632, 636,
+ 636, 2915, 636, 636, 636, 636, 636, 636, 636, 636,
- 636, 636, 637, 637, 2912, 637, 637, 637, 637, 637,
- 637, 637, 637, 637, 637, 638, 638, 2912, 2912, 638,
- 638, 638, 638, 638, 638, 642, 642, 2912, 642, 642,
+ 636, 636, 637, 637, 2915, 637, 637, 637, 637, 637,
+ 637, 637, 637, 637, 637, 638, 638, 2915, 2915, 638,
+ 638, 638, 638, 638, 638, 642, 642, 2915, 642, 642,
642, 642, 642, 642, 642, 642, 642, 642, 643, 643,
- 2912, 643, 643, 643, 643, 643, 643, 643, 643, 643,
- 643, 644, 644, 2912, 2912, 644, 644, 644, 644, 644,
- 644, 648, 648, 2912, 648, 648, 648, 648, 648, 648,
- 648, 648, 648, 648, 649, 649, 2912, 649, 649, 649,
- 649, 649, 649, 649, 649, 649, 649, 650, 650, 2912,
- 2912, 650, 650, 650, 650, 650, 650, 654, 654, 2912,
+ 2915, 643, 643, 643, 643, 643, 643, 643, 643, 643,
+ 643, 644, 644, 2915, 2915, 644, 644, 644, 644, 644,
+ 644, 648, 648, 2915, 648, 648, 648, 648, 648, 648,
+ 648, 648, 648, 648, 649, 649, 2915, 649, 649, 649,
+ 649, 649, 649, 649, 649, 649, 649, 650, 650, 2915,
+ 2915, 650, 650, 650, 650, 650, 650, 654, 654, 2915,
654, 654, 654, 654, 654, 654, 654, 654, 654, 654,
- 655, 655, 2912, 655, 655, 655, 655, 655, 655, 655,
- 655, 655, 655, 656, 656, 2912, 2912, 656, 656, 656,
- 656, 656, 656, 660, 660, 2912, 660, 660, 660, 660,
- 660, 660, 660, 660, 660, 660, 661, 661, 2912, 661,
+ 655, 655, 2915, 655, 655, 655, 655, 655, 655, 655,
+ 655, 655, 655, 656, 656, 2915, 2915, 656, 656, 656,
+ 656, 656, 656, 660, 660, 2915, 660, 660, 660, 660,
+ 660, 660, 660, 660, 660, 660, 661, 661, 2915, 661,
661, 661, 661, 661, 661, 661, 661, 661, 661, 662,
- 662, 662, 662, 2912, 662, 662, 662, 662, 662, 662,
- 662, 662, 663, 663, 2912, 663, 2912, 663, 663, 663,
- 663, 663, 663, 663, 663, 664, 664, 2912, 2912, 664,
- 664, 664, 664, 664, 664, 668, 668, 2912, 668, 668,
+ 662, 662, 662, 2915, 662, 662, 662, 662, 662, 662,
+ 662, 662, 663, 663, 2915, 663, 2915, 663, 663, 663,
+ 663, 663, 663, 663, 663, 664, 664, 2915, 2915, 664,
+ 664, 664, 664, 664, 664, 668, 668, 2915, 668, 668,
668, 668, 668, 668, 668, 668, 668, 668, 669, 669,
- 2912, 669, 669, 669, 669, 669, 669, 669, 669, 669,
- 669, 670, 670, 2912, 2912, 670, 670, 670, 670, 670,
- 670, 674, 674, 2912, 674, 674, 674, 674, 674, 674,
- 674, 674, 674, 674, 675, 675, 2912, 675, 675, 675,
- 675, 675, 675, 675, 675, 675, 675, 676, 676, 2912,
- 2912, 676, 676, 676, 676, 676, 676, 680, 680, 2912,
+ 2915, 669, 669, 669, 669, 669, 669, 669, 669, 669,
+ 669, 670, 670, 2915, 2915, 670, 670, 670, 670, 670,
+ 670, 674, 674, 2915, 674, 674, 674, 674, 674, 674,
+ 674, 674, 674, 674, 675, 675, 2915, 675, 675, 675,
+ 675, 675, 675, 675, 675, 675, 675, 676, 676, 2915,
+ 2915, 676, 676, 676, 676, 676, 676, 680, 680, 2915,
680, 680, 680, 680, 680, 680, 680, 680, 680, 680,
- 681, 681, 2912, 681, 681, 681, 681, 681, 681, 681,
- 681, 681, 681, 682, 682, 2912, 2912, 682, 682, 682,
+ 681, 681, 2915, 681, 681, 681, 681, 681, 681, 681,
+ 681, 681, 681, 682, 682, 2915, 2915, 682, 682, 682,
- 682, 682, 682, 686, 686, 2912, 686, 686, 686, 686,
- 686, 686, 686, 686, 686, 686, 687, 687, 2912, 687,
+ 682, 682, 682, 686, 686, 2915, 686, 686, 686, 686,
+ 686, 686, 686, 686, 686, 686, 687, 687, 2915, 687,
687, 687, 687, 687, 687, 687, 687, 687, 687, 688,
- 688, 688, 688, 688, 688, 2912, 688, 688, 688, 688,
- 688, 688, 689, 689, 2912, 2912, 689, 689, 689, 689,
- 689, 689, 693, 693, 2912, 693, 693, 693, 693, 693,
- 693, 693, 693, 693, 693, 694, 694, 2912, 694, 694,
+ 688, 688, 688, 688, 688, 2915, 688, 688, 688, 688,
+ 688, 688, 689, 689, 2915, 2915, 689, 689, 689, 689,
+ 689, 689, 693, 693, 2915, 693, 693, 693, 693, 693,
+ 693, 693, 693, 693, 693, 694, 694, 2915, 694, 694,
694, 694, 694, 694, 694, 694, 694, 694, 695, 695,
- 2912, 2912, 695, 695, 695, 695, 695, 695, 699, 699,
- 2912, 699, 699, 699, 699, 699, 699, 699, 699, 699,
+ 2915, 2915, 695, 695, 695, 695, 695, 695, 699, 699,
+ 2915, 699, 699, 699, 699, 699, 699, 699, 699, 699,
- 699, 700, 700, 2912, 700, 700, 700, 700, 700, 700,
- 700, 700, 700, 700, 701, 701, 2912, 2912, 701, 701,
- 701, 701, 701, 701, 705, 705, 2912, 705, 705, 705,
- 705, 705, 705, 705, 705, 705, 705, 706, 706, 2912,
+ 699, 700, 700, 2915, 700, 700, 700, 700, 700, 700,
+ 700, 700, 700, 700, 701, 701, 2915, 2915, 701, 701,
+ 701, 701, 701, 701, 705, 705, 2915, 705, 705, 705,
+ 705, 705, 705, 705, 705, 705, 705, 706, 706, 2915,
706, 706, 706, 706, 706, 706, 706, 706, 706, 706,
- 707, 707, 2912, 2912, 707, 707, 707, 707, 707, 707,
- 711, 711, 2912, 711, 711, 711, 711, 711, 711, 711,
- 711, 711, 711, 712, 712, 2912, 712, 712, 712, 712,
+ 707, 707, 2915, 2915, 707, 707, 707, 707, 707, 707,
+ 711, 711, 2915, 711, 711, 711, 711, 711, 711, 711,
+ 711, 711, 711, 712, 712, 2915, 712, 712, 712, 712,
712, 712, 712, 712, 712, 712, 713, 713, 713, 713,
713, 713, 713, 713, 713, 713, 713, 713, 713, 718,
- 718, 718, 718, 718, 718, 2912, 718, 718, 718, 718,
+ 718, 718, 718, 718, 718, 2915, 718, 718, 718, 718,
718, 718, 722, 722, 722, 722, 722, 722, 722, 722,
722, 722, 738, 738, 738, 738, 738, 738, 738, 738,
- 738, 2912, 738, 738, 738, 726, 726, 2912, 726, 726,
+ 738, 2915, 738, 738, 738, 726, 726, 2915, 726, 726,
726, 726, 726, 726, 726, 726, 726, 726, 742, 742,
- 2912, 742, 742, 742, 742, 742, 742, 742, 742, 742,
+ 2915, 742, 742, 742, 742, 742, 742, 742, 742, 742,
742, 743, 743, 743, 743, 743, 743, 743, 743, 743,
- 743, 743, 2912, 743, 751, 751, 2912, 751, 751, 751,
- 751, 751, 751, 751, 751, 751, 751, 752, 752, 2912,
+ 743, 743, 2915, 743, 751, 751, 2915, 751, 751, 751,
+ 751, 751, 751, 751, 751, 751, 751, 752, 752, 2915,
752, 752, 752, 752, 752, 752, 752, 752, 752, 752,
- 753, 753, 2912, 2912, 753, 753, 753, 753, 753, 753,
- 757, 757, 2912, 757, 757, 757, 757, 757, 757, 757,
- 757, 757, 757, 758, 758, 2912, 758, 758, 758, 758,
- 758, 758, 758, 758, 758, 758, 759, 759, 2912, 2912,
- 759, 759, 759, 759, 759, 759, 763, 763, 2912, 763,
+ 753, 753, 2915, 2915, 753, 753, 753, 753, 753, 753,
+ 757, 757, 2915, 757, 757, 757, 757, 757, 757, 757,
+ 757, 757, 757, 758, 758, 2915, 758, 758, 758, 758,
+ 758, 758, 758, 758, 758, 758, 759, 759, 2915, 2915,
+ 759, 759, 759, 759, 759, 759, 763, 763, 2915, 763,
763, 763, 763, 763, 763, 763, 763, 763, 763, 764,
- 764, 2912, 764, 764, 764, 764, 764, 764, 764, 764,
- 764, 764, 765, 765, 2912, 2912, 765, 765, 765, 765,
- 765, 765, 769, 769, 2912, 769, 769, 769, 769, 769,
- 769, 769, 769, 769, 769, 770, 770, 2912, 770, 770,
+ 764, 2915, 764, 764, 764, 764, 764, 764, 764, 764,
+ 764, 764, 765, 765, 2915, 2915, 765, 765, 765, 765,
+ 765, 765, 769, 769, 2915, 769, 769, 769, 769, 769,
+ 769, 769, 769, 769, 769, 770, 770, 2915, 770, 770,
770, 770, 770, 770, 770, 770, 770, 770, 771, 771,
- 2912, 2912, 771, 771, 771, 771, 771, 771, 775, 775,
- 2912, 775, 775, 775, 775, 775, 775, 775, 775, 775,
- 775, 776, 776, 2912, 776, 776, 776, 776, 776, 776,
- 776, 776, 776, 776, 777, 2912, 777, 777, 2912, 777,
- 777, 777, 777, 777, 777, 777, 777, 778, 2912, 778,
- 778, 2912, 778, 778, 778, 2912, 778, 778, 778, 778,
- 779, 779, 2912, 2912, 779, 779, 779, 779, 779, 779,
- 783, 783, 2912, 783, 783, 783, 783, 783, 783, 783,
- 783, 783, 783, 784, 784, 2912, 784, 784, 784, 784,
-
- 784, 784, 784, 784, 784, 784, 785, 2912, 785, 785,
- 2912, 785, 785, 785, 785, 785, 785, 785, 785, 787,
- 787, 2912, 2912, 787, 787, 787, 787, 787, 787, 791,
- 791, 2912, 791, 791, 791, 791, 791, 791, 791, 791,
- 791, 791, 792, 792, 2912, 792, 792, 792, 792, 792,
- 792, 792, 792, 792, 792, 793, 2912, 793, 793, 2912,
+ 2915, 2915, 771, 771, 771, 771, 771, 771, 775, 775,
+ 2915, 775, 775, 775, 775, 775, 775, 775, 775, 775,
+ 775, 776, 776, 2915, 776, 776, 776, 776, 776, 776,
+ 776, 776, 776, 776, 777, 2915, 777, 777, 2915, 777,
+ 777, 777, 777, 777, 777, 777, 777, 778, 2915, 778,
+ 778, 2915, 778, 778, 778, 2915, 778, 778, 778, 778,
+ 779, 779, 2915, 2915, 779, 779, 779, 779, 779, 779,
+ 783, 783, 2915, 783, 783, 783, 783, 783, 783, 783,
+ 783, 783, 783, 784, 784, 2915, 784, 784, 784, 784,
+
+ 784, 784, 784, 784, 784, 784, 785, 2915, 785, 785,
+ 2915, 785, 785, 785, 785, 785, 785, 785, 785, 787,
+ 787, 2915, 2915, 787, 787, 787, 787, 787, 787, 791,
+ 791, 2915, 791, 791, 791, 791, 791, 791, 791, 791,
+ 791, 791, 792, 792, 2915, 792, 792, 792, 792, 792,
+ 792, 792, 792, 792, 792, 793, 2915, 793, 793, 2915,
793, 793, 793, 793, 793, 793, 793, 793, 795, 795,
- 2912, 2912, 795, 795, 795, 795, 795, 795, 799, 799,
- 2912, 799, 799, 799, 799, 799, 799, 799, 799, 799,
- 799, 800, 800, 2912, 800, 800, 800, 800, 800, 800,
+ 2915, 2915, 795, 795, 795, 795, 795, 795, 799, 799,
+ 2915, 799, 799, 799, 799, 799, 799, 799, 799, 799,
+ 799, 800, 800, 2915, 800, 800, 800, 800, 800, 800,
- 800, 800, 800, 800, 801, 801, 2912, 2912, 801, 801,
- 801, 801, 801, 801, 805, 805, 2912, 805, 805, 805,
- 805, 805, 805, 805, 805, 805, 805, 806, 806, 2912,
+ 800, 800, 800, 800, 801, 801, 2915, 2915, 801, 801,
+ 801, 801, 801, 801, 805, 805, 2915, 805, 805, 805,
+ 805, 805, 805, 805, 805, 805, 805, 806, 806, 2915,
806, 806, 806, 806, 806, 806, 806, 806, 806, 806,
- 807, 807, 2912, 2912, 807, 807, 807, 807, 807, 807,
- 811, 811, 2912, 811, 811, 811, 811, 811, 811, 811,
- 811, 811, 811, 812, 812, 2912, 812, 812, 812, 812,
- 812, 812, 812, 812, 812, 812, 813, 813, 2912, 813,
+ 807, 807, 2915, 2915, 807, 807, 807, 807, 807, 807,
+ 811, 811, 2915, 811, 811, 811, 811, 811, 811, 811,
+ 811, 811, 811, 812, 812, 2915, 812, 812, 812, 812,
+ 812, 812, 812, 812, 812, 812, 813, 813, 2915, 813,
813, 813, 813, 813, 813, 813, 813, 813, 813, 814,
- 814, 2912, 814, 814, 814, 814, 814, 814, 814, 814,
+ 814, 2915, 814, 814, 814, 814, 814, 814, 814, 814,
- 814, 814, 815, 815, 2912, 815, 815, 815, 815, 815,
- 815, 815, 815, 815, 815, 817, 817, 2912, 2912, 817,
- 817, 817, 817, 817, 817, 821, 821, 2912, 821, 821,
+ 814, 814, 815, 815, 2915, 815, 815, 815, 815, 815,
+ 815, 815, 815, 815, 815, 817, 817, 2915, 2915, 817,
+ 817, 817, 817, 817, 817, 821, 821, 2915, 821, 821,
821, 821, 821, 821, 821, 821, 821, 821, 822, 822,
- 2912, 822, 822, 822, 822, 822, 822, 822, 822, 822,
- 822, 823, 823, 2912, 2912, 823, 823, 823, 823, 823,
- 823, 827, 827, 2912, 827, 827, 827, 827, 827, 827,
- 827, 827, 827, 827, 828, 828, 2912, 828, 828, 828,
- 828, 828, 828, 828, 828, 828, 828, 829, 829, 2912,
- 2912, 829, 829, 829, 829, 829, 829, 833, 833, 2912,
+ 2915, 822, 822, 822, 822, 822, 822, 822, 822, 822,
+ 822, 823, 823, 2915, 2915, 823, 823, 823, 823, 823,
+ 823, 827, 827, 2915, 827, 827, 827, 827, 827, 827,
+ 827, 827, 827, 827, 828, 828, 2915, 828, 828, 828,
+ 828, 828, 828, 828, 828, 828, 828, 829, 829, 2915,
+ 2915, 829, 829, 829, 829, 829, 829, 833, 833, 2915,
833, 833, 833, 833, 833, 833, 833, 833, 833, 833,
- 834, 834, 2912, 834, 834, 834, 834, 834, 834, 834,
- 834, 834, 834, 835, 835, 2912, 835, 835, 835, 835,
- 835, 835, 835, 835, 835, 835, 836, 836, 2912, 836,
+ 834, 834, 2915, 834, 834, 834, 834, 834, 834, 834,
+ 834, 834, 834, 835, 835, 2915, 835, 835, 835, 835,
+ 835, 835, 835, 835, 835, 835, 836, 836, 2915, 836,
836, 836, 836, 836, 836, 836, 836, 836, 836, 837,
- 837, 2912, 837, 837, 837, 837, 837, 837, 837, 837,
- 837, 837, 838, 838, 2912, 2912, 838, 838, 838, 838,
- 838, 838, 842, 842, 2912, 842, 842, 842, 842, 842,
- 842, 842, 842, 842, 842, 843, 843, 2912, 843, 843,
+ 837, 2915, 837, 837, 837, 837, 837, 837, 837, 837,
+ 837, 837, 838, 838, 2915, 2915, 838, 838, 838, 838,
+ 838, 838, 842, 842, 2915, 842, 842, 842, 842, 842,
+ 842, 842, 842, 842, 842, 843, 843, 2915, 843, 843,
843, 843, 843, 843, 843, 843, 843, 843, 844, 844,
- 2912, 2912, 844, 844, 844, 844, 844, 844, 848, 848,
- 2912, 848, 848, 848, 848, 848, 848, 848, 848, 848,
- 848, 849, 849, 2912, 849, 849, 849, 849, 849, 849,
- 849, 849, 849, 849, 850, 850, 2912, 2912, 850, 850,
- 850, 850, 850, 850, 854, 854, 2912, 854, 854, 854,
- 854, 854, 854, 854, 854, 854, 854, 855, 855, 2912,
+ 2915, 2915, 844, 844, 844, 844, 844, 844, 848, 848,
+ 2915, 848, 848, 848, 848, 848, 848, 848, 848, 848,
+ 848, 849, 849, 2915, 849, 849, 849, 849, 849, 849,
+ 849, 849, 849, 849, 850, 850, 2915, 2915, 850, 850,
+ 850, 850, 850, 850, 854, 854, 2915, 854, 854, 854,
+ 854, 854, 854, 854, 854, 854, 854, 855, 855, 2915,
855, 855, 855, 855, 855, 855, 855, 855, 855, 855,
- 856, 856, 2912, 2912, 856, 856, 856, 856, 856, 856,
- 860, 860, 2912, 860, 860, 860, 860, 860, 860, 860,
- 860, 860, 860, 861, 861, 2912, 861, 861, 861, 861,
+ 856, 856, 2915, 2915, 856, 856, 856, 856, 856, 856,
+ 860, 860, 2915, 860, 860, 860, 860, 860, 860, 860,
+ 860, 860, 860, 861, 861, 2915, 861, 861, 861, 861,
- 861, 861, 861, 861, 861, 861, 449, 449, 2912, 449,
+ 861, 861, 861, 861, 861, 861, 449, 449, 2915, 449,
449, 449, 449, 449, 449, 449, 449, 449, 449, 450,
- 450, 450, 450, 450, 2912, 450, 450, 450, 450, 550,
- 550, 2912, 550, 550, 550, 550, 550, 550, 550, 550,
- 550, 550, 551, 551, 2912, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 994, 994, 2912, 994, 994,
+ 450, 450, 450, 450, 2915, 450, 450, 450, 450, 550,
+ 550, 2915, 550, 550, 550, 550, 550, 550, 550, 550,
+ 550, 550, 551, 551, 2915, 551, 551, 551, 551, 551,
+ 551, 551, 551, 551, 551, 994, 994, 2915, 994, 994,
994, 994, 994, 994, 994, 994, 994, 994, 552, 552,
- 2912, 552, 552, 552, 552, 552, 552, 552, 552, 552,
- 552, 995, 995, 2912, 995, 995, 995, 995, 995, 995,
- 995, 995, 995, 995, 996, 996, 2912, 996, 996, 996,
+ 2915, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 995, 995, 2915, 995, 995, 995, 995, 995, 995,
+ 995, 995, 995, 995, 996, 996, 2915, 996, 996, 996,
- 996, 996, 996, 996, 996, 996, 996, 557, 557, 2912,
+ 996, 996, 996, 996, 996, 996, 996, 557, 557, 2915,
557, 557, 557, 557, 557, 557, 557, 557, 557, 557,
- 997, 997, 2912, 997, 997, 997, 997, 997, 997, 997,
- 997, 997, 997, 562, 2912, 562, 562, 2912, 562, 562,
- 562, 562, 562, 562, 562, 562, 570, 570, 2912, 2912,
- 570, 570, 570, 570, 570, 570, 574, 574, 2912, 574,
+ 997, 997, 2915, 997, 997, 997, 997, 997, 997, 997,
+ 997, 997, 997, 562, 2915, 562, 562, 2915, 562, 562,
+ 562, 562, 562, 562, 562, 562, 570, 570, 2915, 2915,
+ 570, 570, 570, 570, 570, 570, 574, 574, 2915, 574,
574, 574, 574, 574, 574, 574, 574, 574, 574, 575,
- 575, 2912, 575, 575, 575, 575, 575, 575, 575, 575,
- 575, 575, 1000, 1000, 2912, 1000, 1000, 1000, 1000, 1000,
- 1000, 1000, 1000, 1000, 1000, 576, 576, 2912, 2912, 576,
+ 575, 2915, 575, 575, 575, 575, 575, 575, 575, 575,
+ 575, 575, 1000, 1000, 2915, 1000, 1000, 1000, 1000, 1000,
+ 1000, 1000, 1000, 1000, 1000, 576, 576, 2915, 2915, 576,
- 576, 576, 576, 576, 576, 580, 580, 2912, 580, 580,
+ 576, 576, 576, 576, 576, 580, 580, 2915, 580, 580,
580, 580, 580, 580, 580, 580, 580, 580, 581, 581,
- 2912, 581, 581, 581, 581, 581, 581, 581, 581, 581,
- 581, 1001, 1001, 2912, 1001, 1001, 1001, 1001, 1001, 1001,
+ 2915, 581, 581, 581, 581, 581, 581, 581, 581, 581,
+ 581, 1001, 1001, 2915, 1001, 1001, 1001, 1001, 1001, 1001,
1001, 1001, 1001, 1001, 583, 583, 583, 583, 583, 583,
- 583, 583, 2912, 583, 583, 583, 583, 588, 588, 2912,
- 2912, 588, 588, 588, 588, 588, 588, 592, 592, 592,
+ 583, 583, 2915, 583, 583, 583, 583, 588, 588, 2915,
+ 2915, 588, 588, 588, 588, 588, 588, 592, 592, 592,
592, 592, 592, 592, 592, 592, 592, 592, 592, 592,
594, 594, 594, 594, 594, 594, 594, 594, 594, 594,
- 594, 594, 594, 593, 593, 2912, 593, 593, 593, 593,
+ 594, 594, 594, 593, 593, 2915, 593, 593, 593, 593,
- 593, 593, 593, 593, 593, 593, 595, 595, 2912, 595,
+ 593, 593, 593, 593, 593, 593, 595, 595, 2915, 595,
595, 595, 595, 595, 595, 595, 595, 595, 595, 1002,
1002, 1002, 1002, 1002, 1002, 1002, 1002, 1002, 1002, 1002,
- 1002, 1002, 1003, 1003, 2912, 1003, 1003, 1003, 1003, 1003,
- 1003, 1003, 1003, 1003, 1003, 596, 596, 2912, 2912, 596,
- 596, 596, 596, 596, 596, 600, 600, 2912, 600, 600,
+ 1002, 1002, 1003, 1003, 2915, 1003, 1003, 1003, 1003, 1003,
+ 1003, 1003, 1003, 1003, 1003, 596, 596, 2915, 2915, 596,
+ 596, 596, 596, 596, 596, 600, 600, 2915, 600, 600,
600, 600, 600, 600, 600, 600, 600, 600, 601, 601,
- 2912, 601, 601, 601, 601, 601, 601, 601, 601, 601,
- 601, 1004, 1004, 2912, 1004, 1004, 1004, 1004, 1004, 1004,
- 1004, 1004, 1004, 1004, 602, 602, 2912, 2912, 602, 602,
+ 2915, 601, 601, 601, 601, 601, 601, 601, 601, 601,
+ 601, 1004, 1004, 2915, 1004, 1004, 1004, 1004, 1004, 1004,
+ 1004, 1004, 1004, 1004, 602, 602, 2915, 2915, 602, 602,
- 602, 602, 602, 602, 606, 606, 2912, 606, 606, 606,
- 606, 606, 606, 606, 606, 606, 606, 607, 607, 2912,
+ 602, 602, 602, 602, 606, 606, 2915, 606, 606, 606,
+ 606, 606, 606, 606, 606, 606, 606, 607, 607, 2915,
607, 607, 607, 607, 607, 607, 607, 607, 607, 607,
- 1005, 1005, 2912, 1005, 1005, 1005, 1005, 1005, 1005, 1005,
- 1005, 1005, 1005, 608, 608, 2912, 2912, 608, 608, 608,
- 608, 608, 608, 612, 612, 2912, 612, 612, 612, 612,
- 612, 612, 612, 612, 612, 612, 613, 613, 2912, 613,
+ 1005, 1005, 2915, 1005, 1005, 1005, 1005, 1005, 1005, 1005,
+ 1005, 1005, 1005, 608, 608, 2915, 2915, 608, 608, 608,
+ 608, 608, 608, 612, 612, 2915, 612, 612, 612, 612,
+ 612, 612, 612, 612, 612, 612, 613, 613, 2915, 613,
613, 613, 613, 613, 613, 613, 613, 613, 613, 1006,
- 1006, 2912, 1006, 1006, 1006, 1006, 1006, 1006, 1006, 1006,
- 1006, 1006, 614, 614, 2912, 2912, 614, 614, 614, 614,
+ 1006, 2915, 1006, 1006, 1006, 1006, 1006, 1006, 1006, 1006,
+ 1006, 1006, 614, 614, 2915, 2915, 614, 614, 614, 614,
- 614, 614, 618, 618, 2912, 618, 618, 618, 618, 618,
- 618, 618, 618, 618, 618, 619, 619, 2912, 619, 619,
+ 614, 614, 618, 618, 2915, 618, 618, 618, 618, 618,
+ 618, 618, 618, 618, 618, 619, 619, 2915, 619, 619,
619, 619, 619, 619, 619, 619, 619, 619, 1007, 1007,
- 2912, 1007, 1007, 1007, 1007, 1007, 1007, 1007, 1007, 1007,
- 1007, 620, 620, 2912, 2912, 620, 620, 620, 620, 620,
- 620, 624, 624, 2912, 624, 624, 624, 624, 624, 624,
- 624, 624, 624, 624, 625, 625, 2912, 625, 625, 625,
- 625, 625, 625, 625, 625, 625, 625, 1008, 1008, 2912,
+ 2915, 1007, 1007, 1007, 1007, 1007, 1007, 1007, 1007, 1007,
+ 1007, 620, 620, 2915, 2915, 620, 620, 620, 620, 620,
+ 620, 624, 624, 2915, 624, 624, 624, 624, 624, 624,
+ 624, 624, 624, 624, 625, 625, 2915, 625, 625, 625,
+ 625, 625, 625, 625, 625, 625, 625, 1008, 1008, 2915,
1008, 1008, 1008, 1008, 1008, 1008, 1008, 1008, 1008, 1008,
- 626, 626, 2912, 2912, 626, 626, 626, 626, 626, 626,
+ 626, 626, 2915, 2915, 626, 626, 626, 626, 626, 626,
- 630, 630, 2912, 630, 630, 630, 630, 630, 630, 630,
- 630, 630, 630, 631, 631, 2912, 631, 631, 631, 631,
- 631, 631, 631, 631, 631, 631, 1009, 1009, 2912, 1009,
+ 630, 630, 2915, 630, 630, 630, 630, 630, 630, 630,
+ 630, 630, 630, 631, 631, 2915, 631, 631, 631, 631,
+ 631, 631, 631, 631, 631, 631, 1009, 1009, 2915, 1009,
1009, 1009, 1009, 1009, 1009, 1009, 1009, 1009, 1009, 632,
- 632, 2912, 2912, 632, 632, 632, 632, 632, 632, 636,
- 636, 2912, 636, 636, 636, 636, 636, 636, 636, 636,
- 636, 636, 637, 637, 2912, 637, 637, 637, 637, 637,
- 637, 637, 637, 637, 637, 1010, 1010, 2912, 1010, 1010,
+ 632, 2915, 2915, 632, 632, 632, 632, 632, 632, 636,
+ 636, 2915, 636, 636, 636, 636, 636, 636, 636, 636,
+ 636, 636, 637, 637, 2915, 637, 637, 637, 637, 637,
+ 637, 637, 637, 637, 637, 1010, 1010, 2915, 1010, 1010,
1010, 1010, 1010, 1010, 1010, 1010, 1010, 1010, 638, 638,
- 2912, 2912, 638, 638, 638, 638, 638, 638, 642, 642,
+ 2915, 2915, 638, 638, 638, 638, 638, 638, 642, 642,
- 2912, 642, 642, 642, 642, 642, 642, 642, 642, 642,
- 642, 643, 643, 2912, 643, 643, 643, 643, 643, 643,
- 643, 643, 643, 643, 1011, 1011, 2912, 1011, 1011, 1011,
- 1011, 1011, 1011, 1011, 1011, 1011, 1011, 644, 644, 2912,
- 2912, 644, 644, 644, 644, 644, 644, 648, 648, 2912,
+ 2915, 642, 642, 642, 642, 642, 642, 642, 642, 642,
+ 642, 643, 643, 2915, 643, 643, 643, 643, 643, 643,
+ 643, 643, 643, 643, 1011, 1011, 2915, 1011, 1011, 1011,
+ 1011, 1011, 1011, 1011, 1011, 1011, 1011, 644, 644, 2915,
+ 2915, 644, 644, 644, 644, 644, 644, 648, 648, 2915,
648, 648, 648, 648, 648, 648, 648, 648, 648, 648,
- 649, 649, 2912, 649, 649, 649, 649, 649, 649, 649,
- 649, 649, 649, 1012, 1012, 2912, 1012, 1012, 1012, 1012,
- 1012, 1012, 1012, 1012, 1012, 1012, 650, 650, 2912, 2912,
- 650, 650, 650, 650, 650, 650, 654, 654, 2912, 654,
+ 649, 649, 2915, 649, 649, 649, 649, 649, 649, 649,
+ 649, 649, 649, 1012, 1012, 2915, 1012, 1012, 1012, 1012,
+ 1012, 1012, 1012, 1012, 1012, 1012, 650, 650, 2915, 2915,
+ 650, 650, 650, 650, 650, 650, 654, 654, 2915, 654,
654, 654, 654, 654, 654, 654, 654, 654, 654, 655,
- 655, 2912, 655, 655, 655, 655, 655, 655, 655, 655,
- 655, 655, 1013, 1013, 2912, 1013, 1013, 1013, 1013, 1013,
- 1013, 1013, 1013, 1013, 1013, 656, 656, 2912, 2912, 656,
- 656, 656, 656, 656, 656, 660, 660, 2912, 660, 660,
+ 655, 2915, 655, 655, 655, 655, 655, 655, 655, 655,
+ 655, 655, 1013, 1013, 2915, 1013, 1013, 1013, 1013, 1013,
+ 1013, 1013, 1013, 1013, 1013, 656, 656, 2915, 2915, 656,
+ 656, 656, 656, 656, 656, 660, 660, 2915, 660, 660,
660, 660, 660, 660, 660, 660, 660, 660, 661, 661,
- 2912, 661, 661, 661, 661, 661, 661, 661, 661, 661,
- 661, 1014, 1014, 2912, 1014, 1014, 1014, 1014, 1014, 1014,
- 1014, 1014, 1014, 1014, 662, 662, 662, 662, 2912, 662,
- 662, 662, 662, 662, 662, 662, 662, 663, 663, 2912,
-
- 663, 2912, 663, 663, 663, 663, 663, 663, 663, 663,
- 664, 664, 2912, 2912, 664, 664, 664, 664, 664, 664,
- 668, 668, 2912, 668, 668, 668, 668, 668, 668, 668,
- 668, 668, 668, 669, 669, 2912, 669, 669, 669, 669,
- 669, 669, 669, 669, 669, 669, 1015, 1015, 2912, 1015,
+ 2915, 661, 661, 661, 661, 661, 661, 661, 661, 661,
+ 661, 1014, 1014, 2915, 1014, 1014, 1014, 1014, 1014, 1014,
+ 1014, 1014, 1014, 1014, 662, 662, 662, 662, 2915, 662,
+ 662, 662, 662, 662, 662, 662, 662, 663, 663, 2915,
+
+ 663, 2915, 663, 663, 663, 663, 663, 663, 663, 663,
+ 664, 664, 2915, 2915, 664, 664, 664, 664, 664, 664,
+ 668, 668, 2915, 668, 668, 668, 668, 668, 668, 668,
+ 668, 668, 668, 669, 669, 2915, 669, 669, 669, 669,
+ 669, 669, 669, 669, 669, 669, 1015, 1015, 2915, 1015,
1015, 1015, 1015, 1015, 1015, 1015, 1015, 1015, 1015, 670,
- 670, 2912, 2912, 670, 670, 670, 670, 670, 670, 674,
- 674, 2912, 674, 674, 674, 674, 674, 674, 674, 674,
- 674, 674, 675, 675, 2912, 675, 675, 675, 675, 675,
- 675, 675, 675, 675, 675, 1016, 1016, 2912, 1016, 1016,
+ 670, 2915, 2915, 670, 670, 670, 670, 670, 670, 674,
+ 674, 2915, 674, 674, 674, 674, 674, 674, 674, 674,
+ 674, 674, 675, 675, 2915, 675, 675, 675, 675, 675,
+ 675, 675, 675, 675, 675, 1016, 1016, 2915, 1016, 1016,
1016, 1016, 1016, 1016, 1016, 1016, 1016, 1016, 676, 676,
- 2912, 2912, 676, 676, 676, 676, 676, 676, 680, 680,
- 2912, 680, 680, 680, 680, 680, 680, 680, 680, 680,
- 680, 681, 681, 2912, 681, 681, 681, 681, 681, 681,
- 681, 681, 681, 681, 1017, 1017, 2912, 1017, 1017, 1017,
- 1017, 1017, 1017, 1017, 1017, 1017, 1017, 682, 682, 2912,
- 2912, 682, 682, 682, 682, 682, 682, 686, 686, 2912,
+ 2915, 2915, 676, 676, 676, 676, 676, 676, 680, 680,
+ 2915, 680, 680, 680, 680, 680, 680, 680, 680, 680,
+ 680, 681, 681, 2915, 681, 681, 681, 681, 681, 681,
+ 681, 681, 681, 681, 1017, 1017, 2915, 1017, 1017, 1017,
+ 1017, 1017, 1017, 1017, 1017, 1017, 1017, 682, 682, 2915,
+ 2915, 682, 682, 682, 682, 682, 682, 686, 686, 2915,
686, 686, 686, 686, 686, 686, 686, 686, 686, 686,
- 687, 687, 2912, 687, 687, 687, 687, 687, 687, 687,
- 687, 687, 687, 1018, 1018, 2912, 1018, 1018, 1018, 1018,
+ 687, 687, 2915, 687, 687, 687, 687, 687, 687, 687,
+ 687, 687, 687, 1018, 1018, 2915, 1018, 1018, 1018, 1018,
1018, 1018, 1018, 1018, 1018, 1018, 688, 688, 688, 688,
- 688, 688, 2912, 688, 688, 688, 688, 688, 688, 689,
- 689, 2912, 2912, 689, 689, 689, 689, 689, 689, 693,
- 693, 2912, 693, 693, 693, 693, 693, 693, 693, 693,
- 693, 693, 694, 694, 2912, 694, 694, 694, 694, 694,
- 694, 694, 694, 694, 694, 1019, 1019, 2912, 1019, 1019,
+ 688, 688, 2915, 688, 688, 688, 688, 688, 688, 689,
+ 689, 2915, 2915, 689, 689, 689, 689, 689, 689, 693,
+ 693, 2915, 693, 693, 693, 693, 693, 693, 693, 693,
+ 693, 693, 694, 694, 2915, 694, 694, 694, 694, 694,
+ 694, 694, 694, 694, 694, 1019, 1019, 2915, 1019, 1019,
1019, 1019, 1019, 1019, 1019, 1019, 1019, 1019, 695, 695,
- 2912, 2912, 695, 695, 695, 695, 695, 695, 699, 699,
- 2912, 699, 699, 699, 699, 699, 699, 699, 699, 699,
- 699, 700, 700, 2912, 700, 700, 700, 700, 700, 700,
+ 2915, 2915, 695, 695, 695, 695, 695, 695, 699, 699,
+ 2915, 699, 699, 699, 699, 699, 699, 699, 699, 699,
+ 699, 700, 700, 2915, 700, 700, 700, 700, 700, 700,
- 700, 700, 700, 700, 1020, 1020, 2912, 1020, 1020, 1020,
- 1020, 1020, 1020, 1020, 1020, 1020, 1020, 701, 701, 2912,
- 2912, 701, 701, 701, 701, 701, 701, 705, 705, 2912,
+ 700, 700, 700, 700, 1020, 1020, 2915, 1020, 1020, 1020,
+ 1020, 1020, 1020, 1020, 1020, 1020, 1020, 701, 701, 2915,
+ 2915, 701, 701, 701, 701, 701, 701, 705, 705, 2915,
705, 705, 705, 705, 705, 705, 705, 705, 705, 705,
- 706, 706, 2912, 706, 706, 706, 706, 706, 706, 706,
- 706, 706, 706, 1021, 1021, 2912, 1021, 1021, 1021, 1021,
- 1021, 1021, 1021, 1021, 1021, 1021, 707, 707, 2912, 2912,
- 707, 707, 707, 707, 707, 707, 711, 711, 2912, 711,
+ 706, 706, 2915, 706, 706, 706, 706, 706, 706, 706,
+ 706, 706, 706, 1021, 1021, 2915, 1021, 1021, 1021, 1021,
+ 1021, 1021, 1021, 1021, 1021, 1021, 707, 707, 2915, 2915,
+ 707, 707, 707, 707, 707, 707, 711, 711, 2915, 711,
711, 711, 711, 711, 711, 711, 711, 711, 711, 712,
- 712, 2912, 712, 712, 712, 712, 712, 712, 712, 712,
+ 712, 2915, 712, 712, 712, 712, 712, 712, 712, 712,
- 712, 712, 1022, 1022, 2912, 1022, 1022, 1022, 1022, 1022,
+ 712, 712, 1022, 1022, 2915, 1022, 1022, 1022, 1022, 1022,
1022, 1022, 1022, 1022, 1022, 713, 713, 713, 713, 713,
713, 713, 713, 713, 713, 713, 713, 713, 718, 718,
718, 718, 718, 718, 718, 718, 718, 718, 718, 718,
@@ -4312,17 +4312,17 @@ static yyconst flex_int16_t yy_nxt[27006] =
1024, 1024, 1024, 1024, 1029, 1029, 1029, 1029, 1029, 1029,
1029, 1029, 1029, 1029, 1029, 1029, 1029, 1034, 1034, 1034,
1034, 1034, 1034, 1034, 1034, 1034, 1034, 1034, 1034, 1034,
- 722, 722, 2912, 722, 722, 722, 722, 722, 722, 722,
- 722, 722, 722, 1039, 1039, 2912, 1039, 1039, 1039, 1039,
+ 722, 722, 2915, 722, 722, 722, 722, 722, 722, 722,
+ 722, 722, 722, 1039, 1039, 2915, 1039, 1039, 1039, 1039,
- 1039, 1039, 1039, 1039, 1039, 1039, 1040, 1040, 2912, 1040,
+ 1039, 1039, 1039, 1039, 1039, 1039, 1040, 1040, 2915, 1040,
1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 726,
- 726, 2912, 726, 726, 726, 726, 726, 726, 726, 726,
- 726, 726, 742, 742, 2912, 742, 742, 742, 742, 742,
+ 726, 2915, 726, 726, 726, 726, 726, 726, 726, 726,
+ 726, 726, 742, 742, 2915, 742, 742, 742, 742, 742,
742, 742, 742, 742, 742, 743, 743, 743, 743, 743,
- 743, 743, 743, 743, 743, 743, 2912, 743, 751, 751,
- 2912, 751, 751, 751, 751, 751, 751, 751, 751, 751,
- 751, 752, 752, 2912, 752, 752, 752, 752, 752, 752,
+ 743, 743, 743, 743, 743, 743, 2915, 743, 751, 751,
+ 2915, 751, 751, 751, 751, 751, 751, 751, 751, 751,
+ 751, 752, 752, 2915, 752, 752, 752, 752, 752, 752,
752, 752, 752, 752, 738, 738, 738, 738, 738, 738,
738, 738, 738, 738, 738, 738, 738, 1043, 1043, 1043,
@@ -4332,149 +4332,149 @@ static yyconst flex_int16_t yy_nxt[27006] =
1053, 1053, 1053, 1053, 1053, 1053, 1058, 1058, 1058, 1058,
1058, 1058, 1058, 1058, 1058, 1058, 1058, 1058, 1058, 1059,
1059, 1059, 1059, 1059, 1059, 1059, 1059, 1059, 1059, 1059,
- 1059, 1059, 1060, 1060, 2912, 1060, 1060, 1060, 1060, 1060,
- 1060, 1060, 1060, 1060, 1060, 753, 753, 2912, 2912, 753,
- 753, 753, 753, 753, 753, 757, 757, 2912, 757, 757,
+ 1059, 1059, 1060, 1060, 2915, 1060, 1060, 1060, 1060, 1060,
+ 1060, 1060, 1060, 1060, 1060, 753, 753, 2915, 2915, 753,
+ 753, 753, 753, 753, 753, 757, 757, 2915, 757, 757,
757, 757, 757, 757, 757, 757, 757, 757, 758, 758,
- 2912, 758, 758, 758, 758, 758, 758, 758, 758, 758,
- 758, 1061, 1061, 2912, 1061, 1061, 1061, 1061, 1061, 1061,
- 1061, 1061, 1061, 1061, 759, 759, 2912, 2912, 759, 759,
- 759, 759, 759, 759, 763, 763, 2912, 763, 763, 763,
- 763, 763, 763, 763, 763, 763, 763, 764, 764, 2912,
+ 2915, 758, 758, 758, 758, 758, 758, 758, 758, 758,
+ 758, 1061, 1061, 2915, 1061, 1061, 1061, 1061, 1061, 1061,
+ 1061, 1061, 1061, 1061, 759, 759, 2915, 2915, 759, 759,
+ 759, 759, 759, 759, 763, 763, 2915, 763, 763, 763,
+ 763, 763, 763, 763, 763, 763, 763, 764, 764, 2915,
764, 764, 764, 764, 764, 764, 764, 764, 764, 764,
- 1062, 1062, 2912, 1062, 1062, 1062, 1062, 1062, 1062, 1062,
- 1062, 1062, 1062, 765, 765, 2912, 2912, 765, 765, 765,
- 765, 765, 765, 769, 769, 2912, 769, 769, 769, 769,
- 769, 769, 769, 769, 769, 769, 770, 770, 2912, 770,
+ 1062, 1062, 2915, 1062, 1062, 1062, 1062, 1062, 1062, 1062,
+ 1062, 1062, 1062, 765, 765, 2915, 2915, 765, 765, 765,
+ 765, 765, 765, 769, 769, 2915, 769, 769, 769, 769,
+ 769, 769, 769, 769, 769, 769, 770, 770, 2915, 770,
770, 770, 770, 770, 770, 770, 770, 770, 770, 1063,
- 1063, 2912, 1063, 1063, 1063, 1063, 1063, 1063, 1063, 1063,
- 1063, 1063, 771, 771, 2912, 2912, 771, 771, 771, 771,
- 771, 771, 775, 775, 2912, 775, 775, 775, 775, 775,
- 775, 775, 775, 775, 775, 776, 776, 2912, 776, 776,
+ 1063, 2915, 1063, 1063, 1063, 1063, 1063, 1063, 1063, 1063,
+ 1063, 1063, 771, 771, 2915, 2915, 771, 771, 771, 771,
+ 771, 771, 775, 775, 2915, 775, 775, 775, 775, 775,
+ 775, 775, 775, 775, 775, 776, 776, 2915, 776, 776,
776, 776, 776, 776, 776, 776, 776, 776, 1064, 1064,
- 2912, 1064, 1064, 1064, 1064, 1064, 1064, 1064, 1064, 1064,
- 1064, 777, 2912, 777, 777, 2912, 777, 777, 777, 777,
- 777, 777, 777, 777, 778, 2912, 778, 778, 2912, 778,
- 778, 778, 2912, 778, 778, 778, 778, 779, 779, 2912,
+ 2915, 1064, 1064, 1064, 1064, 1064, 1064, 1064, 1064, 1064,
+ 1064, 777, 2915, 777, 777, 2915, 777, 777, 777, 777,
+ 777, 777, 777, 777, 778, 2915, 778, 778, 2915, 778,
+ 778, 778, 2915, 778, 778, 778, 778, 779, 779, 2915,
- 2912, 779, 779, 779, 779, 779, 779, 783, 783, 2912,
+ 2915, 779, 779, 779, 779, 779, 779, 783, 783, 2915,
783, 783, 783, 783, 783, 783, 783, 783, 783, 783,
- 784, 784, 2912, 784, 784, 784, 784, 784, 784, 784,
- 784, 784, 784, 1065, 1065, 2912, 1065, 1065, 1065, 1065,
- 1065, 1065, 1065, 1065, 1065, 1065, 785, 2912, 785, 785,
- 2912, 785, 785, 785, 785, 785, 785, 785, 785, 787,
- 787, 2912, 2912, 787, 787, 787, 787, 787, 787, 791,
- 791, 2912, 791, 791, 791, 791, 791, 791, 791, 791,
- 791, 791, 792, 792, 2912, 792, 792, 792, 792, 792,
- 792, 792, 792, 792, 792, 1067, 1067, 2912, 1067, 1067,
-
- 1067, 1067, 1067, 1067, 1067, 1067, 1067, 1067, 793, 2912,
- 793, 793, 2912, 793, 793, 793, 793, 793, 793, 793,
- 793, 795, 795, 2912, 2912, 795, 795, 795, 795, 795,
- 795, 799, 799, 2912, 799, 799, 799, 799, 799, 799,
- 799, 799, 799, 799, 800, 800, 2912, 800, 800, 800,
- 800, 800, 800, 800, 800, 800, 800, 1069, 1069, 2912,
+ 784, 784, 2915, 784, 784, 784, 784, 784, 784, 784,
+ 784, 784, 784, 1065, 1065, 2915, 1065, 1065, 1065, 1065,
+ 1065, 1065, 1065, 1065, 1065, 1065, 785, 2915, 785, 785,
+ 2915, 785, 785, 785, 785, 785, 785, 785, 785, 787,
+ 787, 2915, 2915, 787, 787, 787, 787, 787, 787, 791,
+ 791, 2915, 791, 791, 791, 791, 791, 791, 791, 791,
+ 791, 791, 792, 792, 2915, 792, 792, 792, 792, 792,
+ 792, 792, 792, 792, 792, 1067, 1067, 2915, 1067, 1067,
+
+ 1067, 1067, 1067, 1067, 1067, 1067, 1067, 1067, 793, 2915,
+ 793, 793, 2915, 793, 793, 793, 793, 793, 793, 793,
+ 793, 795, 795, 2915, 2915, 795, 795, 795, 795, 795,
+ 795, 799, 799, 2915, 799, 799, 799, 799, 799, 799,
+ 799, 799, 799, 799, 800, 800, 2915, 800, 800, 800,
+ 800, 800, 800, 800, 800, 800, 800, 1069, 1069, 2915,
1069, 1069, 1069, 1069, 1069, 1069, 1069, 1069, 1069, 1069,
- 801, 801, 2912, 2912, 801, 801, 801, 801, 801, 801,
- 805, 805, 2912, 805, 805, 805, 805, 805, 805, 805,
- 805, 805, 805, 806, 806, 2912, 806, 806, 806, 806,
+ 801, 801, 2915, 2915, 801, 801, 801, 801, 801, 801,
+ 805, 805, 2915, 805, 805, 805, 805, 805, 805, 805,
+ 805, 805, 805, 806, 806, 2915, 806, 806, 806, 806,
- 806, 806, 806, 806, 806, 806, 1070, 1070, 2912, 1070,
+ 806, 806, 806, 806, 806, 806, 1070, 1070, 2915, 1070,
1070, 1070, 1070, 1070, 1070, 1070, 1070, 1070, 1070, 807,
- 807, 2912, 2912, 807, 807, 807, 807, 807, 807, 811,
- 811, 2912, 811, 811, 811, 811, 811, 811, 811, 811,
- 811, 811, 812, 812, 2912, 812, 812, 812, 812, 812,
- 812, 812, 812, 812, 812, 1071, 1071, 2912, 1071, 1071,
+ 807, 2915, 2915, 807, 807, 807, 807, 807, 807, 811,
+ 811, 2915, 811, 811, 811, 811, 811, 811, 811, 811,
+ 811, 811, 812, 812, 2915, 812, 812, 812, 812, 812,
+ 812, 812, 812, 812, 812, 1071, 1071, 2915, 1071, 1071,
1071, 1071, 1071, 1071, 1071, 1071, 1071, 1071, 813, 813,
- 2912, 813, 813, 813, 813, 813, 813, 813, 813, 813,
- 813, 1072, 1072, 2912, 1072, 1072, 1072, 1072, 1072, 1072,
- 1072, 1072, 1072, 1072, 817, 817, 2912, 2912, 817, 817,
+ 2915, 813, 813, 813, 813, 813, 813, 813, 813, 813,
+ 813, 1072, 1072, 2915, 1072, 1072, 1072, 1072, 1072, 1072,
+ 1072, 1072, 1072, 1072, 817, 817, 2915, 2915, 817, 817,
- 817, 817, 817, 817, 821, 821, 2912, 821, 821, 821,
- 821, 821, 821, 821, 821, 821, 821, 822, 822, 2912,
+ 817, 817, 817, 817, 821, 821, 2915, 821, 821, 821,
+ 821, 821, 821, 821, 821, 821, 821, 822, 822, 2915,
822, 822, 822, 822, 822, 822, 822, 822, 822, 822,
- 1073, 1073, 2912, 1073, 1073, 1073, 1073, 1073, 1073, 1073,
- 1073, 1073, 1073, 823, 823, 2912, 2912, 823, 823, 823,
- 823, 823, 823, 827, 827, 2912, 827, 827, 827, 827,
- 827, 827, 827, 827, 827, 827, 828, 828, 2912, 828,
+ 1073, 1073, 2915, 1073, 1073, 1073, 1073, 1073, 1073, 1073,
+ 1073, 1073, 1073, 823, 823, 2915, 2915, 823, 823, 823,
+ 823, 823, 823, 827, 827, 2915, 827, 827, 827, 827,
+ 827, 827, 827, 827, 827, 827, 828, 828, 2915, 828,
828, 828, 828, 828, 828, 828, 828, 828, 828, 1074,
- 1074, 2912, 1074, 1074, 1074, 1074, 1074, 1074, 1074, 1074,
- 1074, 1074, 829, 829, 2912, 2912, 829, 829, 829, 829,
+ 1074, 2915, 1074, 1074, 1074, 1074, 1074, 1074, 1074, 1074,
+ 1074, 1074, 829, 829, 2915, 2915, 829, 829, 829, 829,
- 829, 829, 833, 833, 2912, 833, 833, 833, 833, 833,
- 833, 833, 833, 833, 833, 834, 834, 2912, 834, 834,
+ 829, 829, 833, 833, 2915, 833, 833, 833, 833, 833,
+ 833, 833, 833, 833, 833, 834, 834, 2915, 834, 834,
834, 834, 834, 834, 834, 834, 834, 834, 1075, 1075,
- 2912, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075,
- 1075, 835, 835, 2912, 835, 835, 835, 835, 835, 835,
- 835, 835, 835, 835, 1076, 1076, 2912, 1076, 1076, 1076,
- 1076, 1076, 1076, 1076, 1076, 1076, 1076, 838, 838, 2912,
- 2912, 838, 838, 838, 838, 838, 838, 842, 842, 2912,
+ 2915, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075,
+ 1075, 835, 835, 2915, 835, 835, 835, 835, 835, 835,
+ 835, 835, 835, 835, 1076, 1076, 2915, 1076, 1076, 1076,
+ 1076, 1076, 1076, 1076, 1076, 1076, 1076, 838, 838, 2915,
+ 2915, 838, 838, 838, 838, 838, 838, 842, 842, 2915,
842, 842, 842, 842, 842, 842, 842, 842, 842, 842,
- 843, 843, 2912, 843, 843, 843, 843, 843, 843, 843,
+ 843, 843, 2915, 843, 843, 843, 843, 843, 843, 843,
- 843, 843, 843, 1077, 1077, 2912, 1077, 1077, 1077, 1077,
- 1077, 1077, 1077, 1077, 1077, 1077, 844, 844, 2912, 2912,
- 844, 844, 844, 844, 844, 844, 848, 848, 2912, 848,
+ 843, 843, 843, 1077, 1077, 2915, 1077, 1077, 1077, 1077,
+ 1077, 1077, 1077, 1077, 1077, 1077, 844, 844, 2915, 2915,
+ 844, 844, 844, 844, 844, 844, 848, 848, 2915, 848,
848, 848, 848, 848, 848, 848, 848, 848, 848, 849,
- 849, 2912, 849, 849, 849, 849, 849, 849, 849, 849,
- 849, 849, 1078, 1078, 2912, 1078, 1078, 1078, 1078, 1078,
- 1078, 1078, 1078, 1078, 1078, 850, 850, 2912, 2912, 850,
- 850, 850, 850, 850, 850, 854, 854, 2912, 854, 854,
+ 849, 2915, 849, 849, 849, 849, 849, 849, 849, 849,
+ 849, 849, 1078, 1078, 2915, 1078, 1078, 1078, 1078, 1078,
+ 1078, 1078, 1078, 1078, 1078, 850, 850, 2915, 2915, 850,
+ 850, 850, 850, 850, 850, 854, 854, 2915, 854, 854,
854, 854, 854, 854, 854, 854, 854, 854, 855, 855,
- 2912, 855, 855, 855, 855, 855, 855, 855, 855, 855,
+ 2915, 855, 855, 855, 855, 855, 855, 855, 855, 855,
- 855, 1079, 1079, 2912, 1079, 1079, 1079, 1079, 1079, 1079,
- 1079, 1079, 1079, 1079, 856, 856, 2912, 2912, 856, 856,
- 856, 856, 856, 856, 860, 860, 2912, 860, 860, 860,
- 860, 860, 860, 860, 860, 860, 860, 861, 861, 2912,
+ 855, 1079, 1079, 2915, 1079, 1079, 1079, 1079, 1079, 1079,
+ 1079, 1079, 1079, 1079, 856, 856, 2915, 2915, 856, 856,
+ 856, 856, 856, 856, 860, 860, 2915, 860, 860, 860,
+ 860, 860, 860, 860, 860, 860, 860, 861, 861, 2915,
861, 861, 861, 861, 861, 861, 861, 861, 861, 861,
- 1080, 1080, 2912, 1080, 1080, 1080, 1080, 1080, 1080, 1080,
- 1080, 1080, 1080, 450, 450, 450, 450, 450, 2912, 450,
+ 1080, 1080, 2915, 1080, 1080, 1080, 1080, 1080, 1080, 1080,
+ 1080, 1080, 1080, 450, 450, 450, 450, 450, 2915, 450,
450, 450, 450, 976, 976, 976, 976, 976, 976, 976,
- 976, 976, 976, 976, 976, 976, 550, 550, 2912, 550,
+ 976, 976, 976, 976, 976, 976, 550, 550, 2915, 550,
550, 550, 550, 550, 550, 550, 550, 550, 550, 551,
- 551, 2912, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 552, 552, 2912, 552, 552, 552, 552, 552,
- 552, 552, 552, 552, 552, 1281, 1281, 2912, 1281, 1281,
+ 551, 2915, 551, 551, 551, 551, 551, 551, 551, 551,
+ 551, 551, 552, 552, 2915, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 1281, 1281, 2915, 1281, 1281,
1281, 1281, 1281, 1281, 1281, 1281, 1281, 1281, 557, 557,
- 2912, 557, 557, 557, 557, 557, 557, 557, 557, 557,
- 557, 562, 2912, 562, 562, 2912, 562, 562, 562, 562,
- 562, 562, 562, 562, 570, 570, 2912, 570, 570, 570,
- 570, 570, 570, 570, 570, 570, 570, 576, 576, 2912,
+ 2915, 557, 557, 557, 557, 557, 557, 557, 557, 557,
+ 557, 562, 2915, 562, 562, 2915, 562, 562, 562, 562,
+ 562, 562, 562, 562, 570, 570, 2915, 570, 570, 570,
+ 570, 570, 570, 570, 570, 570, 570, 576, 576, 2915,
576, 576, 576, 576, 576, 576, 576, 576, 576, 576,
582, 582, 582, 582, 582, 582, 582, 582, 582, 582,
- 582, 582, 582, 588, 588, 2912, 588, 588, 588, 588,
- 588, 588, 588, 588, 588, 588, 596, 596, 2912, 596,
+ 582, 582, 582, 588, 588, 2915, 588, 588, 588, 588,
+ 588, 588, 588, 588, 588, 588, 596, 596, 2915, 596,
596, 596, 596, 596, 596, 596, 596, 596, 596, 602,
- 602, 2912, 602, 602, 602, 602, 602, 602, 602, 602,
- 602, 602, 608, 608, 2912, 608, 608, 608, 608, 608,
- 608, 608, 608, 608, 608, 614, 614, 2912, 614, 614,
+ 602, 2915, 602, 602, 602, 602, 602, 602, 602, 602,
+ 602, 602, 608, 608, 2915, 608, 608, 608, 608, 608,
+ 608, 608, 608, 608, 608, 614, 614, 2915, 614, 614,
614, 614, 614, 614, 614, 614, 614, 614, 620, 620,
- 2912, 620, 620, 620, 620, 620, 620, 620, 620, 620,
- 620, 626, 626, 2912, 626, 626, 626, 626, 626, 626,
- 626, 626, 626, 626, 632, 632, 2912, 632, 632, 632,
+ 2915, 620, 620, 620, 620, 620, 620, 620, 620, 620,
+ 620, 626, 626, 2915, 626, 626, 626, 626, 626, 626,
+ 626, 626, 626, 626, 632, 632, 2915, 632, 632, 632,
- 632, 632, 632, 632, 632, 632, 632, 638, 638, 2912,
+ 632, 632, 632, 632, 632, 632, 632, 638, 638, 2915,
638, 638, 638, 638, 638, 638, 638, 638, 638, 638,
- 644, 644, 2912, 644, 644, 644, 644, 644, 644, 644,
- 644, 644, 644, 650, 650, 2912, 650, 650, 650, 650,
- 650, 650, 650, 650, 650, 650, 656, 656, 2912, 656,
+ 644, 644, 2915, 644, 644, 644, 644, 644, 644, 644,
+ 644, 644, 644, 650, 650, 2915, 650, 650, 650, 650,
+ 650, 650, 650, 650, 650, 650, 656, 656, 2915, 656,
656, 656, 656, 656, 656, 656, 656, 656, 656, 664,
- 664, 2912, 664, 664, 664, 664, 664, 664, 664, 664,
- 664, 664, 670, 670, 2912, 670, 670, 670, 670, 670,
- 670, 670, 670, 670, 670, 676, 676, 2912, 676, 676,
+ 664, 2915, 664, 664, 664, 664, 664, 664, 664, 664,
+ 664, 664, 670, 670, 2915, 670, 670, 670, 670, 670,
+ 670, 670, 670, 670, 670, 676, 676, 2915, 676, 676,
676, 676, 676, 676, 676, 676, 676, 676, 682, 682,
- 2912, 682, 682, 682, 682, 682, 682, 682, 682, 682,
- 682, 689, 689, 2912, 689, 689, 689, 689, 689, 689,
- 689, 689, 689, 689, 695, 695, 2912, 695, 695, 695,
- 695, 695, 695, 695, 695, 695, 695, 701, 701, 2912,
+ 2915, 682, 682, 682, 682, 682, 682, 682, 682, 682,
+ 682, 689, 689, 2915, 689, 689, 689, 689, 689, 689,
+ 689, 689, 689, 689, 695, 695, 2915, 695, 695, 695,
+ 695, 695, 695, 695, 695, 695, 695, 701, 701, 2915,
701, 701, 701, 701, 701, 701, 701, 701, 701, 701,
- 707, 707, 2912, 707, 707, 707, 707, 707, 707, 707,
+ 707, 707, 2915, 707, 707, 707, 707, 707, 707, 707,
707, 707, 707, 718, 718, 718, 718, 718, 718, 718,
718, 718, 718, 718, 718, 718, 1024, 1024, 1024, 1024,
1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1029,
@@ -4486,15 +4486,15 @@ static yyconst flex_int16_t yy_nxt[27006] =
1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287,
1287, 1292, 1292, 1292, 1292, 1292, 1292, 1292, 1292, 1292,
1292, 1292, 1292, 1292, 1297, 1297, 1297, 1297, 1297, 1297,
- 1297, 1297, 1297, 1297, 1297, 1297, 1297, 1302, 1302, 2912,
+ 1297, 1297, 1297, 1297, 1297, 1297, 1297, 1302, 1302, 2915,
1302, 1302, 1302, 1302, 1302, 1302, 1302, 1302, 1302, 1302,
1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303,
1303, 1303, 1303, 1308, 1308, 1308, 1308, 1308, 1308, 1308,
1308, 1308, 1308, 1308, 1308, 1308, 1313, 1313, 1313, 1313,
1313, 1313, 1313, 1313, 1313, 1313, 1313, 1313, 1313, 722,
- 722, 2912, 722, 722, 722, 722, 722, 722, 722, 722,
- 722, 722, 1318, 1318, 2912, 1318, 1318, 1318, 1318, 1318,
+ 722, 2915, 722, 722, 722, 722, 722, 722, 722, 722,
+ 722, 722, 1318, 1318, 2915, 1318, 1318, 1318, 1318, 1318,
1318, 1318, 1318, 1318, 1318, 738, 738, 738, 738, 738,
738, 738, 738, 738, 738, 738, 738, 738, 1043, 1043,
1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043,
@@ -4507,53 +4507,53 @@ static yyconst flex_int16_t yy_nxt[27006] =
1324, 1324, 1324, 1329, 1329, 1329, 1329, 1329, 1329, 1329,
1329, 1329, 1329, 1329, 1329, 1329, 1334, 1334, 1334, 1334,
1334, 1334, 1334, 1334, 1334, 1334, 1334, 1334, 1334, 1339,
- 1339, 2912, 1339, 1339, 1339, 1339, 1339, 1339, 1339, 1339,
+ 1339, 2915, 1339, 1339, 1339, 1339, 1339, 1339, 1339, 1339,
1339, 1339, 1340, 1340, 1340, 1340, 1340, 1340, 1340, 1340,
1340, 1340, 1340, 1340, 1340, 1345, 1345, 1345, 1345, 1345,
1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1350, 1350,
1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350,
- 1350, 726, 726, 2912, 726, 726, 726, 726, 726, 726,
- 726, 726, 726, 726, 753, 753, 2912, 753, 753, 753,
- 753, 753, 753, 753, 753, 753, 753, 759, 759, 2912,
+ 1350, 726, 726, 2915, 726, 726, 726, 726, 726, 726,
+ 726, 726, 726, 726, 753, 753, 2915, 753, 753, 753,
+ 753, 753, 753, 753, 753, 753, 753, 759, 759, 2915,
759, 759, 759, 759, 759, 759, 759, 759, 759, 759,
- 765, 765, 2912, 765, 765, 765, 765, 765, 765, 765,
- 765, 765, 765, 771, 771, 2912, 771, 771, 771, 771,
- 771, 771, 771, 771, 771, 771, 779, 779, 2912, 779,
+ 765, 765, 2915, 765, 765, 765, 765, 765, 765, 765,
+ 765, 765, 765, 771, 771, 2915, 771, 771, 771, 771,
+ 771, 771, 771, 771, 771, 771, 779, 779, 2915, 779,
779, 779, 779, 779, 779, 779, 779, 779, 779, 787,
- 787, 2912, 787, 787, 787, 787, 787, 787, 787, 787,
- 787, 787, 795, 795, 2912, 795, 795, 795, 795, 795,
+ 787, 2915, 787, 787, 787, 787, 787, 787, 787, 787,
+ 787, 787, 795, 795, 2915, 795, 795, 795, 795, 795,
- 795, 795, 795, 795, 795, 801, 801, 2912, 801, 801,
+ 795, 795, 795, 795, 795, 801, 801, 2915, 801, 801,
801, 801, 801, 801, 801, 801, 801, 801, 807, 807,
- 2912, 807, 807, 807, 807, 807, 807, 807, 807, 807,
- 807, 813, 813, 2912, 813, 813, 813, 813, 813, 813,
- 813, 813, 813, 813, 817, 817, 2912, 817, 817, 817,
- 817, 817, 817, 817, 817, 817, 817, 823, 823, 2912,
+ 2915, 807, 807, 807, 807, 807, 807, 807, 807, 807,
+ 807, 813, 813, 2915, 813, 813, 813, 813, 813, 813,
+ 813, 813, 813, 813, 817, 817, 2915, 817, 817, 817,
+ 817, 817, 817, 817, 817, 817, 817, 823, 823, 2915,
823, 823, 823, 823, 823, 823, 823, 823, 823, 823,
- 829, 829, 2912, 829, 829, 829, 829, 829, 829, 829,
- 829, 829, 829, 835, 835, 2912, 835, 835, 835, 835,
- 835, 835, 835, 835, 835, 835, 838, 838, 2912, 838,
+ 829, 829, 2915, 829, 829, 829, 829, 829, 829, 829,
+ 829, 829, 829, 835, 835, 2915, 835, 835, 835, 835,
+ 835, 835, 835, 835, 835, 835, 838, 838, 2915, 838,
838, 838, 838, 838, 838, 838, 838, 838, 838, 844,
- 844, 2912, 844, 844, 844, 844, 844, 844, 844, 844,
- 844, 844, 850, 850, 2912, 850, 850, 850, 850, 850,
- 850, 850, 850, 850, 850, 856, 856, 2912, 856, 856,
+ 844, 2915, 844, 844, 844, 844, 844, 844, 844, 844,
+ 844, 844, 850, 850, 2915, 850, 850, 850, 850, 850,
+ 850, 850, 850, 850, 850, 856, 856, 2915, 856, 856,
856, 856, 856, 856, 856, 856, 856, 856, 450, 450,
- 450, 450, 450, 2912, 450, 450, 450, 450, 1362, 1362,
+ 450, 450, 450, 2915, 450, 450, 450, 450, 1362, 1362,
1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362,
1362, 1373, 1373, 1373, 1373, 1373, 1373, 1373, 1373, 1373,
1373, 1373, 1373, 1373, 1403, 1403, 1403, 1403, 1403, 1403,
1403, 1403, 1403, 1403, 1403, 1403, 1403, 1135, 1135, 1135,
1135, 1135, 1135, 1135, 1135, 1135, 1135, 1135, 1135, 1135,
- 550, 550, 2912, 550, 550, 550, 550, 550, 550, 550,
- 550, 550, 550, 551, 551, 2912, 551, 551, 551, 551,
+ 550, 550, 2915, 550, 550, 550, 550, 550, 550, 550,
+ 550, 550, 550, 551, 551, 2915, 551, 551, 551, 551,
551, 551, 551, 551, 551, 551, 1467, 1467, 1467, 1467,
1467, 1467, 1467, 1467, 1467, 1467, 1467, 1467, 1467, 1550,
1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550,
1550, 1550, 1607, 1607, 1607, 1607, 1607, 1607, 1607, 1607,
- 1607, 1607, 1607, 1607, 1607, 552, 552, 2912, 552, 552,
+ 1607, 1607, 1607, 1607, 1607, 552, 552, 2915, 552, 552,
552, 552, 552, 552, 552, 552, 552, 552, 718, 718,
718, 718, 718, 718, 718, 718, 718, 718, 718, 718,
@@ -4580,7 +4580,7 @@ static yyconst flex_int16_t yy_nxt[27006] =
1643, 1643, 1643, 1643, 1643, 1643, 1648, 1648, 1648, 1648,
1648, 1648, 1648, 1648, 1648, 1648, 1648, 1648, 1648, 722,
- 722, 2912, 722, 722, 722, 722, 722, 722, 722, 722,
+ 722, 2915, 722, 722, 722, 722, 722, 722, 722, 722,
722, 722, 738, 738, 738, 738, 738, 738, 738, 738,
738, 738, 738, 738, 738, 1043, 1043, 1043, 1043, 1043,
1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1048, 1048,
@@ -4605,23 +4605,23 @@ static yyconst flex_int16_t yy_nxt[27006] =
1668, 1668, 1668, 1668, 1668, 1668, 1668, 1673, 1673, 1673,
1673, 1673, 1673, 1673, 1673, 1673, 1673, 1673, 1673, 1673,
1678, 1678, 1678, 1678, 1678, 1678, 1678, 1678, 1678, 1678,
- 1678, 1678, 1678, 450, 450, 450, 450, 450, 2912, 450,
- 450, 450, 450, 550, 550, 2912, 550, 550, 550, 550,
- 550, 550, 550, 550, 550, 550, 551, 551, 2912, 551,
+ 1678, 1678, 1678, 450, 450, 450, 450, 450, 2915, 450,
+ 450, 450, 450, 550, 550, 2915, 550, 550, 550, 550,
+ 550, 550, 550, 550, 550, 550, 551, 551, 2915, 551,
551, 551, 551, 551, 551, 551, 551, 551, 551, 1362,
1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362,
1362, 1362, 1373, 1373, 1373, 1373, 1373, 1373, 1373, 1373,
1373, 1373, 1373, 1373, 1373, 1403, 1403, 1403, 1403, 1403,
1403, 1403, 1403, 1403, 1403, 1403, 1403, 1403, 1413, 1413,
- 1413, 1413, 1413, 1413, 1413, 1413, 2912, 1413, 1413, 1413,
+ 1413, 1413, 1413, 1413, 1413, 1413, 2915, 1413, 1413, 1413,
1413, 1421, 1421, 1421, 1421, 1421, 1421, 1421, 1421, 1421,
1421, 1421, 1421, 1421, 1734, 1734, 1734, 1734, 1734, 1734,
1734, 1734, 1734, 1734, 1734, 1734, 1734, 1476, 1476, 1476,
- 1476, 1476, 1476, 1476, 1476, 2912, 1476, 1476, 1476, 1476,
+ 1476, 1476, 1476, 1476, 1476, 2915, 1476, 1476, 1476, 1476,
1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550,
- 1550, 1550, 1550, 1864, 1864, 1864, 1864, 1864, 1864, 1864,
- 1864, 1864, 1864, 1864, 1864, 1864, 1607, 1607, 1607, 1607,
+ 1550, 1550, 1550, 1866, 1866, 1866, 1866, 1866, 1866, 1866,
+ 1866, 1866, 1866, 1866, 1866, 1866, 1607, 1607, 1607, 1607,
1607, 1607, 1607, 1607, 1607, 1607, 1607, 1607, 1607, 718,
718, 718, 718, 718, 718, 718, 718, 718, 718, 718,
@@ -4673,45 +4673,45 @@ static yyconst flex_int16_t yy_nxt[27006] =
1673, 1673, 1673, 1673, 1673, 1673, 1673, 1673, 1678, 1678,
1678, 1678, 1678, 1678, 1678, 1678, 1678, 1678, 1678, 1678,
1678, 726, 726, 726, 726, 726, 726, 726, 726, 726,
- 726, 726, 726, 450, 450, 450, 450, 450, 2912, 450,
+ 726, 726, 726, 450, 450, 450, 450, 450, 2915, 450,
450, 450, 450, 1373, 1373, 1373, 1373, 1373, 1373, 1373,
- 1373, 2912, 1373, 1373, 1373, 1373, 1413, 1413, 1413, 1413,
- 1413, 1413, 1413, 1413, 2912, 1413, 1413, 1413, 1413, 550,
+ 1373, 2915, 1373, 1373, 1373, 1373, 1413, 1413, 1413, 1413,
+ 1413, 1413, 1413, 1413, 2915, 1413, 1413, 1413, 1413, 550,
- 550, 2912, 550, 550, 550, 550, 550, 550, 550, 550,
- 550, 550, 551, 551, 2912, 551, 551, 551, 551, 551,
+ 550, 2915, 550, 550, 550, 550, 550, 550, 550, 550,
+ 550, 550, 551, 551, 2915, 551, 551, 551, 551, 551,
551, 551, 551, 551, 551, 1734, 1734, 1734, 1734, 1734,
- 1734, 1734, 1734, 1734, 1734, 1734, 1734, 1734, 2011, 2011,
- 2011, 2011, 2011, 2011, 2011, 2011, 2912, 2011, 2011, 2011,
- 2011, 994, 994, 2912, 994, 994, 994, 994, 994, 994,
- 994, 994, 994, 994, 2022, 2022, 2022, 2022, 2022, 2022,
- 2022, 2022, 2022, 2022, 2022, 2022, 2022, 1476, 1476, 1476,
- 1476, 1476, 1476, 1476, 1476, 2912, 1476, 1476, 1476, 1476,
- 2091, 2091, 2091, 2091, 2091, 2091, 2091, 2091, 2091, 2091,
-
- 2091, 2091, 2091, 1864, 1864, 1864, 1864, 1864, 1864, 1864,
- 1864, 1864, 1864, 1864, 1864, 1864, 1607, 1607, 1607, 1607,
- 1607, 1607, 1607, 1607, 2912, 1607, 1607, 1607, 1607, 1403,
+ 1734, 1734, 1734, 1734, 1734, 1734, 1734, 1734, 2013, 2013,
+ 2013, 2013, 2013, 2013, 2013, 2013, 2915, 2013, 2013, 2013,
+ 2013, 994, 994, 2915, 994, 994, 994, 994, 994, 994,
+ 994, 994, 994, 994, 2024, 2024, 2024, 2024, 2024, 2024,
+ 2024, 2024, 2024, 2024, 2024, 2024, 2024, 1476, 1476, 1476,
+ 1476, 1476, 1476, 1476, 1476, 2915, 1476, 1476, 1476, 1476,
+ 2093, 2093, 2093, 2093, 2093, 2093, 2093, 2093, 2093, 2093,
+
+ 2093, 2093, 2093, 1866, 1866, 1866, 1866, 1866, 1866, 1866,
+ 1866, 1866, 1866, 1866, 1866, 1866, 1607, 1607, 1607, 1607,
+ 1607, 1607, 1607, 1607, 2915, 1607, 1607, 1607, 1607, 1403,
1403, 1403, 1403, 1403, 1403, 1403, 1403, 1403, 1403, 1403,
1403, 1403, 1467, 1467, 1467, 1467, 1467, 1467, 1467, 1467,
1467, 1467, 1467, 1467, 1467, 976, 976, 976, 976, 976,
- 976, 976, 976, 976, 976, 976, 976, 976, 2172, 2172,
- 2172, 2172, 2172, 2172, 2172, 2172, 2172, 2172, 2172, 2172,
- 2172, 1135, 1135, 1135, 1135, 1135, 1135, 1135, 1135, 1135,
+ 976, 976, 976, 976, 976, 976, 976, 976, 2174, 2174,
+ 2174, 2174, 2174, 2174, 2174, 2174, 2174, 2174, 2174, 2174,
+ 2174, 1135, 1135, 1135, 1135, 1135, 1135, 1135, 1135, 1135,
1135, 1135, 1135, 1135, 1421, 1421, 1421, 1421, 1421, 1421,
- 1421, 1421, 1421, 1421, 1421, 1421, 1421, 2812, 2812, 2812,
- 2812, 2812, 2812, 2812, 2812, 2812, 2812, 2812, 2812, 2812,
- 123, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
-
- 2912, 2912, 2912, 2912, 2912
+ 1421, 1421, 1421, 1421, 1421, 1421, 1421, 2815, 2815, 2815,
+ 2815, 2815, 2815, 2815, 2815, 2815, 2815, 2815, 2815, 2815,
+ 123, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+
+ 2915, 2915, 2915, 2915, 2915
} ;
static yyconst flex_int16_t yy_chk[27006] =
@@ -4916,7 +4916,7 @@ static yyconst flex_int16_t yy_chk[27006] =
49, 50, 469, 50, 50, 51, 469, 51, 51, 52,
51, 52, 52, 53, 52, 53, 53, 54, 134, 54,
- 54, 63, 134, 63, 63, 64, 3329, 64, 64, 138,
+ 54, 63, 134, 63, 63, 64, 3332, 64, 64, 138,
65, 63, 65, 65, 66, 64, 66, 66, 138, 134,
152, 65, 152, 65, 517, 66, 67, 66, 67, 67,
68, 517, 68, 68, 142, 79, 79, 67, 79, 79,
@@ -5004,10 +5004,10 @@ static yyconst flex_int16_t yy_chk[27006] =
77, 77, 77, 77, 77, 77, 77, 77, 77, 77,
77, 77, 77, 77, 77, 77, 77, 77, 77, 77,
- 77, 77, 77, 77, 80, 80, 3187, 80, 80, 81,
+ 77, 77, 77, 77, 80, 80, 3190, 80, 80, 81,
81, 462, 81, 81, 82, 82, 488, 82, 82, 86,
462, 86, 86, 87, 87, 488, 87, 87, 466, 88,
- 88, 81, 88, 88, 466, 3008, 82, 83, 83, 83,
+ 88, 81, 88, 88, 466, 3011, 82, 83, 83, 83,
83, 83, 83, 83, 83, 83, 83, 83, 83, 83,
83, 83, 83, 83, 83, 83, 83, 83, 83, 83,
83, 83, 83, 83, 83, 83, 83, 83, 83, 83,
@@ -5017,7 +5017,7 @@ static yyconst flex_int16_t yy_chk[27006] =
83, 83, 83, 83, 83, 83, 83, 83, 83, 83,
83, 83, 83, 83, 83, 83, 83, 83, 83, 83,
83, 83, 83, 83, 83, 83, 83, 83, 83, 83,
- 83, 89, 2910, 89, 89, 90, 200, 90, 90, 93,
+ 83, 89, 2913, 89, 89, 90, 200, 90, 90, 93,
93, 201, 93, 93, 94, 94, 135, 94, 94, 135,
490, 490, 135, 135, 135, 137, 137, 200, 137, 486,
89, 137, 201, 486, 90, 91, 91, 91, 91, 91,
@@ -5030,7 +5030,7 @@ static yyconst flex_int16_t yy_chk[27006] =
91, 91, 91, 91, 91, 91, 91, 91, 91, 91,
91, 91, 91, 91, 91, 91, 91, 91, 91, 91,
91, 91, 91, 91, 91, 91, 91, 91, 91, 95,
- 2907, 95, 95, 96, 139, 96, 96, 154, 105, 452,
+ 2910, 95, 95, 96, 139, 96, 96, 154, 105, 452,
105, 105, 139, 452, 143, 499, 493, 139, 143, 493,
154, 154, 143, 154, 139, 520, 499, 143, 95, 105,
452, 520, 96, 97, 97, 97, 97, 97, 97, 97,
@@ -5071,8 +5071,8 @@ static yyconst flex_int16_t yy_chk[27006] =
103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
103, 103, 103, 103, 103, 103, 103, 103, 103, 106,
- 141, 106, 106, 476, 2905, 476, 141, 165, 165, 165,
- 165, 165, 165, 165, 165, 165, 165, 141, 2904, 141,
+ 141, 106, 106, 476, 2908, 476, 141, 165, 165, 165,
+ 165, 165, 165, 165, 165, 165, 165, 141, 2907, 141,
106, 107, 107, 107, 107, 107, 107, 107, 107, 107,
107, 107, 107, 107, 107, 107, 107, 107, 107, 107,
107, 107, 107, 107, 107, 107, 107, 107, 107, 107,
@@ -5152,18 +5152,18 @@ static yyconst flex_int16_t yy_chk[27006] =
147, 147, 150, 147, 130, 334, 150, 512, 508, 150,
150, 461, 461, 534, 461, 150, 150, 461, 534, 150,
150, 150, 151, 151, 317, 150, 151, 506, 151, 508,
- 151, 506, 317, 525, 317, 151, 525, 151, 2902, 151,
+ 151, 506, 317, 525, 317, 151, 525, 151, 2905, 151,
183, 183, 183, 183, 183, 183, 183, 183, 183, 183,
- 332, 545, 2901, 545, 130, 130, 130, 130, 130, 130,
+ 332, 545, 2904, 545, 130, 130, 130, 130, 130, 130,
130, 130, 130, 130, 131, 131, 131, 131, 131, 131,
- 334, 131, 2898, 131, 131, 131, 131, 131, 131, 131,
+ 334, 131, 2901, 131, 131, 131, 131, 131, 131, 131,
131, 131, 131, 131, 131, 131, 131, 131, 131, 131,
131, 131, 131, 131, 131, 131, 131, 131, 131, 131,
131, 131, 131, 131, 131, 131, 131, 131, 131, 131,
131, 131, 131, 131, 131, 131, 131, 184, 184, 184,
184, 184, 184, 184, 184, 184, 184, 459, 481, 197,
- 459, 2896, 481, 459, 459, 459, 131, 131, 131, 131,
+ 459, 2899, 481, 459, 459, 459, 131, 131, 131, 131,
131, 131, 131, 131, 131, 131, 132, 527, 478, 481,
132, 188, 188, 188, 188, 188, 188, 188, 188, 188,
@@ -5172,7 +5172,7 @@ static yyconst flex_int16_t yy_chk[27006] =
197, 197, 197, 197, 197, 197, 197, 197, 202, 202,
202, 202, 202, 202, 202, 202, 202, 202, 203, 203,
203, 203, 203, 203, 203, 203, 203, 203, 526, 530,
- 542, 586, 535, 530, 526, 542, 2893, 132, 132, 132,
+ 542, 586, 535, 530, 526, 542, 2896, 132, 132, 132,
132, 132, 132, 132, 132, 132, 132, 156, 535, 884,
884, 156, 586, 156, 207, 207, 207, 207, 207, 207,
207, 207, 207, 207, 208, 208, 208, 208, 208, 208,
@@ -5260,17 +5260,17 @@ static yyconst flex_int16_t yy_chk[27006] =
305, 305, 305, 305, 305, 305, 306, 306, 306, 306,
306, 306, 306, 306, 306, 306, 310, 310, 310, 310,
310, 310, 310, 310, 310, 310, 311, 311, 311, 311,
- 311, 311, 311, 311, 311, 311, 549, 939, 2890, 1091,
+ 311, 311, 311, 311, 311, 311, 549, 939, 2893, 1091,
549, 1091, 587, 594, 214, 214, 214, 214, 214, 214,
214, 214, 214, 214, 214, 214, 214, 214, 214, 214,
- 214, 214, 214, 587, 594, 927, 929, 745, 318, 2888,
+ 214, 214, 214, 587, 594, 927, 929, 745, 318, 2891,
939, 927, 929, 214, 214, 219, 318, 549, 318, 324,
324, 324, 324, 324, 324, 324, 324, 324, 324, 325,
325, 325, 325, 325, 325, 325, 325, 325, 325, 326,
326, 326, 326, 326, 326, 326, 326, 326, 326, 329,
329, 329, 329, 329, 329, 329, 329, 329, 329, 745,
- 745, 1094, 2886, 1094, 219, 219, 219, 219, 219, 219,
+ 745, 1094, 2889, 1094, 219, 219, 219, 219, 219, 219,
219, 219, 219, 219, 219, 219, 219, 219, 219, 219,
219, 219, 219, 340, 340, 340, 340, 340, 340, 340,
340, 340, 340, 219, 219, 224, 341, 341, 341, 341,
@@ -5323,11 +5323,11 @@ static yyconst flex_int16_t yy_chk[27006] =
439, 439, 439, 439, 439, 439, 440, 440, 440, 440,
440, 440, 440, 440, 440, 440, 729, 728, 1170, 1192,
- 2885, 1192, 1170, 744, 249, 249, 249, 249, 249, 249,
+ 2888, 1192, 1170, 744, 249, 249, 249, 249, 249, 249,
249, 249, 249, 249, 249, 249, 249, 249, 249, 249,
249, 249, 249, 444, 444, 444, 444, 444, 444, 444,
444, 444, 444, 249, 249, 254, 445, 445, 445, 445,
- 445, 445, 445, 445, 445, 445, 463, 467, 473, 2881,
+ 445, 445, 445, 445, 445, 445, 463, 467, 473, 2884,
465, 467, 473, 903, 463, 467, 465, 471, 729, 463,
467, 473, 903, 497, 518, 728, 463, 465, 518, 465,
471, 497, 744, 471, 471, 492, 471, 497, 1158, 492,
@@ -5335,19 +5335,19 @@ static yyconst flex_int16_t yy_chk[27006] =
731, 497, 497, 1158, 254, 254, 254, 254, 254, 254,
254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
254, 254, 254, 474, 516, 492, 518, 474, 516, 876,
- 474, 474, 2880, 254, 254, 264, 474, 474, 492, 876,
- 474, 474, 474, 475, 475, 713, 474, 475, 2877, 475,
+ 474, 474, 2883, 254, 254, 264, 474, 474, 492, 876,
+ 474, 474, 474, 475, 475, 713, 474, 475, 2880, 475,
561, 475, 928, 713, 561, 713, 475, 516, 475, 872,
475, 928, 516, 528, 714, 731, 533, 528, 516, 561,
533, 872, 714, 866, 714, 540, 1160, 866, 521, 540,
- 2875, 895, 521, 1160, 264, 264, 264, 264, 264, 264,
+ 2878, 895, 521, 1160, 264, 264, 264, 264, 264, 264,
264, 264, 264, 264, 264, 264, 264, 264, 264, 264,
264, 264, 264, 521, 528, 584, 967, 528, 533, 521,
866, 866, 540, 264, 264, 269, 540, 895, 967, 540,
- 1218, 2874, 1218, 521, 570, 570, 570, 570, 570, 570,
+ 1218, 2877, 1218, 521, 570, 570, 570, 570, 570, 570,
570, 570, 570, 570, 576, 576, 576, 576, 576, 576,
- 576, 576, 576, 576, 897, 899, 1187, 588, 1228, 2870,
+ 576, 576, 576, 576, 897, 899, 1187, 588, 1228, 2873,
1228, 1187, 897, 899, 584, 584, 584, 584, 584, 584,
584, 584, 584, 584, 269, 269, 269, 269, 269, 269,
269, 269, 269, 269, 269, 269, 269, 269, 269, 269,
@@ -5357,7 +5357,7 @@ static yyconst flex_int16_t yy_chk[27006] =
588, 588, 588, 588, 588, 588, 602, 602, 602, 602,
602, 602, 602, 602, 602, 602, 608, 608, 608, 608,
608, 608, 608, 608, 608, 608, 614, 614, 614, 614,
- 614, 614, 614, 614, 614, 614, 1126, 1161, 1237, 2869,
+ 614, 614, 614, 614, 614, 614, 1126, 1161, 1237, 2872,
1237, 1161, 1126, 1161, 274, 274, 274, 274, 274, 274,
274, 274, 274, 274, 274, 274, 274, 274, 274, 274,
274, 274, 274, 620, 620, 620, 620, 620, 620, 620,
@@ -5383,29 +5383,29 @@ static yyconst flex_int16_t yy_chk[27006] =
689, 689, 689, 689, 689, 689, 695, 695, 695, 695,
695, 695, 695, 695, 695, 695, 701, 701, 701, 701,
701, 701, 701, 701, 701, 701, 707, 707, 707, 707,
- 707, 707, 707, 707, 707, 707, 718, 1225, 1357, 1905,
- 2866, 1905, 1357, 1225, 302, 302, 302, 302, 302, 302,
+ 707, 707, 707, 707, 707, 707, 718, 1225, 1357, 1907,
+ 2869, 1907, 1357, 1225, 302, 302, 302, 302, 302, 302,
302, 302, 302, 302, 302, 302, 302, 302, 302, 302,
302, 302, 302, 726, 726, 726, 726, 726, 726, 726,
726, 726, 726, 302, 302, 307, 753, 753, 753, 753,
753, 753, 753, 753, 753, 753, 719, 901, 977, 948,
- 902, 719, 879, 1002, 2147, 2147, 879, 901, 902, 948,
+ 902, 719, 879, 1002, 2149, 2149, 879, 901, 902, 948,
1083, 718, 718, 718, 718, 718, 718, 718, 718, 718,
- 718, 720, 937, 879, 1002, 1360, 720, 1083, 2864, 1360,
+ 718, 720, 937, 879, 1002, 1360, 720, 1083, 2867, 1360,
937, 977, 739, 738, 307, 307, 307, 307, 307, 307,
307, 307, 307, 307, 307, 307, 307, 307, 307, 307,
307, 307, 307, 738, 738, 738, 738, 738, 738, 738,
738, 738, 738, 307, 307, 312, 719, 719, 719, 719,
719, 719, 719, 719, 719, 719, 721, 943, 983, 940,
- 951, 721, 739, 940, 2862, 943, 740, 983, 940, 1186,
+ 951, 721, 739, 940, 2865, 943, 740, 983, 940, 1186,
951, 720, 720, 720, 720, 720, 720, 720, 720, 720,
720, 1186, 739, 739, 739, 739, 739, 739, 739, 739,
- 739, 739, 749, 2858, 312, 312, 312, 312, 312, 312,
+ 739, 739, 749, 2861, 312, 312, 312, 312, 312, 312,
312, 312, 312, 312, 312, 312, 312, 312, 312, 312,
312, 312, 312, 869, 878, 880, 740, 869, 878, 880,
- 2773, 2773, 2854, 312, 312, 337, 721, 721, 721, 721,
+ 2776, 2776, 2857, 312, 312, 337, 721, 721, 721, 721,
721, 721, 721, 721, 721, 721, 740, 740, 740, 740,
740, 740, 740, 740, 740, 740, 741, 748, 880, 1098,
@@ -5415,9 +5415,9 @@ static yyconst flex_int16_t yy_chk[27006] =
337, 337, 337, 337, 337, 337, 337, 337, 337, 337,
342, 893, 1093, 893, 1505, 893, 741, 893, 748, 907,
986, 748, 1093, 907, 893, 748, 1159, 1090, 1159, 986,
- 748, 1090, 1505, 2853, 748, 1159, 741, 741, 741, 741,
+ 748, 1090, 1505, 2856, 748, 1159, 741, 741, 741, 741,
741, 741, 741, 741, 741, 741, 759, 759, 759, 759,
- 759, 759, 759, 759, 759, 759, 907, 2844, 1090, 342,
+ 759, 759, 759, 759, 759, 759, 907, 2847, 1090, 342,
342, 342, 342, 342, 342, 342, 342, 342, 342, 342,
342, 342, 342, 342, 342, 342, 342, 342, 765, 765,
@@ -5426,7 +5426,7 @@ static yyconst flex_int16_t yy_chk[27006] =
771, 779, 779, 779, 779, 779, 779, 779, 779, 779,
779, 787, 787, 787, 787, 787, 787, 787, 787, 787,
787, 795, 795, 795, 795, 795, 795, 795, 795, 795,
- 795, 1184, 1252, 2775, 2775, 2777, 2777, 1252, 1184, 347,
+ 795, 1184, 1252, 2778, 2778, 2780, 2780, 1252, 1184, 347,
347, 347, 347, 347, 347, 347, 347, 347, 347, 347,
347, 347, 347, 347, 347, 347, 347, 347, 801, 801,
801, 801, 801, 801, 801, 801, 801, 801, 347, 347,
@@ -5435,7 +5435,7 @@ static yyconst flex_int16_t yy_chk[27006] =
807, 817, 817, 817, 817, 817, 817, 817, 817, 817,
817, 823, 823, 823, 823, 823, 823, 823, 823, 823,
823, 829, 829, 829, 829, 829, 829, 829, 829, 829,
- 829, 1227, 1268, 2842, 2841, 2839, 2831, 1227, 1268, 352,
+ 829, 1227, 1268, 2845, 2844, 2842, 2834, 1227, 1268, 352,
352, 352, 352, 352, 352, 352, 352, 352, 352, 352,
352, 352, 352, 352, 352, 352, 352, 352, 838, 838,
838, 838, 838, 838, 838, 838, 838, 838, 352, 352,
@@ -5443,150 +5443,150 @@ static yyconst flex_int16_t yy_chk[27006] =
844, 850, 850, 850, 850, 850, 850, 850, 850, 850,
850, 856, 856, 856, 856, 856, 856, 856, 856, 856,
- 856, 1095, 1096, 2823, 1110, 2822, 1118, 1119, 1096, 892,
+ 856, 1095, 1096, 2826, 1110, 2825, 1118, 1119, 1096, 892,
1165, 1095, 1096, 892, 1110, 1096, 1118, 1119, 1165, 357,
357, 357, 357, 357, 357, 357, 357, 357, 357, 357,
357, 357, 357, 357, 357, 357, 357, 357, 868, 870,
881, 882, 868, 870, 881, 882, 892, 898, 357, 357,
- 370, 898, 892, 2799, 1155, 906, 2797, 904, 868, 906,
+ 370, 898, 892, 2802, 1155, 906, 2800, 904, 868, 906,
870, 904, 925, 882, 954, 1155, 925, 1107, 954, 881,
- 1085, 1107, 1168, 2796, 1085, 868, 898, 881, 882, 904,
+ 1085, 1107, 1168, 2799, 1085, 868, 898, 881, 882, 904,
870, 1168, 1229, 925, 898, 954, 904, 1189, 906, 912,
- 925, 1085, 906, 912, 904, 1229, 2795, 1189, 1107, 370,
+ 925, 1085, 906, 912, 904, 1229, 2798, 1189, 1107, 370,
370, 370, 370, 370, 370, 370, 370, 370, 370, 370,
370, 370, 370, 370, 370, 370, 370, 370, 908, 909,
- 910, 911, 908, 909, 910, 911, 912, 2794, 370, 370,
+ 910, 911, 908, 909, 910, 911, 912, 2797, 370, 370,
379, 913, 914, 916, 919, 913, 914, 916, 919, 908,
- 921, 2793, 1113, 911, 921, 910, 1113, 914, 909, 1164,
- 1164, 1164, 913, 1173, 1173, 908, 909, 910, 911, 2792,
+ 921, 2796, 1113, 911, 921, 910, 1113, 914, 909, 1164,
+ 1164, 1164, 913, 1173, 1173, 908, 909, 910, 911, 2795,
1220, 1164, 1374, 938, 1173, 1270, 1374, 938, 913, 914,
- 916, 919, 1220, 1113, 926, 2770, 1270, 921, 926, 379,
+ 916, 919, 1220, 1113, 926, 2773, 1270, 921, 926, 379,
379, 379, 379, 379, 379, 379, 379, 379, 379, 379,
379, 379, 379, 379, 379, 379, 379, 379, 917, 924,
- 938, 932, 917, 924, 2769, 932, 926, 938, 379, 379,
+ 938, 932, 917, 924, 2772, 932, 926, 938, 379, 379,
388, 926, 933, 1274, 934, 944, 933, 941, 934, 944,
935, 941, 1191, 1274, 935, 917, 917, 924, 1381, 932,
- 1191, 2768, 1381, 917, 2767, 917, 924, 917, 932, 917,
+ 1191, 2771, 1381, 917, 2770, 917, 924, 917, 932, 917,
917, 933, 917, 934, 917, 935, 947, 941, 944, 933,
- 947, 934, 944, 944, 941, 2766, 2757, 935, 2756, 388,
+ 947, 934, 944, 944, 941, 2769, 2760, 935, 2759, 388,
388, 388, 388, 388, 388, 388, 388, 388, 388, 388,
- 388, 388, 388, 388, 388, 388, 388, 388, 918, 2749,
- 950, 1382, 918, 947, 950, 1382, 2747, 947, 388, 388,
+ 388, 388, 388, 388, 388, 388, 388, 388, 918, 2752,
+ 950, 1382, 918, 947, 950, 1382, 2750, 947, 388, 388,
393, 930, 936, 966, 946, 930, 936, 966, 946, 918,
- 1114, 950, 950, 1382, 1114, 918, 918, 1116, 2745, 2743,
- 2465, 1116, 936, 918, 966, 918, 930, 918, 930, 918,
- 918, 1114, 918, 952, 918, 2465, 930, 952, 930, 936,
- 930, 946, 930, 930, 930, 966, 2742, 946, 1116, 393,
+ 1114, 950, 950, 1382, 1114, 918, 918, 1116, 2748, 2746,
+ 2468, 1116, 936, 918, 966, 918, 930, 918, 930, 918,
+ 918, 1114, 918, 952, 918, 2468, 930, 952, 930, 936,
+ 930, 946, 930, 930, 930, 966, 2745, 946, 1116, 393,
393, 393, 393, 393, 393, 393, 393, 393, 393, 393,
393, 393, 393, 393, 393, 393, 393, 393, 931, 949,
- 952, 1903, 931, 949, 2741, 952, 2739, 1903, 393, 393,
- 398, 945, 2737, 953, 955, 945, 965, 953, 955, 931,
- 965, 949, 1383, 931, 1134, 931, 1383, 2736, 1134, 2719,
+ 952, 1905, 931, 949, 2744, 952, 2742, 1905, 393, 393,
+ 398, 945, 2740, 953, 955, 945, 965, 953, 955, 931,
+ 965, 949, 1383, 931, 1134, 931, 1383, 2739, 1134, 2722,
- 2710, 1204, 945, 931, 975, 1204, 949, 931, 975, 931,
+ 2713, 1204, 945, 931, 975, 1204, 949, 931, 975, 931,
931, 931, 968, 972, 945, 1134, 968, 972, 945, 945,
- 953, 955, 1204, 965, 975, 975, 2709, 955, 965, 398,
+ 953, 955, 1204, 965, 975, 975, 2712, 955, 965, 398,
398, 398, 398, 398, 398, 398, 398, 398, 398, 398,
398, 398, 398, 398, 398, 398, 398, 398, 971, 968,
- 972, 972, 971, 979, 2708, 2707, 2706, 979, 398, 398,
- 411, 987, 988, 989, 2704, 987, 988, 989, 991, 993,
- 1387, 2699, 991, 993, 1387, 979, 1213, 971, 2695, 1217,
- 1213, 1250, 979, 1217, 989, 971, 1024, 988, 989, 2694,
- 979, 990, 2691, 1250, 2689, 990, 1213, 1213, 987, 988,
+ 972, 972, 971, 979, 2711, 2710, 2709, 979, 398, 398,
+ 411, 987, 988, 989, 2707, 987, 988, 989, 991, 993,
+ 1387, 2702, 991, 993, 1387, 979, 1213, 971, 2698, 1217,
+ 1213, 1250, 979, 1217, 989, 971, 1024, 988, 989, 2697,
+ 979, 990, 2694, 1250, 2692, 990, 1213, 1213, 987, 988,
1217, 988, 989, 1025, 1086, 991, 993, 993, 1086, 411,
411, 411, 411, 411, 411, 411, 411, 411, 411, 411,
411, 411, 411, 411, 411, 411, 411, 411, 990, 992,
- 2685, 2683, 1388, 992, 990, 1086, 1388, 1273, 411, 411,
- 416, 1369, 1101, 1086, 1121, 1369, 1101, 2681, 1121, 1273,
+ 2688, 2686, 1388, 992, 990, 1086, 1388, 1273, 411, 411,
+ 416, 1369, 1101, 1086, 1121, 1369, 1101, 2684, 1121, 1273,
992, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,
- 1024, 2679, 1026, 1101, 2678, 1121, 992, 1026, 1025, 1025,
+ 1024, 2682, 1026, 1101, 2681, 1121, 992, 1026, 1025, 1025,
1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1027, 1101,
- 2677, 1121, 1401, 1027, 1369, 2676, 1401, 2675, 1043, 416,
+ 2680, 1121, 1401, 1027, 1369, 2679, 1401, 2678, 1043, 416,
416, 416, 416, 416, 416, 416, 416, 416, 416, 416,
416, 416, 416, 416, 416, 416, 416, 416, 1043, 1043,
1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 416, 416,
421, 1058, 1058, 1058, 1058, 1058, 1058, 1058, 1058, 1058,
- 1058, 2674, 1026, 1026, 1026, 1026, 1026, 1026, 1026, 1026,
- 1026, 1026, 1028, 2673, 2672, 2654, 2648, 1028, 1027, 1027,
- 1027, 1027, 1027, 1027, 1027, 1027, 1027, 1027, 1029, 2647,
- 1409, 1410, 1392, 1029, 1409, 1410, 1392, 2646, 1044, 421,
+ 1058, 2677, 1026, 1026, 1026, 1026, 1026, 1026, 1026, 1026,
+ 1026, 1026, 1028, 2676, 2675, 2657, 2651, 1028, 1027, 1027,
+ 1027, 1027, 1027, 1027, 1027, 1027, 1027, 1027, 1029, 2650,
+ 1409, 1410, 1392, 1029, 1409, 1410, 1392, 2649, 1044, 421,
421, 421, 421, 421, 421, 421, 421, 421, 421, 421,
421, 421, 421, 421, 421, 421, 421, 421, 1044, 1044,
1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 421, 421,
- 431, 2645, 1368, 1089, 1392, 1157, 1368, 1089, 2644, 1157,
- 2643, 2641, 1028, 1028, 1028, 1028, 1028, 1028, 1028, 1028,
+ 431, 2648, 1368, 1089, 1392, 1157, 1368, 1089, 2647, 1157,
+ 2646, 2644, 1028, 1028, 1028, 1028, 1028, 1028, 1028, 1028,
1028, 1028, 1031, 1089, 1089, 1089, 1157, 1031, 1029, 1029,
- 1029, 1029, 1029, 1029, 1029, 1029, 1029, 1029, 1032, 2637,
+ 1029, 1029, 1029, 1029, 1029, 1029, 1029, 1029, 1032, 2640,
1368, 1102, 1127, 1032, 1097, 1102, 1127, 1157, 1097, 431,
431, 431, 431, 431, 431, 431, 431, 431, 431, 431,
- 431, 431, 431, 431, 431, 431, 431, 431, 2635, 1097,
- 1127, 2633, 1102, 1127, 2631, 1097, 2630, 1418, 431, 431,
+ 431, 431, 431, 431, 431, 431, 431, 431, 2638, 1097,
+ 1127, 2636, 1102, 1127, 2634, 1097, 2633, 1418, 431, 431,
436, 1418, 1102, 1104, 1395, 1141, 1389, 1104, 1395, 1141,
- 1389, 2627, 1031, 1031, 1031, 1031, 1031, 1031, 1031, 1031,
+ 1389, 2630, 1031, 1031, 1031, 1031, 1031, 1031, 1031, 1031,
- 1031, 1031, 1033, 2625, 1104, 1104, 1141, 1033, 1032, 1032,
+ 1031, 1031, 1033, 2628, 1104, 1104, 1141, 1033, 1032, 1032,
1032, 1032, 1032, 1032, 1032, 1032, 1032, 1032, 1034, 1389,
- 1099, 1122, 1141, 1034, 1099, 1122, 2621, 1395, 2614, 436,
+ 1099, 1122, 1141, 1034, 1099, 1122, 2624, 1395, 2617, 436,
436, 436, 436, 436, 436, 436, 436, 436, 436, 436,
- 436, 436, 436, 436, 436, 436, 436, 436, 2612, 1129,
- 1422, 1099, 1122, 1129, 1422, 2611, 2610, 1099, 436, 436,
- 441, 1427, 1122, 2609, 1150, 1427, 1129, 1129, 1150, 2608,
+ 436, 436, 436, 436, 436, 436, 436, 436, 2615, 1129,
+ 1422, 1099, 1122, 1129, 1422, 2614, 2613, 1099, 436, 436,
+ 441, 1427, 1122, 2612, 1150, 1427, 1129, 1129, 1150, 2611,
1129, 1129, 1033, 1033, 1033, 1033, 1033, 1033, 1033, 1033,
- 1033, 1033, 1036, 2607, 2606, 1150, 1150, 1036, 1034, 1034,
- 1034, 1034, 1034, 1034, 1034, 1034, 1034, 1034, 1037, 2605,
+ 1033, 1033, 1036, 2610, 2609, 1150, 1150, 1036, 1034, 1034,
+ 1034, 1034, 1034, 1034, 1034, 1034, 1034, 1034, 1037, 2608,
- 1100, 1130, 1145, 1037, 1100, 1130, 1145, 2588, 2584, 441,
+ 1100, 1130, 1145, 1037, 1100, 1130, 1145, 2591, 2587, 441,
441, 441, 441, 441, 441, 441, 441, 441, 441, 441,
- 441, 441, 441, 441, 441, 441, 441, 441, 2577, 2576,
- 1147, 1100, 1130, 1145, 1147, 2575, 2574, 1100, 441, 441,
- 446, 1219, 1179, 1210, 1207, 1219, 1179, 1210, 1207, 2573,
- 2572, 1147, 1036, 1036, 1036, 1036, 1036, 1036, 1036, 1036,
- 1036, 1036, 1219, 1179, 1210, 1207, 2566, 1147, 1037, 1037,
+ 441, 441, 441, 441, 441, 441, 441, 441, 2580, 2579,
+ 1147, 1100, 1130, 1145, 1147, 2578, 2577, 1100, 441, 441,
+ 446, 1219, 1179, 1210, 1207, 1219, 1179, 1210, 1207, 2576,
+ 2575, 1147, 1036, 1036, 1036, 1036, 1036, 1036, 1036, 1036,
+ 1036, 1036, 1219, 1179, 1210, 1207, 2569, 1147, 1037, 1037,
1037, 1037, 1037, 1037, 1037, 1037, 1037, 1037, 1038, 1179,
- 1120, 1207, 1396, 1038, 1120, 1210, 1396, 2562, 1045, 446,
+ 1120, 1207, 1396, 1038, 1120, 1210, 1396, 2565, 1045, 446,
446, 446, 446, 446, 446, 446, 446, 446, 446, 446,
446, 446, 446, 446, 446, 446, 446, 446, 1125, 1128,
- 1376, 1120, 1125, 1128, 1376, 2560, 2558, 1120, 446, 446,
- 450, 1396, 2557, 2556, 450, 1385, 1128, 1128, 1125, 1385,
- 1376, 1128, 1376, 1400, 2555, 2552, 1046, 1400, 1045, 1125,
- 1128, 450, 1140, 1146, 2550, 2537, 1140, 1146, 1038, 1038,
+ 1376, 1120, 1125, 1128, 1376, 2563, 2561, 1120, 446, 446,
+ 450, 1396, 2560, 2559, 450, 1385, 1128, 1128, 1125, 1385,
+ 1376, 1128, 1376, 1400, 2558, 2555, 1046, 1400, 1045, 1125,
+ 1128, 450, 1140, 1146, 2553, 2540, 1140, 1146, 1038, 1038,
1038, 1038, 1038, 1038, 1038, 1038, 1038, 1038, 1045, 1045,
1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1047, 1400,
1385, 1183, 1202, 1140, 1146, 1183, 1202, 1430, 1048, 1140,
- 1146, 1430, 2532, 1223, 1393, 1154, 1046, 1223, 1393, 1154,
- 2530, 450, 450, 450, 450, 450, 450, 450, 450, 450,
+ 1146, 1430, 2535, 1223, 1393, 1154, 1046, 1223, 1393, 1154,
+ 2533, 450, 450, 450, 450, 450, 450, 450, 450, 450,
450, 451, 1183, 1202, 1223, 451, 1046, 1046, 1046, 1046,
- 1046, 1046, 1046, 1046, 1046, 1046, 1154, 2529, 1047, 1050,
- 1223, 1393, 451, 2528, 1154, 1429, 2527, 1148, 1048, 1429,
- 1178, 1148, 1203, 1051, 1178, 2526, 1203, 2525, 1047, 1047,
+ 1046, 1046, 1046, 1046, 1046, 1046, 1154, 2532, 1047, 1050,
+ 1223, 1393, 451, 2531, 1154, 1429, 2530, 1148, 1048, 1429,
+ 1178, 1148, 1203, 1051, 1178, 2529, 1203, 2528, 1047, 1047,
1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1048, 1048,
- 1048, 1048, 1048, 1048, 1048, 1048, 1048, 1048, 1148, 2524,
- 2523, 1178, 1429, 1203, 1235, 2499, 2496, 1178, 1235, 1050,
- 1148, 2486, 451, 451, 451, 451, 451, 451, 451, 451,
- 451, 451, 454, 1051, 2485, 1235, 454, 2476, 454, 1050,
- 1050, 1050, 1050, 1050, 1050, 1050, 1050, 1050, 1050, 2473,
-
- 2466, 2464, 1052, 1051, 1051, 1051, 1051, 1051, 1051, 1051,
- 1051, 1051, 1051, 2463, 2460, 1431, 1443, 1442, 1434, 1431,
+ 1048, 1048, 1048, 1048, 1048, 1048, 1048, 1048, 1148, 2527,
+ 2526, 1178, 1429, 1203, 1235, 2502, 2499, 1178, 1235, 1050,
+ 1148, 2489, 451, 451, 451, 451, 451, 451, 451, 451,
+ 451, 451, 454, 1051, 2488, 1235, 454, 2479, 454, 1050,
+ 1050, 1050, 1050, 1050, 1050, 1050, 1050, 1050, 1050, 2476,
+
+ 2469, 2467, 1052, 1051, 1051, 1051, 1051, 1051, 1051, 1051,
+ 1051, 1051, 1051, 2466, 2463, 1431, 1443, 1442, 1434, 1431,
1443, 1442, 1434, 1156, 1176, 1208, 1447, 1156, 1176, 1208,
- 1447, 2453, 2450, 454, 454, 454, 454, 454, 454, 454,
+ 1447, 2456, 2453, 454, 454, 454, 454, 454, 454, 454,
454, 454, 454, 454, 454, 454, 454, 454, 454, 454,
454, 454, 1052, 1180, 1156, 1176, 1208, 1180, 1431, 1434,
- 1442, 2448, 454, 454, 457, 1156, 1208, 1176, 457, 2435,
- 2434, 2433, 1052, 1052, 1052, 1052, 1052, 1052, 1052, 1052,
- 1052, 1052, 1180, 1391, 1180, 2431, 2430, 1391, 2429, 457,
- 457, 457, 457, 457, 457, 2428, 457, 457, 2427, 457,
+ 1442, 2451, 454, 454, 457, 1156, 1208, 1176, 457, 2438,
+ 2437, 2436, 1052, 1052, 1052, 1052, 1052, 1052, 1052, 1052,
+ 1052, 1052, 1180, 1391, 1180, 2434, 2433, 1391, 2432, 457,
+ 457, 457, 457, 457, 457, 2431, 457, 457, 2430, 457,
457, 457, 457, 457, 457, 457, 457, 457, 457, 457,
- 457, 2403, 1391, 457, 543, 543, 543, 543, 543, 543,
+ 457, 2406, 1391, 457, 543, 543, 543, 543, 543, 543,
543, 543, 543, 543, 543, 543, 543, 543, 543, 543,
543, 543, 543, 543, 543, 543, 543, 543, 543, 543,
543, 543, 543, 543, 543, 543, 543, 543, 543, 543,
@@ -5605,11 +5605,11 @@ static yyconst flex_int16_t yy_chk[27006] =
560, 560, 560, 560, 560, 560, 560, 560, 560, 560,
560, 560, 560, 560, 560, 560, 560, 560, 560, 560,
560, 560, 565, 1177, 1185, 1206, 1212, 1177, 1185, 1206,
- 1212, 1239, 1451, 2402, 2397, 1239, 1451, 1256, 1262, 2395,
+ 1212, 1239, 1451, 2405, 2400, 1239, 1451, 1256, 1262, 2398,
- 2383, 1256, 1262, 1177, 1185, 1454, 1212, 1455, 2382, 1454,
- 2381, 1455, 1239, 1239, 1177, 1185, 1206, 1212, 1256, 1262,
- 2378, 1177, 1206, 565, 565, 565, 565, 565, 565, 565,
+ 2386, 1256, 1262, 1177, 1185, 1454, 1212, 1455, 2385, 1454,
+ 2384, 1455, 1239, 1239, 1177, 1185, 1206, 1212, 1256, 1262,
+ 2380, 1177, 1206, 565, 565, 565, 565, 565, 565, 565,
565, 565, 565, 565, 565, 565, 565, 565, 565, 565,
565, 565, 566, 566, 566, 566, 566, 566, 566, 566,
566, 566, 566, 566, 566, 566, 566, 566, 566, 566,
@@ -5619,18 +5619,18 @@ static yyconst flex_int16_t yy_chk[27006] =
569, 569, 569, 569, 569, 569, 569, 569, 569, 571,
1131, 1200, 1214, 1209, 1131, 1200, 1214, 1209, 1216, 1445,
- 2377, 2368, 1216, 1445, 1367, 1221, 1272, 1053, 1367, 1221,
- 1272, 1472, 1131, 1473, 2367, 1472, 1200, 1473, 1200, 1445,
+ 2379, 2370, 1216, 1445, 1367, 1221, 1272, 1053, 1367, 1221,
+ 1272, 1472, 1131, 1473, 2369, 1472, 1200, 1473, 1200, 1445,
1055, 1131, 1200, 1214, 1209, 1367, 1200, 1272, 1272, 1216,
1200, 1131, 1200, 1200, 1200, 1209, 1221, 1056, 571, 571,
571, 571, 571, 571, 571, 571, 571, 571, 571, 571,
571, 571, 571, 571, 571, 571, 571, 1053, 1205, 1457,
- 1490, 1390, 1205, 1457, 1490, 1390, 2364, 571, 571, 577,
- 1055, 1224, 1215, 2357, 1457, 1224, 1215, 1053, 1053, 1053,
+ 1490, 1390, 1205, 1457, 1490, 1390, 2366, 571, 571, 577,
+ 1055, 1224, 1215, 2359, 1457, 1224, 1215, 1053, 1053, 1053,
1053, 1053, 1053, 1053, 1053, 1053, 1053, 1056, 1390, 1205,
1055, 1055, 1055, 1055, 1055, 1055, 1055, 1055, 1055, 1055,
- 1215, 1224, 1224, 1215, 2356, 1205, 1490, 1056, 1056, 1056,
+ 1215, 1224, 1224, 1215, 2358, 1205, 1490, 1056, 1056, 1056,
1056, 1056, 1056, 1056, 1056, 1056, 1056, 1319, 577, 577,
577, 577, 577, 577, 577, 577, 577, 577, 577, 577,
577, 577, 577, 577, 577, 577, 577, 1319, 1319, 1319,
@@ -5645,355 +5645,355 @@ static yyconst flex_int16_t yy_chk[27006] =
582, 582, 582, 582, 582, 582, 582, 582, 582, 582,
582, 582, 582, 582, 582, 582, 582, 582, 582, 582,
582, 582, 582, 585, 1201, 1211, 1233, 1222, 1201, 1211,
- 1233, 1222, 1259, 1432, 1384, 2354, 1259, 1432, 1384, 1234,
- 1238, 2353, 585, 1234, 1238, 1201, 1509, 2350, 1512, 1201,
+ 1233, 1222, 1259, 1432, 1384, 2356, 1259, 1432, 1384, 1234,
+ 1238, 2355, 585, 1234, 1238, 1201, 1509, 2352, 1512, 1201,
1509, 1201, 1512, 1259, 1057, 1384, 1211, 1233, 1222, 1201,
- 1432, 2338, 1238, 1201, 1222, 1201, 1201, 1201, 1211, 1259,
+ 1432, 2340, 1238, 1201, 1222, 1201, 1201, 1201, 1211, 1259,
1234, 1238, 585, 585, 585, 585, 585, 585, 585, 585,
585, 585, 585, 585, 585, 585, 585, 585, 585, 585,
585, 585, 585, 585, 585, 585, 585, 585, 585, 585,
585, 585, 585, 589, 1057, 1240, 1245, 1236, 1247, 1240,
1245, 1236, 1247, 1246, 1253, 1248, 1483, 1246, 1253, 1248,
- 1483, 2333, 2331, 2325, 1057, 1057, 1057, 1057, 1057, 1057,
+ 1483, 2335, 2333, 2327, 1057, 1057, 1057, 1057, 1057, 1057,
1057, 1057, 1057, 1057, 1236, 1248, 1240, 1245, 1236, 1247,
- 2322, 2309, 1246, 1243, 1246, 1253, 1248, 1243, 2308, 1245,
- 2307, 1483, 589, 589, 589, 589, 589, 589, 589, 589,
+ 2324, 2311, 1246, 1243, 1246, 1253, 1248, 1243, 2310, 1245,
+ 2309, 1483, 589, 589, 589, 589, 589, 589, 589, 589,
589, 589, 589, 589, 589, 589, 589, 589, 589, 589,
- 589, 1243, 1244, 1255, 1243, 1257, 1244, 1255, 2306, 1257,
+ 589, 1243, 1244, 1255, 1243, 1257, 1244, 1255, 2308, 1257,
- 1258, 589, 589, 597, 1258, 1243, 2305, 1260, 2304, 2303,
+ 1258, 589, 589, 597, 1258, 1243, 2307, 1260, 2306, 2305,
1263, 1260, 1265, 1244, 1263, 1267, 1265, 1269, 1264, 1267,
- 1244, 1269, 1264, 2302, 1255, 1257, 1257, 1263, 1263, 1461,
- 2301, 1258, 1263, 1461, 1244, 1264, 1264, 1258, 1260, 1264,
- 1264, 1263, 2278, 1265, 1260, 1277, 1267, 1460, 1269, 1277,
+ 1244, 1269, 1264, 2304, 1255, 1257, 1257, 1263, 1263, 1461,
+ 2303, 1258, 1263, 1461, 1244, 1264, 1264, 1258, 1260, 1264,
+ 1264, 1263, 2280, 1265, 1260, 1277, 1267, 1460, 1269, 1277,
1282, 1460, 597, 597, 597, 597, 597, 597, 597, 597,
597, 597, 597, 597, 597, 597, 597, 597, 597, 597,
- 597, 1266, 1461, 1280, 1277, 1266, 1277, 1280, 2274, 2262,
- 1460, 597, 597, 603, 2261, 1397, 1283, 1372, 1280, 1397,
- 2258, 1372, 1394, 1266, 1284, 2255, 1394, 1398, 2250, 1284,
+ 597, 1266, 1461, 1280, 1277, 1266, 1277, 1280, 2276, 2264,
+ 1460, 597, 597, 603, 2263, 1397, 1283, 1372, 1280, 1397,
+ 2260, 1372, 1394, 1266, 1284, 2257, 1394, 1398, 2252, 1284,
- 2234, 1398, 1266, 2232, 1280, 1285, 2231, 1372, 1372, 1372,
+ 2236, 1398, 1266, 2234, 1280, 1285, 2233, 1372, 1372, 1372,
1285, 2229, 1266, 1394, 1397, 1282, 1282, 1282, 1282, 1282,
- 1282, 1282, 1282, 1282, 1282, 1286, 1398, 1516, 2227, 1394,
+ 1282, 1282, 1282, 1282, 1282, 1286, 1398, 1516, 2228, 1394,
1286, 1516, 603, 603, 603, 603, 603, 603, 603, 603,
603, 603, 603, 603, 603, 603, 603, 603, 603, 603,
603, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283,
1283, 603, 603, 609, 1284, 1284, 1284, 1284, 1284, 1284,
- 1284, 1284, 1284, 1284, 2226, 1285, 1285, 1285, 1285, 1285,
- 1285, 1285, 1285, 1285, 1285, 1287, 2220, 1380, 1522, 1536,
- 1287, 1380, 1522, 1536, 2212, 1286, 1286, 1286, 1286, 1286,
+ 1284, 1284, 1284, 1284, 2222, 1285, 1285, 1285, 1285, 1285,
+ 1285, 1285, 1285, 1285, 1285, 1287, 2214, 1380, 1522, 1536,
+ 1287, 1380, 1522, 1536, 2213, 1286, 1286, 1286, 1286, 1286,
- 1286, 1286, 1286, 1286, 1286, 1288, 2211, 1380, 1380, 1380,
+ 1286, 1286, 1286, 1286, 1286, 1288, 2210, 1380, 1380, 1380,
1288, 1320, 609, 609, 609, 609, 609, 609, 609, 609,
609, 609, 609, 609, 609, 609, 609, 609, 609, 609,
609, 1320, 1320, 1320, 1320, 1320, 1320, 1320, 1320, 1320,
- 1320, 609, 609, 615, 2208, 1408, 2206, 1412, 1428, 1408,
+ 1320, 609, 609, 615, 2208, 1408, 2205, 1412, 1428, 1408,
1433, 1412, 1428, 2203, 1433, 1287, 1287, 1287, 1287, 1287,
1287, 1287, 1287, 1287, 1287, 1289, 1408, 1408, 1412, 1428,
- 1289, 2201, 2200, 2194, 2189, 1288, 1288, 1288, 1288, 1288,
- 1288, 1288, 1288, 1288, 1288, 1290, 1433, 1437, 2184, 2181,
+ 1289, 2202, 2196, 2191, 2186, 1288, 1288, 1288, 1288, 1288,
+ 1288, 1288, 1288, 1288, 1288, 1290, 1433, 1437, 2183, 2181,
1290, 1437, 615, 615, 615, 615, 615, 615, 615, 615,
615, 615, 615, 615, 615, 615, 615, 615, 615, 615,
- 615, 1420, 1439, 1446, 1543, 1420, 1439, 1446, 1543, 2179,
- 1438, 615, 615, 621, 1438, 2168, 1448, 2166, 2164, 1437,
+ 615, 1420, 1439, 1446, 1543, 1420, 1439, 1446, 1543, 2170,
+ 1438, 615, 615, 621, 1438, 2168, 1448, 2166, 2162, 1437,
1448, 1420, 1420, 1446, 1446, 1289, 1289, 1289, 1289, 1289,
- 1289, 1289, 1289, 1289, 1289, 1291, 2160, 1544, 1456, 1438,
+ 1289, 1289, 1289, 1289, 1289, 1291, 2161, 1544, 1456, 1438,
1291, 1544, 1456, 1448, 1439, 1290, 1290, 1290, 1290, 1290,
- 1290, 1290, 1290, 1290, 1290, 1292, 2159, 1440, 1448, 1456,
+ 1290, 1290, 1290, 1290, 1290, 1292, 2156, 1440, 1448, 1456,
1292, 1440, 621, 621, 621, 621, 621, 621, 621, 621,
621, 621, 621, 621, 621, 621, 621, 621, 621, 621,
- 621, 1444, 2154, 1475, 1452, 1444, 1440, 1475, 1452, 2144,
-
- 1458, 621, 621, 627, 1458, 1449, 1450, 2136, 2133, 1449,
- 1450, 1452, 1452, 2132, 1475, 1291, 1291, 1291, 1291, 1291,
- 1291, 1291, 1291, 1291, 1291, 1293, 2131, 1450, 2130, 1458,
- 1293, 2129, 1444, 1449, 1450, 1292, 1292, 1292, 1292, 1292,
- 1292, 1292, 1292, 1292, 1292, 1294, 2128, 1449, 1450, 2127,
- 1294, 2126, 627, 627, 627, 627, 627, 627, 627, 627,
+ 621, 1444, 2146, 1475, 1452, 1444, 1440, 1475, 1452, 2138,
+
+ 1458, 621, 621, 627, 1458, 1449, 1450, 2135, 2134, 1449,
+ 1450, 1452, 1452, 2133, 1475, 1291, 1291, 1291, 1291, 1291,
+ 1291, 1291, 1291, 1291, 1291, 1293, 2132, 1450, 2131, 1458,
+ 1293, 2130, 1444, 1449, 1450, 1292, 1292, 1292, 1292, 1292,
+ 1292, 1292, 1292, 1292, 1292, 1294, 2129, 1449, 1450, 2128,
+ 1294, 2127, 627, 627, 627, 627, 627, 627, 627, 627,
627, 627, 627, 627, 627, 627, 627, 627, 627, 627,
- 627, 1462, 1466, 1493, 1495, 1462, 1466, 1493, 1495, 2125,
+ 627, 1462, 1466, 1493, 1495, 1462, 1466, 1493, 1495, 2110,
1463, 627, 627, 633, 1463, 1467, 1468, 1487, 1526, 1467,
- 1468, 1487, 1526, 2108, 1466, 1293, 1293, 1293, 1293, 1293,
+ 1468, 1487, 1526, 2109, 1466, 1293, 1293, 1293, 1293, 1293,
- 1293, 1293, 1293, 1293, 1293, 1295, 2107, 1467, 1468, 1463,
+ 1293, 1293, 1293, 1293, 1293, 1295, 2106, 1467, 1468, 1463,
1295, 1493, 1487, 1495, 1462, 1294, 1294, 1294, 1294, 1294,
- 1294, 1294, 1294, 1294, 1294, 1296, 1526, 1465, 2104, 2093,
+ 1294, 1294, 1294, 1294, 1294, 1296, 1526, 1465, 2095, 2093,
1296, 1465, 633, 633, 633, 633, 633, 633, 633, 633,
633, 633, 633, 633, 633, 633, 633, 633, 633, 633,
- 633, 1469, 2091, 1548, 1510, 1469, 1465, 1548, 1510, 2085,
- 2084, 633, 633, 639, 1470, 1471, 1481, 1484, 1470, 1471,
- 1481, 1484, 2081, 1469, 1510, 1295, 1295, 1295, 1295, 1295,
+ 633, 1469, 2087, 1548, 1510, 1469, 1465, 1548, 1510, 2086,
+ 2083, 633, 633, 639, 1470, 1471, 1481, 1484, 1470, 1471,
+ 1481, 1484, 2077, 1469, 1510, 1295, 1295, 1295, 1295, 1295,
1295, 1295, 1295, 1295, 1295, 1297, 1470, 1471, 1485, 1484,
- 1297, 2075, 1485, 2071, 2052, 1296, 1296, 1296, 1296, 1296,
+ 1297, 2073, 1485, 2054, 2051, 1296, 1296, 1296, 1296, 1296,
- 1296, 1296, 1296, 1296, 1296, 1299, 2049, 1488, 2047, 1481,
+ 1296, 1296, 1296, 1296, 1296, 1299, 2049, 1488, 2048, 1481,
1299, 1488, 639, 639, 639, 639, 639, 639, 639, 639,
639, 639, 639, 639, 639, 639, 639, 639, 639, 639,
- 639, 1485, 1491, 2046, 1520, 1519, 1491, 1488, 1520, 1519,
- 2045, 639, 639, 645, 1489, 1492, 1511, 1524, 1489, 1492,
- 1511, 1524, 2044, 2038, 1520, 1297, 1297, 1297, 1297, 1297,
+ 639, 1485, 1491, 2047, 1520, 1519, 1491, 1488, 1520, 1519,
+ 2046, 639, 639, 645, 1489, 1492, 1511, 1524, 1489, 1492,
+ 1511, 1524, 2040, 2039, 1520, 1297, 1297, 1297, 1297, 1297,
1297, 1297, 1297, 1297, 1297, 1300, 1511, 1511, 1491, 1523,
- 1300, 1519, 1524, 1523, 2037, 1299, 1299, 1299, 1299, 1299,
- 1299, 1299, 1299, 1299, 1299, 1301, 2030, 1489, 1492, 1998,
- 1301, 1997, 645, 645, 645, 645, 645, 645, 645, 645,
+ 1300, 1519, 1524, 1523, 2032, 1299, 1299, 1299, 1299, 1299,
+ 1299, 1299, 1299, 1299, 1299, 1301, 2000, 1489, 1492, 1999,
+ 1301, 1992, 645, 645, 645, 645, 645, 645, 645, 645,
645, 645, 645, 645, 645, 645, 645, 645, 645, 645,
- 645, 1521, 1523, 1545, 1559, 1521, 1990, 1545, 1559, 1985,
- 1494, 645, 645, 651, 1494, 1514, 1539, 1982, 1981, 1514,
- 1539, 1521, 1521, 1974, 1545, 1300, 1300, 1300, 1300, 1300,
- 1300, 1300, 1300, 1300, 1300, 1303, 1968, 1965, 1525, 1540,
+ 645, 1521, 1523, 1545, 1559, 1521, 1987, 1545, 1559, 1984,
+ 1494, 645, 645, 651, 1494, 1514, 1539, 1983, 1976, 1514,
+ 1539, 1521, 1521, 1970, 1545, 1300, 1300, 1300, 1300, 1300,
+ 1300, 1300, 1300, 1300, 1300, 1303, 1967, 1964, 1525, 1540,
1303, 1494, 1525, 1540, 1514, 1301, 1301, 1301, 1301, 1301,
- 1301, 1301, 1301, 1301, 1301, 1304, 1962, 1517, 1539, 1494,
+ 1301, 1301, 1301, 1301, 1301, 1304, 1961, 1517, 1539, 1494,
1304, 1517, 651, 651, 651, 651, 651, 651, 651, 651,
651, 651, 651, 651, 651, 651, 651, 651, 651, 651,
- 651, 1959, 1540, 1956, 1549, 1525, 1527, 1517, 1549, 1953,
+ 651, 1958, 1540, 1955, 1549, 1525, 1527, 1517, 1549, 1953,
1527, 651, 651, 657, 1528, 1537, 1951, 1551, 1528, 1537,
- 1560, 1551, 1949, 1944, 1560, 1303, 1303, 1303, 1303, 1303,
- 1303, 1303, 1303, 1303, 1303, 1305, 1943, 1527, 1551, 1561,
- 1305, 1549, 1942, 1561, 1537, 1304, 1304, 1304, 1304, 1304,
- 1304, 1304, 1304, 1304, 1304, 1306, 1934, 1920, 1914, 1560,
+ 1560, 1551, 1946, 1945, 1560, 1303, 1303, 1303, 1303, 1303,
+ 1303, 1303, 1303, 1303, 1303, 1305, 1944, 1527, 1551, 1561,
+ 1305, 1549, 1936, 1561, 1537, 1304, 1304, 1304, 1304, 1304,
+ 1304, 1304, 1304, 1304, 1304, 1306, 1922, 1916, 1912, 1560,
1306, 1528, 657, 657, 657, 657, 657, 657, 657, 657,
657, 657, 657, 657, 657, 657, 657, 657, 657, 657,
- 657, 1910, 1561, 1568, 1570, 1533, 1904, 1568, 1570, 1533,
+ 657, 1906, 1561, 1568, 1570, 1533, 1904, 1568, 1570, 1533,
1542, 657, 657, 665, 1542, 1546, 1547, 1582, 1562, 1546,
- 1547, 1582, 1562, 1902, 1901, 1305, 1305, 1305, 1305, 1305,
+ 1547, 1582, 1562, 1903, 1902, 1305, 1305, 1305, 1305, 1305,
1305, 1305, 1305, 1305, 1305, 1307, 1533, 1547, 1555, 1542,
- 1307, 1900, 1555, 1546, 1547, 1306, 1306, 1306, 1306, 1306,
- 1306, 1306, 1306, 1306, 1306, 1308, 1899, 1546, 1547, 1555,
+ 1307, 1901, 1555, 1546, 1547, 1306, 1306, 1306, 1306, 1306,
+ 1306, 1306, 1306, 1306, 1306, 1308, 1891, 1546, 1547, 1555,
1308, 1562, 665, 665, 665, 665, 665, 665, 665, 665,
665, 665, 665, 665, 665, 665, 665, 665, 665, 665,
- 665, 1558, 1569, 1555, 1576, 1558, 1569, 1889, 1576, 1888,
+ 665, 1558, 1569, 1555, 1576, 1558, 1569, 1890, 1576, 1884,
1553, 665, 665, 671, 1553, 1556, 1882, 1583, 1580, 1556,
1577, 1583, 1580, 1569, 1577, 1307, 1307, 1307, 1307, 1307,
1307, 1307, 1307, 1307, 1307, 1310, 1558, 1576, 1580, 1553,
- 1310, 1880, 1583, 1873, 1556, 1308, 1308, 1308, 1308, 1308,
+ 1310, 1875, 1583, 1861, 1556, 1308, 1308, 1308, 1308, 1308,
- 1308, 1308, 1308, 1308, 1308, 1311, 1577, 1566, 1859, 1855,
+ 1308, 1308, 1308, 1308, 1308, 1311, 1577, 1566, 1857, 1856,
1311, 1566, 671, 671, 671, 671, 671, 671, 671, 671,
671, 671, 671, 671, 671, 671, 671, 671, 671, 671,
- 671, 1571, 1854, 1581, 1585, 1571, 1566, 1581, 1585, 1849,
+ 671, 1571, 1851, 1581, 1585, 1571, 1566, 1581, 1585, 1848,
1567, 671, 671, 677, 1567, 1572, 1571, 1590, 1594, 1572,
1846, 1590, 1594, 1581, 1581, 1310, 1310, 1310, 1310, 1310,
- 1310, 1310, 1310, 1310, 1310, 1312, 1844, 1573, 1567, 1595,
- 1312, 1573, 1843, 1595, 1572, 1311, 1311, 1311, 1311, 1311,
- 1311, 1311, 1311, 1311, 1311, 1313, 1840, 1588, 1839, 1838,
+ 1310, 1310, 1310, 1310, 1310, 1312, 1845, 1573, 1567, 1595,
+ 1312, 1573, 1842, 1595, 1572, 1311, 1311, 1311, 1311, 1311,
+ 1311, 1311, 1311, 1311, 1311, 1313, 1841, 1588, 1840, 1829,
1313, 1588, 677, 677, 677, 677, 677, 677, 677, 677,
677, 677, 677, 677, 677, 677, 677, 677, 677, 677,
- 677, 1827, 1821, 1587, 1592, 1573, 1588, 1587, 1592, 1819,
- 1586, 677, 677, 683, 1586, 1818, 1596, 1817, 1816, 1598,
- 1596, 1592, 1592, 1598, 1815, 1312, 1312, 1312, 1312, 1312,
+ 677, 1823, 1821, 1587, 1592, 1573, 1588, 1587, 1592, 1820,
+ 1586, 677, 677, 683, 1586, 1819, 1596, 1818, 1817, 1598,
+ 1596, 1592, 1592, 1598, 1816, 1312, 1312, 1312, 1312, 1312,
1312, 1312, 1312, 1312, 1312, 1315, 1587, 1596, 1586, 1601,
- 1315, 1814, 1812, 1601, 1809, 1313, 1313, 1313, 1313, 1313,
- 1313, 1313, 1313, 1313, 1313, 1316, 1598, 1600, 1808, 1601,
+ 1315, 1814, 1811, 1601, 1810, 1313, 1313, 1313, 1313, 1313,
+ 1313, 1313, 1313, 1313, 1313, 1316, 1598, 1600, 1802, 1601,
1316, 1600, 683, 683, 683, 683, 683, 683, 683, 683,
683, 683, 683, 683, 683, 683, 683, 683, 683, 683,
- 683, 1597, 1599, 1604, 1619, 1597, 1599, 1604, 1619, 1800,
+ 683, 1597, 1599, 1604, 1619, 1597, 1599, 1604, 1619, 1780,
- 1778, 683, 683, 690, 1600, 1602, 1597, 1615, 1616, 1602,
+ 1772, 683, 683, 690, 1600, 1602, 1597, 1615, 1616, 1602,
1603, 1615, 1616, 1599, 1603, 1315, 1315, 1315, 1315, 1315,
- 1315, 1315, 1315, 1315, 1315, 1317, 1602, 1602, 1770, 1599,
- 1317, 1764, 1603, 1763, 1750, 1316, 1316, 1316, 1316, 1316,
- 1316, 1316, 1316, 1316, 1316, 1620, 1616, 1691, 1743, 1620,
+ 1315, 1315, 1315, 1315, 1315, 1317, 1602, 1602, 1764, 1599,
+ 1317, 1763, 1603, 1750, 1743, 1316, 1316, 1316, 1316, 1316,
+ 1316, 1316, 1316, 1316, 1316, 1620, 1616, 1691, 1742, 1620,
1615, 1691, 690, 690, 690, 690, 690, 690, 690, 690,
690, 690, 690, 690, 690, 690, 690, 690, 690, 690,
- 690, 1436, 1606, 1622, 1692, 1436, 1606, 1622, 1692, 1742,
+ 690, 1436, 1606, 1622, 1692, 1436, 1606, 1622, 1692, 1723,
1693, 690, 690, 696, 1693, 1720, 1609, 1513, 1620, 1720,
1609, 1513, 1436, 1606, 1606, 1317, 1317, 1317, 1317, 1317,
1317, 1317, 1317, 1317, 1317, 1321, 1436, 1513, 1609, 1613,
- 1721, 1723, 1622, 1613, 1721, 1322, 1755, 1436, 1513, 1767,
- 1755, 1775, 1776, 1767, 1784, 1775, 1776, 1718, 1784, 1711,
+ 1721, 1718, 1622, 1613, 1721, 1322, 1755, 1436, 1513, 1767,
+ 1755, 1777, 1778, 1767, 1786, 1777, 1778, 1711, 1786, 1695,
1613, 1613, 696, 696, 696, 696, 696, 696, 696, 696,
696, 696, 696, 696, 696, 696, 696, 696, 696, 696,
- 696, 1695, 1614, 1790, 1791, 1321, 1614, 1790, 1791, 1621,
- 1822, 696, 696, 702, 1822, 1322, 1623, 1578, 1829, 1831,
- 1618, 1578, 1829, 1831, 1614, 1321, 1321, 1321, 1321, 1321,
+ 696, 1621, 1614, 1792, 1793, 1321, 1614, 1792, 1793, 1618,
+ 1824, 696, 696, 702, 1824, 1322, 1623, 1578, 1831, 1833,
+ 1617, 1578, 1831, 1833, 1614, 1321, 1321, 1321, 1321, 1321,
1321, 1321, 1321, 1321, 1321, 1322, 1322, 1322, 1322, 1322,
- 1322, 1322, 1322, 1322, 1322, 1323, 1578, 1578, 1617, 1841,
+ 1322, 1322, 1322, 1322, 1322, 1323, 1578, 1578, 1591, 1843,
- 1578, 1829, 1578, 1841, 1578, 1324, 1851, 1591, 1589, 1578,
- 1851, 1578, 702, 702, 702, 702, 702, 702, 702, 702,
+ 1578, 1831, 1578, 1843, 1578, 1324, 1853, 1589, 1584, 1578,
+ 1853, 1578, 702, 702, 702, 702, 702, 702, 702, 702,
702, 702, 702, 702, 702, 702, 702, 702, 702, 702,
702, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1623,
- 1623, 702, 702, 708, 1837, 1323, 1906, 1915, 1837, 1930,
- 1906, 1915, 1584, 1930, 1931, 1324, 1842, 1575, 1931, 1933,
- 1842, 1325, 1574, 1933, 1837, 1323, 1323, 1323, 1323, 1323,
+ 1623, 702, 702, 708, 1839, 1323, 1908, 1917, 1839, 1932,
+ 1908, 1917, 1575, 1932, 1933, 1324, 1844, 1574, 1933, 1935,
+ 1844, 1325, 1565, 1935, 1839, 1323, 1323, 1323, 1323, 1323,
1323, 1323, 1323, 1323, 1323, 1324, 1324, 1324, 1324, 1324,
- 1324, 1324, 1324, 1324, 1324, 1842, 1993, 1871, 1565, 1564,
- 1993, 1871, 708, 708, 708, 708, 708, 708, 708, 708,
+ 1324, 1324, 1324, 1324, 1324, 1844, 1995, 1873, 1564, 1563,
+ 1995, 1873, 708, 708, 708, 708, 708, 708, 708, 708,
708, 708, 708, 708, 708, 708, 708, 708, 708, 708,
- 708, 1325, 1871, 2016, 1687, 2017, 2026, 2016, 1687, 2017,
- 2026, 708, 708, 734, 1690, 1722, 1705, 1563, 1690, 1722,
+ 708, 1325, 1873, 2018, 1687, 2019, 2028, 2018, 1687, 2019,
+ 2028, 708, 708, 734, 1690, 1722, 1705, 1557, 1690, 1722,
1705, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325,
- 1325, 1326, 1687, 1557, 2053, 1690, 1722, 1705, 2053, 2061,
- 1687, 1327, 1552, 2061, 1550, 2062, 1687, 1541, 1687, 2062,
+ 1325, 1326, 1687, 1552, 2055, 1690, 1722, 1705, 2055, 2063,
+ 1687, 1327, 1550, 2063, 1541, 2064, 1687, 1538, 1687, 2064,
734, 734, 734, 1705, 734, 734, 734, 734, 734, 734,
734, 734, 734, 734, 734, 734, 734, 734, 734, 734,
734, 734, 734, 734, 734, 734, 734, 734, 754, 1697,
- 2064, 1326, 2062, 1697, 2064, 2070, 2089, 1538, 2098, 2070,
+ 2066, 1326, 2064, 1697, 2066, 2072, 2091, 1535, 2100, 2072,
- 2089, 1327, 2098, 2134, 2148, 2218, 1328, 2134, 2148, 2218,
+ 2091, 1327, 2100, 2136, 2150, 2220, 1328, 2136, 2150, 2220,
1697, 1326, 1326, 1326, 1326, 1326, 1326, 1326, 1326, 1326,
1326, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327,
- 1327, 1697, 1535, 1534, 1532, 1531, 1329, 754, 754, 754,
+ 1327, 1697, 1534, 1532, 1531, 1530, 1329, 754, 754, 754,
754, 754, 754, 754, 754, 754, 754, 754, 754, 754,
- 754, 754, 754, 754, 754, 754, 1328, 2074, 2095, 2099,
- 2235, 2074, 2095, 2099, 2235, 1530, 754, 754, 760, 1653,
- 2241, 1529, 1518, 1515, 2241, 2235, 1328, 1328, 1328, 1328,
- 1328, 1328, 1328, 1328, 1328, 1328, 1329, 1330, 1508, 1653,
+ 754, 754, 754, 754, 754, 754, 1328, 2076, 2097, 2101,
+ 2231, 2076, 2097, 2101, 2231, 1529, 754, 754, 760, 1653,
+ 2243, 1518, 1515, 1508, 2243, 2231, 1328, 1328, 1328, 1328,
+ 1328, 1328, 1328, 1328, 1328, 1328, 1329, 1330, 1507, 1653,
1653, 1653, 1653, 1653, 1653, 1653, 1653, 1653, 1653, 1331,
- 2099, 2254, 1507, 2095, 2074, 2254, 1329, 1329, 1329, 1329,
+ 2101, 2256, 1506, 2097, 2076, 2256, 1329, 1329, 1329, 1329,
1329, 1329, 1329, 1329, 1329, 1329, 1624, 760, 760, 760,
760, 760, 760, 760, 760, 760, 760, 760, 760, 760,
- 760, 760, 760, 760, 760, 760, 2069, 1330, 2265, 2243,
- 2069, 2271, 2265, 2243, 1506, 2271, 760, 760, 766, 1331,
- 2268, 2287, 1504, 1503, 2268, 2287, 2069, 1330, 1330, 1330,
- 1330, 1330, 1330, 1330, 1330, 1330, 1330, 1332, 2243, 1331,
+ 760, 760, 760, 760, 760, 760, 2071, 1330, 2237, 2245,
+ 2071, 2267, 2237, 2245, 1504, 2267, 760, 760, 766, 1331,
+ 2270, 2273, 1503, 2237, 2270, 2273, 2071, 1330, 1330, 1330,
+ 1330, 1330, 1330, 1330, 1330, 1330, 1330, 1332, 2245, 1331,
1331, 1331, 1331, 1331, 1331, 1331, 1331, 1331, 1331, 1333,
1502, 1624, 1624, 1624, 1624, 1624, 1624, 1624, 1624, 1624,
- 1624, 1625, 1501, 1500, 1499, 2268, 1625, 766, 766, 766,
+ 1624, 1625, 1501, 1500, 1499, 2270, 1625, 766, 766, 766,
766, 766, 766, 766, 766, 766, 766, 766, 766, 766,
- 766, 766, 766, 766, 766, 766, 2072, 1332, 2289, 2318,
- 2072, 2326, 2289, 2318, 1498, 2326, 766, 766, 772, 1333,
- 2334, 2362, 1497, 1496, 2334, 2362, 2072, 1332, 1332, 1332,
+ 766, 766, 766, 766, 766, 766, 2074, 1332, 2289, 2291,
+ 2074, 2320, 2289, 2291, 1498, 2320, 766, 766, 772, 1333,
+ 2328, 2336, 1497, 1496, 2328, 2336, 2074, 1332, 1332, 1332,
1332, 1332, 1332, 1332, 1332, 1332, 1332, 1334, 1486, 1333,
1333, 1333, 1333, 1333, 1333, 1333, 1333, 1333, 1333, 1336,
1482, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625,
1625, 1626, 1464, 1459, 1441, 1399, 1626, 772, 772, 772,
772, 772, 772, 772, 772, 772, 772, 772, 772, 772,
- 772, 772, 772, 772, 772, 772, 2076, 1334, 2409, 2416,
+ 772, 772, 772, 772, 772, 772, 2078, 1334, 2364, 2412,
- 2076, 2440, 2409, 2416, 1386, 2440, 772, 772, 780, 1336,
- 2445, 2451, 1355, 1351, 2445, 2451, 2076, 1334, 1334, 1334,
- 1334, 1334, 1334, 1334, 1334, 1334, 1334, 1337, 2416, 1336,
+ 2078, 2443, 2364, 2412, 1386, 2443, 772, 772, 780, 1336,
+ 2448, 2419, 1355, 1351, 2448, 2419, 2078, 1334, 1334, 1334,
+ 1334, 1334, 1334, 1334, 1334, 1334, 1334, 1337, 1346, 1336,
1336, 1336, 1336, 1336, 1336, 1336, 1336, 1336, 1336, 1338,
- 1346, 1626, 1626, 1626, 1626, 1626, 1626, 1626, 1626, 1626,
+ 2419, 1626, 1626, 1626, 1626, 1626, 1626, 1626, 1626, 1626,
1626, 1627, 1339, 1335, 1314, 1309, 1627, 780, 780, 780,
780, 780, 780, 780, 780, 780, 780, 780, 780, 780,
- 780, 780, 780, 780, 780, 780, 2161, 1337, 2471, 2477,
- 2161, 2484, 2471, 2477, 1302, 2484, 780, 780, 788, 1338,
- 2484, 2489, 2484, 1298, 1279, 2489, 2161, 1337, 1337, 1337,
+ 780, 780, 780, 780, 780, 780, 2163, 1337, 2454, 2474,
+ 2163, 2480, 2454, 2474, 1302, 2480, 780, 780, 788, 1338,
+ 2492, 2510, 1298, 1279, 2492, 2510, 2163, 1337, 1337, 1337,
1337, 1337, 1337, 1337, 1337, 1337, 1337, 1340, 1276, 1338,
1338, 1338, 1338, 1338, 1338, 1338, 1338, 1338, 1338, 1341,
1275, 1627, 1627, 1627, 1627, 1627, 1627, 1627, 1627, 1627,
1627, 1628, 1261, 1251, 1249, 1232, 1628, 788, 788, 788,
788, 788, 788, 788, 788, 788, 788, 788, 788, 788,
- 788, 788, 788, 788, 788, 788, 2173, 1340, 2507, 2513,
- 2173, 2567, 2507, 2513, 1231, 2567, 788, 788, 796, 1341,
- 2596, 2639, 1230, 1226, 2596, 2639, 2173, 1340, 1340, 1340,
+ 788, 788, 788, 788, 788, 788, 2175, 1340, 2516, 2570,
+ 2175, 2487, 2516, 2570, 1231, 2487, 788, 788, 796, 1341,
+ 2487, 2599, 2487, 1230, 1226, 2599, 2175, 1340, 1340, 1340,
1340, 1340, 1340, 1340, 1340, 1340, 1340, 1342, 1199, 1341,
1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1343,
1198, 1628, 1628, 1628, 1628, 1628, 1628, 1628, 1628, 1628,
1628, 1629, 1197, 1196, 1195, 1194, 1629, 796, 796, 796,
796, 796, 796, 796, 796, 796, 796, 796, 796, 796,
- 796, 796, 796, 796, 796, 796, 2251, 1342, 2656, 2660,
- 2251, 2702, 2656, 2660, 1193, 2702, 796, 796, 802, 1343,
- 2726, 2727, 1182, 1175, 2726, 2727, 2251, 1342, 1342, 1342,
- 1342, 1342, 1342, 1342, 1342, 1342, 1342, 1344, 2660, 1343,
+ 796, 796, 796, 796, 796, 796, 2253, 1342, 2642, 2659,
+ 2253, 2705, 2642, 2659, 1193, 2705, 796, 796, 802, 1343,
+ 2729, 2663, 1182, 1175, 2729, 2663, 2253, 1342, 1342, 1342,
+ 1342, 1342, 1342, 1342, 1342, 1342, 1342, 1344, 1174, 1343,
1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1345,
- 1174, 1629, 1629, 1629, 1629, 1629, 1629, 1629, 1629, 1629,
- 1629, 1630, 2726, 1172, 1171, 1169, 1630, 802, 802, 802,
+ 2663, 1629, 1629, 1629, 1629, 1629, 1629, 1629, 1629, 1629,
+ 1629, 1630, 2729, 1172, 1171, 1169, 1630, 802, 802, 802,
802, 802, 802, 802, 802, 802, 802, 802, 802, 802,
- 802, 802, 802, 802, 802, 802, 2252, 1344, 2751, 2759,
- 2252, 2762, 2751, 2759, 1167, 2762, 802, 802, 808, 1345,
- 2785, 2803, 1153, 1144, 2785, 2803, 2252, 1344, 1344, 1344,
+ 802, 802, 802, 802, 802, 802, 2254, 1344, 2730, 2762,
+ 2254, 2765, 2730, 2762, 1167, 2765, 802, 802, 808, 1345,
+ 2754, 2788, 1153, 1144, 2754, 2788, 2254, 1344, 1344, 1344,
1344, 1344, 1344, 1344, 1344, 1344, 1344, 1347, 1137, 1345,
1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1348,
- 2751, 1630, 1630, 1630, 1630, 1630, 1630, 1630, 1630, 1630,
- 1630, 1631, 1132, 1092, 1059, 1049, 1631, 808, 808, 808,
+ 1132, 1630, 1630, 1630, 1630, 1630, 1630, 1630, 1630, 1630,
+ 1630, 1631, 2754, 1092, 1059, 1049, 1631, 808, 808, 808,
808, 808, 808, 808, 808, 808, 808, 808, 808, 808,
- 808, 808, 808, 808, 808, 808, 2256, 1347, 2807, 2819,
+ 808, 808, 808, 808, 808, 808, 2258, 1347, 2806, 2810,
- 2256, 2826, 2807, 2819, 1030, 2826, 808, 808, 818, 1348,
- 2827, 2829, 973, 969, 2827, 2829, 2256, 1347, 1347, 1347,
+ 2258, 2822, 2806, 2810, 1030, 2822, 808, 808, 818, 1348,
+ 2829, 2830, 973, 969, 2829, 2830, 2258, 1347, 1347, 1347,
1347, 1347, 1347, 1347, 1347, 1347, 1347, 1349, 963, 1348,
1348, 1348, 1348, 1348, 1348, 1348, 1348, 1348, 1348, 1350,
962, 1631, 1631, 1631, 1631, 1631, 1631, 1631, 1631, 1631,
1631, 1632, 961, 960, 959, 958, 1632, 818, 818, 818,
818, 818, 818, 818, 818, 818, 818, 818, 818, 818,
- 818, 818, 818, 818, 818, 818, 2327, 1349, 2830, 2847,
- 2327, 2850, 2830, 2847, 923, 2850, 818, 818, 824, 1350,
- 2868, 2879, 922, 891, 2868, 2879, 2327, 1349, 1349, 1349,
+ 818, 818, 818, 818, 818, 818, 2329, 1349, 2832, 2833,
+ 2329, 2850, 2832, 2833, 923, 2850, 818, 818, 824, 1350,
+ 2853, 2871, 922, 891, 2853, 2871, 2329, 1349, 1349, 1349,
1349, 1349, 1349, 1349, 1349, 1349, 1349, 1352, 886, 1350,
1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1353,
883, 1632, 1632, 1632, 1632, 1632, 1632, 1632, 1632, 1632,
1632, 1633, 871, 816, 794, 786, 1633, 824, 824, 824,
824, 824, 824, 824, 824, 824, 824, 824, 824, 824,
- 824, 824, 824, 824, 824, 824, 2335, 1352, 2889, 2897,
- 2335, 2290, 2889, 2897, 737, 2290, 824, 824, 830, 1353,
- 564, 562, 547, 546, 544, 541, 2335, 1352, 1352, 1352,
- 1352, 1352, 1352, 1352, 1352, 1352, 1352, 1354, 539, 1353,
- 1353, 1353, 1353, 1353, 1353, 1353, 1353, 1353, 1353, 2290,
-
- 537, 1633, 1633, 1633, 1633, 1633, 1633, 1633, 1633, 1633,
- 1633, 2113, 2114, 536, 532, 2113, 2114, 830, 830, 830,
+ 824, 824, 824, 824, 824, 824, 2337, 1352, 2882, 2892,
+ 2337, 2292, 2882, 2892, 737, 2292, 824, 824, 830, 1353,
+ 2900, 564, 562, 547, 2900, 546, 2337, 1352, 1352, 1352,
+ 1352, 1352, 1352, 1352, 1352, 1352, 1352, 1354, 544, 1353,
+ 1353, 1353, 1353, 1353, 1353, 1353, 1353, 1353, 1353, 2292,
+
+ 541, 1633, 1633, 1633, 1633, 1633, 1633, 1633, 1633, 1633,
+ 1633, 2115, 2116, 539, 537, 2115, 2116, 830, 830, 830,
830, 830, 830, 830, 830, 830, 830, 830, 830, 830,
- 830, 830, 830, 830, 830, 830, 2272, 1354, 2114, 2288,
- 2272, 531, 529, 2288, 2113, 524, 830, 830, 839, 1579,
- 1699, 2504, 1702, 1579, 1699, 2504, 1702, 1354, 1354, 1354,
- 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1634, 523, 2272,
- 1579, 1699, 1634, 1702, 522, 519, 2288, 2504, 1579, 1579,
- 1635, 515, 1579, 514, 1579, 1635, 1579, 510, 509, 507,
+ 830, 830, 830, 830, 830, 830, 2274, 1354, 2116, 2290,
+ 2274, 536, 532, 2290, 2115, 531, 830, 830, 839, 1579,
+ 1699, 2507, 1702, 1579, 1699, 2507, 1702, 1354, 1354, 1354,
+ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1634, 529, 2274,
+ 1579, 1699, 1634, 1702, 524, 523, 2290, 2507, 1579, 1579,
+ 1635, 522, 1579, 519, 1579, 1635, 1579, 515, 514, 510,
1636, 1579, 1699, 1579, 1702, 1636, 1654, 839, 839, 839,
839, 839, 839, 839, 839, 839, 839, 839, 839, 839,
839, 839, 839, 839, 839, 839, 1654, 1654, 1654, 1654,
- 1654, 1654, 1654, 1654, 1654, 1654, 839, 839, 845, 2396,
- 2384, 505, 503, 2396, 2384, 502, 501, 1634, 1634, 1634,
- 1634, 1634, 1634, 1634, 1634, 1634, 1634, 2384, 2384, 2396,
+ 1654, 1654, 1654, 1654, 1654, 1654, 839, 839, 845, 2399,
+ 2381, 509, 507, 2399, 2381, 505, 503, 1634, 1634, 1634,
+ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 2381, 2381, 2399,
1635, 1635, 1635, 1635, 1635, 1635, 1635, 1635, 1635, 1635,
1636, 1636, 1636, 1636, 1636, 1636, 1636, 1636, 1636, 1636,
- 1637, 500, 498, 496, 495, 1637, 494, 845, 845, 845,
+ 1637, 502, 501, 500, 498, 1637, 496, 845, 845, 845,
845, 845, 845, 845, 845, 845, 845, 845, 845, 845,
- 845, 845, 845, 845, 845, 845, 1708, 1724, 491, 1714,
+ 845, 845, 845, 845, 845, 845, 1708, 1724, 495, 1714,
- 1708, 1724, 489, 1714, 487, 1727, 845, 845, 851, 1727,
- 1757, 1760, 2407, 485, 1757, 1760, 2407, 1708, 1724, 1638,
- 1714, 2417, 484, 2388, 1638, 2417, 1727, 2388, 482, 1639,
- 479, 1757, 1760, 1708, 1639, 477, 1714, 2407, 472, 1760,
+ 1708, 1724, 494, 1714, 491, 1727, 845, 845, 851, 1727,
+ 1757, 1760, 2410, 489, 1757, 1760, 2410, 1708, 1724, 1638,
+ 1714, 2420, 487, 2391, 1638, 2420, 1727, 2391, 485, 1639,
+ 484, 1757, 1760, 1708, 1639, 482, 1714, 2410, 479, 1760,
1637, 1637, 1637, 1637, 1637, 1637, 1637, 1637, 1637, 1637,
- 1640, 2388, 470, 1760, 464, 1640, 2417, 851, 851, 851,
+ 1640, 2391, 477, 1760, 472, 1640, 2420, 851, 851, 851,
851, 851, 851, 851, 851, 851, 851, 851, 851, 851,
- 851, 851, 851, 851, 851, 851, 460, 408, 385, 376,
- 327, 177, 174, 166, 155, 153, 851, 851, 857, 1638,
+ 851, 851, 851, 851, 851, 851, 470, 464, 460, 408,
+ 385, 376, 327, 177, 174, 166, 851, 851, 857, 1638,
1638, 1638, 1638, 1638, 1638, 1638, 1638, 1638, 1638, 1639,
1639, 1639, 1639, 1639, 1639, 1639, 1639, 1639, 1639, 1641,
- 148, 1766, 1773, 1781, 1641, 1766, 1773, 1781, 146, 140,
+ 155, 1766, 1775, 1769, 1641, 1766, 1775, 1769, 153, 148,
1640, 1640, 1640, 1640, 1640, 1640, 1640, 1640, 1640, 1640,
- 136, 123, 1766, 1773, 1781, 1781, 0, 857, 857, 857,
+ 1769, 1769, 1766, 1775, 1769, 146, 140, 857, 857, 857,
857, 857, 857, 857, 857, 857, 857, 857, 857, 857,
- 857, 857, 857, 857, 857, 857, 1642, 0, 2593, 2855,
- 0, 1642, 2593, 2855, 0, 0, 857, 857, 862, 1696,
- 0, 1746, 862, 1696, 1694, 1746, 2855, 2855, 1694, 1641,
+ 857, 857, 857, 857, 857, 857, 1642, 136, 2596, 2387,
+ 123, 1642, 2596, 2387, 0, 0, 857, 857, 862, 1696,
+ 0, 1746, 862, 1696, 1694, 1746, 2387, 2387, 1694, 1641,
1641, 1641, 1641, 1641, 1641, 1641, 1641, 1641, 1641, 862,
- 1643, 0, 1746, 2593, 1783, 1643, 0, 1696, 1783, 1753,
+ 1643, 0, 1746, 2596, 1783, 1643, 0, 1696, 1783, 1753,
- 1792, 1644, 1694, 1753, 1792, 1696, 1644, 862, 1746, 0,
- 1694, 1696, 1694, 1696, 0, 1786, 1783, 1645, 1694, 1786,
- 1753, 1792, 1645, 0, 0, 0, 1642, 1642, 1642, 1642,
- 1642, 1642, 1642, 1642, 1642, 1642, 1753, 1786, 0, 862,
+ 1794, 1644, 1694, 1753, 1794, 1696, 1644, 862, 1746, 0,
+ 1694, 1696, 1694, 1696, 0, 1783, 1783, 1645, 1694, 0,
+ 1753, 1794, 1645, 0, 0, 0, 1642, 1642, 1642, 1642,
+ 1642, 1642, 1642, 1642, 1642, 1642, 1753, 0, 0, 862,
862, 862, 862, 862, 862, 862, 862, 862, 862, 863,
- 0, 0, 0, 863, 2512, 0, 0, 0, 2512, 0,
+ 0, 0, 0, 863, 2515, 0, 0, 0, 2515, 0,
1643, 1643, 1643, 1643, 1643, 1643, 1643, 1643, 1643, 1643,
863, 1644, 1644, 1644, 1644, 1644, 1644, 1644, 1644, 1644,
- 1644, 1788, 2512, 0, 0, 1788, 863, 1645, 1645, 1645,
- 1645, 1645, 1645, 1645, 1645, 1645, 1645, 1789, 1789, 0,
+ 1644, 1785, 2515, 0, 0, 1785, 863, 1645, 1645, 1645,
+ 1645, 1645, 1645, 1645, 1645, 1645, 1645, 1788, 2447, 0,
- 2856, 1789, 0, 1788, 2856, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 2856, 2856, 1789,
+ 0, 1788, 2447, 1785, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 2447, 1788,
863, 863, 863, 863, 863, 863, 863, 863, 863, 863,
905, 905, 905, 905, 905, 905, 905, 905, 905, 905,
905, 905, 905, 905, 905, 905, 905, 905, 905, 905,
@@ -6014,11 +6014,11 @@ static yyconst flex_int16_t yy_chk[27006] =
974, 974, 974, 974, 974, 974, 974, 974, 974, 974,
974, 974, 974, 974, 974, 974, 974, 974, 980, 1686,
- 0, 1700, 980, 1686, 980, 1700, 1810, 0, 1813, 0,
- 1810, 1646, 1813, 1824, 0, 2857, 1646, 1824, 1797, 2857,
- 1686, 1647, 1797, 0, 0, 0, 1647, 1686, 1810, 1700,
- 1813, 1648, 2857, 2857, 1824, 1686, 1648, 1700, 0, 1797,
- 1797, 1649, 0, 1686, 0, 1700, 1649, 0, 1700, 980,
+ 0, 1700, 980, 1686, 980, 1700, 1790, 0, 1791, 1791,
+ 1790, 1646, 1791, 1826, 0, 2858, 1646, 1826, 1799, 2858,
+ 1686, 1647, 1799, 0, 0, 0, 1647, 1686, 1790, 1700,
+ 1791, 1648, 2858, 2858, 1826, 1686, 1648, 1700, 0, 1799,
+ 1799, 1649, 0, 1686, 0, 1700, 1649, 0, 1700, 980,
980, 980, 980, 980, 980, 980, 980, 980, 980, 980,
980, 980, 980, 980, 980, 980, 980, 980, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 980, 980,
@@ -6029,16 +6029,16 @@ static yyconst flex_int16_t yy_chk[27006] =
1648, 1649, 1649, 1649, 1649, 1649, 1649, 1649, 1649, 1649,
1649, 999, 999, 999, 999, 999, 999, 999, 999, 999,
999, 999, 999, 999, 999, 999, 999, 999, 999, 999,
- 1081, 1688, 1698, 1811, 1081, 1688, 1698, 1811, 0, 0,
- 1833, 1820, 0, 0, 1833, 1820, 1688, 1650, 0, 1701,
- 0, 1081, 1650, 1701, 1703, 1811, 0, 0, 1703, 1688,
- 1698, 1833, 1811, 1820, 0, 1857, 0, 1688, 1698, 1857,
- 1820, 0, 0, 1651, 1698, 1688, 1698, 1701, 1651, 0,
- 2444, 0, 1703, 1652, 2444, 1701, 1857, 1857, 1652, 0,
-
- 1703, 1701, 1741, 1701, 0, 1860, 1741, 1703, 1703, 1860,
- 2444, 1081, 1081, 1081, 1081, 1081, 1081, 1081, 1081, 1081,
- 1081, 1082, 0, 1741, 0, 1082, 1860, 1650, 1650, 1650,
+ 1081, 1688, 1698, 1812, 1081, 1688, 1698, 1812, 0, 1815,
+ 1813, 1822, 0, 1815, 1813, 1822, 1688, 1650, 0, 1701,
+ 0, 1081, 1650, 1701, 1703, 1812, 0, 0, 1703, 1688,
+ 1698, 1815, 1813, 1822, 0, 0, 0, 1688, 1698, 1813,
+ 1822, 0, 0, 1651, 1698, 1688, 1698, 1701, 1651, 0,
+ 2498, 0, 1703, 1652, 2498, 1701, 0, 0, 1652, 0,
+
+ 1703, 1701, 1741, 1701, 0, 1835, 1741, 1703, 1703, 1835,
+ 2498, 1081, 1081, 1081, 1081, 1081, 1081, 1081, 1081, 1081,
+ 1081, 1082, 0, 1741, 0, 1082, 1835, 1650, 1650, 1650,
1650, 1650, 1650, 1650, 1650, 1650, 1650, 1741, 0, 0,
0, 0, 1082, 0, 0, 1704, 0, 1706, 1741, 1704,
0, 1706, 1082, 1651, 1651, 1651, 1651, 1651, 1651, 1651,
@@ -6094,10 +6094,10 @@ static yyconst flex_int16_t yy_chk[27006] =
1106, 1106, 1106, 1106, 1106, 1106, 1106, 1106, 1106, 1106,
1106, 1106, 1106, 1106, 1106, 1106, 1106, 1106, 1106, 1106,
1106, 1106, 1117, 1707, 0, 1709, 1117, 1707, 1117, 1709,
- 0, 1876, 0, 0, 1908, 1876, 1922, 2495, 1908, 0,
- 1922, 2495, 0, 1923, 0, 1917, 0, 1923, 0, 1917,
- 0, 1707, 1876, 1709, 1709, 1908, 0, 2495, 1922, 1707,
- 1707, 1709, 1917, 1917, 1923, 1923, 1917, 1707, 1876, 1709,
+ 0, 1862, 0, 0, 1859, 1862, 1910, 1931, 1859, 0,
+ 1910, 1931, 1924, 0, 1949, 1919, 1924, 0, 1949, 1919,
+ 0, 1707, 1862, 1709, 1709, 1859, 1859, 1910, 1931, 1707,
+ 1707, 1709, 1919, 1919, 1924, 1949, 1919, 1707, 0, 1709,
0, 0, 0, 1117, 1117, 1117, 1117, 1117, 1117, 1117,
1117, 1117, 1117, 1117, 1117, 1117, 1117, 1117, 1117, 1117,
1117, 1117, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -6139,22 +6139,22 @@ static yyconst flex_int16_t yy_chk[27006] =
1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
- 1358, 1710, 0, 1712, 1358, 1710, 1713, 1712, 2538, 0,
- 1713, 0, 2538, 1715, 1929, 1870, 1655, 1715, 1929, 1870,
- 1716, 1358, 0, 1726, 1716, 0, 0, 1726, 2538, 1710,
- 1710, 1712, 1712, 1656, 1713, 1929, 1870, 1710, 0, 1712,
+ 1358, 1710, 0, 1712, 1358, 1710, 1713, 1712, 2541, 0,
+ 1713, 0, 2541, 1715, 2002, 1872, 1655, 1715, 2002, 1872,
+ 1716, 1358, 0, 1726, 1716, 0, 0, 1726, 2541, 1710,
+ 1710, 1712, 1712, 1656, 1713, 2002, 1872, 1710, 0, 1712,
0, 1715, 1713, 1713, 1358, 1710, 1657, 1712, 1716, 1715,
1713, 1726, 0, 0, 1715, 0, 1716, 1715, 0, 1726,
- 1870, 1947, 0, 1658, 1716, 1947, 1655, 1726, 0, 0,
+ 1872, 1878, 0, 1658, 1716, 1878, 1655, 1726, 0, 0,
0, 1358, 1358, 1358, 1358, 1358, 1358, 1358, 1358, 1358,
- 1358, 1359, 1947, 1656, 0, 1359, 1655, 1655, 1655, 1655,
- 1655, 1655, 1655, 1655, 1655, 1655, 1657, 0, 0, 0,
- 0, 0, 1359, 1656, 1656, 1656, 1656, 1656, 1656, 1656,
- 1656, 1656, 1656, 1658, 0, 0, 1657, 1657, 1657, 1657,
- 1657, 1657, 1657, 1657, 1657, 1657, 1924, 0, 1926, 0,
- 1924, 0, 1926, 1658, 1658, 1658, 1658, 1658, 1658, 1658,
- 1658, 1658, 1658, 0, 0, 1926, 1926, 1924, 1924, 1926,
+ 1358, 1359, 1878, 1656, 0, 1359, 1655, 1655, 1655, 1655,
+ 1655, 1655, 1655, 1655, 1655, 1655, 1657, 2859, 1878, 0,
+ 0, 2859, 1359, 1656, 1656, 1656, 1656, 1656, 1656, 1656,
+ 1656, 1656, 1656, 1658, 2859, 2859, 1657, 1657, 1657, 1657,
+ 1657, 1657, 1657, 1657, 1657, 1657, 1925, 2860, 1926, 0,
+ 1925, 2860, 1926, 1658, 1658, 1658, 1658, 1658, 1658, 1658,
+ 1658, 1658, 1658, 0, 2860, 2860, 0, 1925, 1925, 1926,
1926, 0, 1359, 1359, 1359, 1359, 1359, 1359, 1359, 1359,
1359, 1359, 1361, 1361, 1361, 1361, 1361, 1361, 1361, 1361,
@@ -6166,12 +6166,12 @@ static yyconst flex_int16_t yy_chk[27006] =
1361, 1361, 1361, 1361, 1361, 1361, 1361, 1361, 1361, 1361,
1361, 1361, 1361, 1361, 1361, 1361, 1361, 1361, 1361, 1361,
1361, 1361, 1361, 1361, 1361, 1361, 1364, 1717, 0, 1719,
- 1364, 1717, 1364, 1719, 0, 2000, 0, 1977, 1992, 2000,
+ 1364, 1717, 1364, 1719, 0, 2012, 0, 1979, 1994, 2012,
- 2010, 1977, 1992, 0, 2010, 2020, 0, 2028, 2059, 2020,
- 2039, 2028, 2059, 0, 2039, 1717, 2000, 1719, 1977, 1992,
- 0, 2010, 1719, 1717, 1977, 1719, 0, 2020, 2028, 2059,
- 1717, 1717, 2039, 1719, 1992, 0, 0, 1364, 1364, 1364,
+ 0, 1979, 1994, 0, 2022, 0, 2030, 2041, 2022, 0,
+ 2030, 2041, 0, 0, 0, 1717, 2012, 1719, 1979, 1994,
+ 0, 0, 1719, 1717, 1979, 1719, 2022, 2030, 0, 2041,
+ 1717, 1717, 0, 1719, 1994, 0, 0, 1364, 1364, 1364,
1364, 1364, 1364, 1364, 1364, 1364, 1364, 1364, 1364, 1364,
1364, 1364, 1364, 1364, 1364, 1364, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 1364, 1364, 1371, 1371,
@@ -6249,33 +6249,33 @@ static yyconst flex_int16_t yy_chk[27006] =
1605, 1605, 1605, 1605, 1605, 1605, 1605, 1605, 1605, 1605,
1605, 1605, 1605, 1605, 1605, 1605, 1605, 1605, 1605, 1605,
1605, 1605, 1605, 1605, 1605, 1605, 1605, 1605, 1605, 1605,
- 1659, 1725, 1728, 2583, 0, 1725, 1728, 2583, 1730, 0,
- 1660, 1729, 1730, 1732, 2092, 1729, 0, 1732, 2092, 0,
- 1731, 2041, 0, 2583, 1731, 2041, 1725, 0, 0, 1725,
- 1728, 0, 0, 0, 1730, 2092, 1730, 1725, 1728, 1729,
- 1661, 1732, 0, 2041, 1730, 1725, 1728, 1729, 1731, 1732,
+ 1659, 1725, 1728, 2586, 0, 1725, 1728, 2586, 1730, 0,
+ 1660, 1729, 1730, 1732, 2061, 1729, 0, 1732, 2061, 0,
+ 1731, 2043, 0, 2586, 1731, 2043, 1725, 0, 0, 1725,
+ 1728, 0, 0, 0, 1730, 2061, 1730, 1725, 1728, 1729,
+ 1661, 1732, 0, 2043, 1730, 1725, 1728, 1729, 1731, 1732,
1659, 1662, 1730, 1731, 1729, 1729, 1731, 1732, 0, 0,
- 1660, 0, 2041, 0, 1731, 0, 0, 0, 0, 0,
+ 1660, 0, 2043, 0, 1731, 0, 0, 0, 0, 0,
1659, 1659, 1659, 1659, 1659, 1659, 1659, 1659, 1659, 1659,
1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660,
1661, 1663, 0, 0, 0, 0, 0, 0, 0, 0,
- 2778, 1662, 1664, 1736, 2778, 0, 0, 1736, 0, 0,
+ 2781, 1662, 1664, 1736, 2781, 0, 0, 1736, 0, 0,
1661, 1661, 1661, 1661, 1661, 1661, 1661, 1661, 1661, 1661,
- 2778, 1662, 1662, 1662, 1662, 1662, 1662, 1662, 1662, 1662,
- 1662, 1736, 1665, 2040, 2043, 0, 1736, 2040, 2043, 1736,
- 2832, 1663, 1666, 0, 2832, 0, 0, 1736, 0, 2051,
- 2040, 2040, 1664, 2051, 0, 2040, 2043, 0, 0, 0,
+ 2781, 1662, 1662, 1662, 1662, 1662, 1662, 1662, 1662, 1662,
+ 1662, 1736, 1665, 1928, 2045, 0, 1736, 1928, 2045, 1736,
+ 2835, 1663, 1666, 0, 2835, 0, 0, 1736, 0, 2053,
+ 1928, 1928, 1664, 2053, 1928, 1928, 2045, 0, 0, 0,
- 2832, 1663, 1663, 1663, 1663, 1663, 1663, 1663, 1663, 1663,
- 1663, 2051, 1664, 1664, 1664, 1664, 1664, 1664, 1664, 1664,
+ 2835, 1663, 1663, 1663, 1663, 1663, 1663, 1663, 1663, 1663,
+ 1663, 2053, 1664, 1664, 1664, 1664, 1664, 1664, 1664, 1664,
1664, 1664, 1665, 1667, 0, 0, 0, 0, 0, 0,
- 2051, 0, 1666, 1668, 1744, 0, 1747, 0, 1744, 0,
+ 2053, 0, 1666, 1668, 1744, 0, 1747, 0, 1744, 0,
1747, 0, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665,
1665, 1665, 1666, 1666, 1666, 1666, 1666, 1666, 1666, 1666,
- 1666, 1666, 1744, 1669, 1747, 2096, 0, 2050, 0, 2096,
- 1744, 2050, 1747, 1667, 1670, 0, 0, 1744, 1744, 0,
- 1747, 0, 0, 1668, 2050, 2050, 2096, 1748, 0, 2050,
+ 1666, 1666, 1744, 1669, 1747, 2094, 0, 2042, 0, 2094,
+ 1744, 2042, 1747, 1667, 1670, 0, 0, 1744, 1744, 0,
+ 1747, 0, 0, 1668, 2042, 2042, 2094, 1748, 0, 2042,
0, 1748, 0, 1667, 1667, 1667, 1667, 1667, 1667, 1667,
1667, 1667, 1667, 1668, 1668, 1668, 1668, 1668, 1668, 1668,
@@ -6283,53 +6283,53 @@ static yyconst flex_int16_t yy_chk[27006] =
0, 0, 0, 1748, 1670, 1672, 1745, 0, 0, 0,
1745, 1748, 0, 1669, 1669, 1669, 1669, 1669, 1669, 1669,
1669, 1669, 1669, 0, 1670, 1670, 1670, 1670, 1670, 1670,
- 1670, 1670, 1670, 1670, 1745, 1673, 2054, 2153, 0, 0,
- 2054, 2153, 1745, 1745, 1671, 1674, 0, 0, 0, 0,
- 1745, 0, 0, 2150, 0, 1672, 2054, 2150, 2054, 2153,
+ 1670, 1670, 1670, 1670, 1745, 1673, 2052, 2056, 0, 0,
+ 2052, 2056, 1745, 1745, 1671, 1674, 0, 0, 0, 0,
+ 1745, 0, 2155, 2052, 2052, 1672, 2155, 2056, 2052, 2056,
0, 0, 0, 0, 1671, 1671, 1671, 1671, 1671, 1671,
- 1671, 1671, 1671, 1671, 2150, 1672, 1672, 1672, 1672, 1672,
+ 1671, 1671, 1671, 1671, 2155, 1672, 1672, 1672, 1672, 1672,
1672, 1672, 1672, 1672, 1672, 1673, 1675, 0, 0, 0,
0, 0, 0, 0, 0, 1674, 1676, 1749, 0, 1751,
0, 1749, 0, 1751, 0, 1673, 1673, 1673, 1673, 1673,
1673, 1673, 1673, 1673, 1673, 1674, 1674, 1674, 1674, 1674,
- 1674, 1674, 1674, 1674, 1674, 1749, 1677, 1751, 2066, 0,
- 2087, 0, 2066, 1749, 2087, 1751, 1675, 1678, 0, 0,
- 1749, 1749, 0, 1751, 0, 0, 1676, 0, 2066, 2066,
- 1754, 2087, 2087, 0, 1754, 0, 1675, 1675, 1675, 1675,
+ 1674, 1674, 1674, 1674, 1674, 1749, 1677, 1751, 2068, 0,
+ 2089, 0, 2068, 1749, 2089, 1751, 1675, 1678, 0, 0,
+ 1749, 1749, 0, 1751, 0, 0, 1676, 0, 2068, 2068,
+ 1754, 2089, 2089, 0, 1754, 0, 1675, 1675, 1675, 1675,
1675, 1675, 1675, 1675, 1675, 1675, 1676, 1676, 1676, 1676,
1676, 1676, 1676, 1676, 1676, 1676, 1677, 1679, 1754, 0,
0, 0, 0, 0, 0, 0, 1754, 1678, 1680, 1752,
0, 0, 0, 1752, 1754, 0, 1677, 1677, 1677, 1677,
1677, 1677, 1677, 1677, 1677, 1677, 0, 1678, 1678, 1678,
- 1678, 1678, 1678, 1678, 1678, 1678, 1678, 1752, 1681, 2073,
- 2228, 0, 0, 2073, 2228, 1752, 1752, 1679, 1682, 0,
- 0, 0, 0, 1752, 0, 0, 0, 0, 1680, 2073,
- 0, 1756, 2228, 0, 0, 1756, 0, 1679, 1679, 1679,
- 1679, 1679, 1679, 1679, 1679, 1679, 1679, 2073, 1680, 1680,
+ 1678, 1678, 1678, 1678, 1678, 1678, 1678, 1752, 1681, 2075,
+ 2230, 0, 0, 2075, 2230, 1752, 1752, 1679, 1682, 0,
+ 0, 0, 0, 1752, 0, 0, 0, 0, 1680, 2075,
+ 0, 1756, 2230, 0, 0, 1756, 0, 1679, 1679, 1679,
+ 1679, 1679, 1679, 1679, 1679, 1679, 1679, 2075, 1680, 1680,
1680, 1680, 1680, 1680, 1680, 1680, 1680, 1680, 1681, 1756,
0, 0, 0, 0, 0, 0, 0, 1756, 1682, 0,
0, 0, 1758, 0, 0, 1756, 1758, 0, 1681, 1681,
1681, 1681, 1681, 1681, 1681, 1681, 1681, 1681, 1682, 1682,
1682, 1682, 1682, 1682, 1682, 1682, 1682, 1682, 1684, 1737,
- 1758, 1758, 1684, 1737, 0, 1759, 0, 2151, 1758, 1759,
- 1761, 2151, 0, 0, 1761, 0, 1758, 1762, 0, 1684,
- 0, 1762, 1765, 0, 1737, 0, 1765, 1737, 2151, 0,
- 2198, 0, 0, 1759, 2198, 1737, 1684, 0, 1761, 1761,
- 0, 1759, 0, 1737, 2171, 1762, 1761, 1759, 2171, 1759,
- 1765, 2198, 0, 1762, 1761, 0, 0, 0, 1765, 1768,
- 1762, 1762, 0, 1768, 2171, 2171, 1765, 0, 0, 1684,
+ 1758, 1758, 1684, 1737, 0, 1759, 0, 2098, 1758, 1759,
+ 1761, 2098, 0, 0, 1761, 0, 1758, 1762, 0, 1684,
+ 0, 1762, 1765, 0, 1737, 0, 1765, 1737, 2098, 0,
+ 2152, 0, 0, 1759, 2152, 1737, 1684, 0, 1761, 1761,
+ 0, 1759, 0, 1737, 2153, 1762, 1761, 1759, 2153, 1759,
+ 1765, 2152, 0, 1762, 1761, 0, 0, 0, 1765, 1770,
+ 1762, 1762, 0, 1770, 0, 2153, 1765, 0, 0, 1684,
1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1685,
- 1771, 1768, 1769, 1685, 1771, 2216, 1769, 1768, 0, 2216,
- 0, 0, 1772, 2264, 1774, 1768, 1772, 2264, 1774, 0,
- 1685, 1768, 0, 1768, 0, 1768, 2216, 1685, 1771, 0,
- 1769, 0, 0, 2055, 2264, 1685, 1771, 2055, 1769, 0,
- 1772, 0, 1774, 1685, 1771, 1769, 1769, 2230, 1772, 1771,
- 1774, 2230, 0, 2055, 2055, 2055, 1772, 0, 1774, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 2230,
+ 1768, 1770, 1771, 1685, 1768, 1773, 1771, 1770, 0, 1773,
+ 2200, 0, 1774, 0, 2200, 1770, 1774, 1768, 1768, 0,
+ 1685, 1770, 1776, 1770, 0, 1770, 1776, 1685, 1768, 0,
+ 1771, 2200, 0, 1773, 0, 1685, 1768, 0, 1771, 0,
+ 1774, 1773, 0, 1685, 1768, 1771, 1771, 2057, 1774, 1773,
+ 1776, 2057, 0, 0, 1773, 0, 1774, 0, 1776, 0,
+ 0, 0, 0, 0, 0, 0, 1776, 2057, 2057, 2057,
1685, 1685, 1685, 1685, 1685, 1685, 1685, 1685, 1685, 1685,
1689, 1689, 1689, 1689, 1689, 1689, 1689, 1689, 1689, 1689,
@@ -6349,1344 +6349,1344 @@ static yyconst flex_int16_t yy_chk[27006] =
1733, 1733, 1733, 1733, 1733, 1733, 1733, 1733, 1733, 1733,
1733, 1733, 1733, 1733, 1733, 1733, 1733, 1733, 1733, 1733,
1733, 1733, 1733, 1733, 1733, 1733, 1733, 1733, 1733, 1733,
- 1733, 1733, 1733, 1733, 1733, 1733, 1733, 1733, 1738, 1777,
- 0, 1779, 1738, 1777, 1738, 1779, 2236, 0, 0, 0,
- 2236, 2237, 0, 2269, 0, 2237, 0, 2269, 2238, 0,
- 2282, 0, 2238, 0, 2282, 0, 2240, 1777, 2236, 1779,
- 2240, 2237, 0, 2237, 2269, 1777, 0, 1779, 2238, 2238,
+ 1733, 1733, 1733, 1733, 1733, 1733, 1733, 1733, 1738, 1779,
+ 0, 1781, 1738, 1779, 1738, 1781, 0, 2218, 0, 0,
+ 2173, 2218, 2232, 0, 2173, 2266, 2232, 2238, 2239, 2266,
+ 2271, 2238, 2239, 0, 2271, 0, 0, 1779, 2218, 1781,
+ 2173, 2173, 0, 0, 2232, 1779, 2266, 1781, 2239, 2238,
- 2238, 2282, 1777, 1777, 0, 1779, 2240, 2240, 2240, 1738,
+ 2239, 2271, 1779, 1779, 0, 1781, 0, 0, 0, 1738,
1738, 1738, 1738, 1738, 1738, 1738, 1738, 1738, 1738, 1738,
- 1738, 1738, 1738, 1738, 1738, 1738, 1738, 1738, 1780, 2253,
- 2285, 2320, 1780, 2253, 2285, 2320, 2321, 0, 1738, 1738,
- 2321, 2336, 2470, 2380, 2500, 2336, 2470, 2380, 2500, 2253,
- 1780, 2285, 0, 2320, 0, 0, 1780, 0, 2321, 0,
- 2253, 0, 2336, 2470, 1780, 2380, 2500, 0, 0, 0,
- 0, 0, 1780, 1782, 1782, 1782, 1782, 1782, 1782, 1782,
- 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782,
- 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782,
-
- 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782,
- 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782,
- 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782,
- 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782,
- 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782,
- 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1785, 0, 2437,
- 1793, 1785, 2458, 2437, 1793, 0, 2458, 0, 0, 1795,
- 0, 0, 1796, 1795, 0, 0, 1796, 2443, 0, 1785,
- 2437, 2443, 1798, 2458, 0, 1785, 1798, 0, 1793, 2437,
- 0, 1801, 2458, 1785, 1796, 1801, 1793, 1795, 2443, 0,
-
- 1796, 1785, 1787, 1787, 1793, 1795, 1787, 1787, 1796, 1787,
- 1798, 2386, 0, 1795, 0, 2386, 1796, 0, 1798, 1801,
- 2506, 2443, 0, 1787, 2506, 0, 1798, 1801, 2386, 2386,
- 0, 0, 2565, 2386, 1801, 1801, 2565, 0, 0, 0,
- 0, 2506, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 2565, 1787, 1787, 1787, 1787, 1787, 1787,
- 1787, 1787, 1787, 1787, 1787, 1787, 1787, 1787, 1787, 1787,
- 1787, 1787, 1787, 1787, 1787, 1787, 1787, 1787, 1787, 1787,
- 1787, 1787, 1787, 1787, 1787, 1799, 1802, 1803, 0, 1799,
- 1802, 1803, 0, 0, 0, 0, 1804, 2662, 1805, 1806,
-
- 1804, 2662, 1805, 1806, 0, 0, 0, 2687, 1823, 1803,
- 1799, 2687, 1823, 1799, 1802, 1803, 1803, 2662, 2662, 1802,
- 0, 1799, 1802, 1803, 1804, 1804, 1805, 1806, 2687, 1799,
- 1802, 1803, 1804, 1807, 1805, 1806, 1823, 1807, 0, 1805,
- 1804, 0, 1805, 1806, 1823, 1825, 0, 1826, 0, 1825,
- 1832, 1826, 1823, 0, 1832, 1807, 0, 0, 0, 1828,
- 1834, 1807, 1807, 1828, 1834, 1825, 0, 0, 0, 1807,
- 1830, 0, 1835, 1825, 1830, 1826, 1835, 1807, 1832, 0,
- 1834, 1825, 0, 1826, 0, 1830, 1832, 1828, 1834, 1825,
- 1826, 1826, 1828, 1836, 1832, 1828, 1834, 1836, 1830, 0,
-
- 1835, 0, 0, 1828, 1834, 1845, 1830, 1847, 1835, 1845,
- 0, 1847, 0, 0, 1830, 1835, 1835, 0, 1850, 1848,
- 0, 1836, 1850, 1848, 0, 0, 0, 0, 1836, 1836,
- 1852, 0, 0, 1845, 1852, 1847, 1847, 1836, 2379, 0,
- 0, 1845, 2379, 1847, 1848, 0, 1850, 1848, 1845, 1845,
- 0, 1847, 1853, 1856, 1850, 1848, 1853, 1856, 1852, 1852,
- 2379, 0, 1850, 1848, 1858, 2385, 1852, 0, 1858, 2385,
- 0, 0, 2483, 1861, 1852, 1856, 2483, 1861, 1862, 0,
- 1853, 1856, 1862, 0, 2379, 0, 0, 2385, 1853, 1856,
- 0, 0, 1858, 0, 2483, 1853, 1853, 1856, 2535, 2589,
-
- 1858, 1861, 2535, 2589, 0, 0, 1862, 1858, 1858, 1861,
- 2483, 2385, 0, 0, 1862, 1861, 0, 1861, 0, 0,
- 2535, 2589, 1862, 1863, 1863, 1863, 1863, 1863, 1863, 1863,
- 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863,
- 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863,
- 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863,
- 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863,
- 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863,
- 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863,
- 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863,
-
- 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1866, 0, 1872,
- 1874, 1866, 1875, 1872, 1874, 0, 1875, 0, 0, 1877,
- 2571, 0, 1878, 1877, 2571, 0, 1878, 0, 0, 1879,
- 0, 0, 1866, 1879, 0, 1866, 0, 1872, 1874, 1874,
- 1875, 0, 2571, 1866, 0, 1872, 1874, 1877, 1875, 1875,
- 1878, 1866, 1872, 1872, 1874, 1877, 1875, 1879, 1878, 0,
- 1883, 2571, 0, 1877, 1883, 1879, 1878, 1884, 1879, 0,
- 1885, 1884, 1886, 1879, 1885, 0, 1886, 1887, 0, 1890,
- 0, 1887, 1884, 1890, 0, 0, 0, 2578, 1883, 1891,
- 1892, 2578, 0, 1891, 1892, 1884, 1883, 2487, 1885, 0,
-
- 1886, 2487, 1886, 1884, 1883, 1887, 1885, 1890, 1886, 2578,
- 0, 1884, 1885, 1887, 1885, 1890, 1886, 1891, 1892, 2487,
- 1893, 1887, 1890, 1890, 1893, 1891, 1892, 1894, 2578, 0,
- 1892, 1894, 1895, 1891, 1892, 2487, 1895, 1896, 1897, 0,
- 2587, 1896, 1897, 0, 2587, 0, 0, 2618, 1893, 1898,
- 2619, 2618, 0, 1898, 2619, 1894, 1893, 0, 0, 0,
- 1895, 2587, 2587, 1894, 1893, 1896, 1897, 1897, 1895, 2618,
- 1895, 1894, 2619, 1896, 1897, 1896, 1895, 1898, 0, 1907,
- 1911, 1896, 1897, 1907, 1911, 1898, 1898, 0, 1898, 0,
- 1909, 1912, 2468, 1898, 1909, 1912, 2468, 1913, 0, 2701,
-
- 2569, 1913, 1919, 2701, 2569, 0, 1919, 1907, 1911, 1911,
- 0, 0, 0, 2468, 0, 1907, 1911, 1909, 1909, 1912,
- 2701, 2569, 0, 1907, 1911, 1913, 1909, 1912, 1912, 2468,
- 1919, 1919, 1916, 1913, 1909, 1912, 1916, 2569, 1919, 1918,
- 1913, 1913, 0, 1918, 1921, 0, 1919, 2705, 1921, 1916,
- 1916, 2705, 0, 0, 0, 0, 0, 0, 1927, 0,
- 1916, 1918, 1927, 0, 0, 0, 0, 1918, 1916, 2705,
- 1932, 0, 1921, 1925, 1932, 1918, 1916, 1925, 2616, 0,
- 1921, 1918, 2616, 1918, 0, 1918, 1927, 1921, 1921, 2705,
- 1925, 1925, 2729, 0, 1927, 1925, 2729, 0, 1932, 2616,
-
- 0, 1925, 1927, 0, 0, 1932, 1932, 0, 0, 1925,
- 0, 0, 0, 2729, 1932, 2616, 0, 1925, 1928, 1928,
- 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928,
- 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928,
- 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928,
- 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928,
- 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928,
- 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928,
- 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928,
- 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928, 1928,
-
- 1928, 1928, 1935, 0, 1937, 1938, 1935, 1939, 1937, 1938,
- 0, 1939, 2722, 0, 2642, 0, 2722, 0, 2642, 1941,
- 0, 0, 2712, 1941, 1940, 0, 2712, 0, 1940, 0,
- 1935, 0, 1937, 1938, 2722, 1939, 2642, 0, 1935, 1937,
- 1937, 1938, 2642, 1939, 2712, 1939, 1935, 1941, 1937, 1938,
- 1945, 1939, 1940, 1940, 1945, 1941, 0, 1946, 1941, 0,
- 1940, 1946, 1948, 1941, 2712, 0, 1948, 2721, 1940, 1950,
- 2753, 2721, 0, 1950, 2753, 1952, 0, 0, 1945, 1952,
- 2765, 0, 0, 0, 2765, 1946, 1945, 2649, 2721, 2721,
- 1948, 2649, 2753, 1946, 1945, 1948, 2761, 1950, 1948, 1946,
-
- 2761, 1946, 2765, 1952, 0, 1950, 1948, 0, 1952, 2649,
- 0, 1952, 1950, 1950, 0, 2649, 0, 2761, 0, 1952,
- 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954,
- 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954,
- 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954,
- 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954,
- 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954,
- 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954,
- 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954,
- 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954, 1954,
-
- 1954, 1954, 1954, 1954, 1955, 1957, 1958, 2771, 1955, 1957,
- 1958, 2771, 0, 0, 2781, 1960, 2814, 1961, 2781, 1960,
- 2814, 1961, 0, 0, 1963, 0, 1964, 1966, 1963, 2771,
- 1964, 1966, 1955, 1957, 1958, 1958, 2781, 1955, 2814, 0,
- 1955, 1957, 1958, 1960, 1960, 1961, 1961, 0, 1955, 1957,
- 1958, 1960, 1963, 1961, 1964, 1966, 0, 0, 0, 1960,
- 1963, 1961, 1964, 1966, 1966, 0, 0, 0, 1963, 1964,
- 1964, 1966, 1967, 0, 1969, 1970, 1967, 1971, 1969, 1970,
- 0, 1971, 0, 2805, 2818, 0, 1973, 2805, 2818, 1975,
- 1973, 0, 0, 1975, 2825, 0, 0, 0, 2825, 0,
-
- 1967, 1967, 1969, 1970, 2805, 1971, 2818, 0, 1967, 0,
- 1969, 1970, 1969, 1971, 1973, 2825, 1967, 1975, 1969, 1970,
- 1976, 1971, 1973, 1973, 1976, 1975, 0, 1978, 0, 0,
- 1973, 1978, 1975, 1975, 1979, 0, 0, 1980, 1979, 1983,
- 1984, 1980, 0, 1983, 1984, 2846, 1987, 1976, 1976, 2846,
- 1987, 0, 0, 0, 0, 1978, 1976, 2848, 0, 0,
- 0, 2848, 1979, 1978, 1976, 1980, 2846, 1983, 1984, 1984,
- 1979, 1978, 1983, 1980, 1987, 1983, 1984, 0, 1979, 2848,
- 0, 1980, 1987, 1983, 1984, 1986, 1980, 1989, 0, 1986,
- 1987, 1989, 1991, 0, 1994, 0, 1991, 1995, 1994, 0,
-
- 0, 1995, 0, 0, 0, 2849, 1999, 1996, 0, 2849,
- 1999, 1996, 1986, 1986, 0, 1989, 1989, 0, 0, 0,
- 1991, 1986, 1994, 1989, 0, 1995, 2849, 1995, 1991, 1986,
- 1994, 1989, 1996, 1995, 1999, 1996, 1991, 0, 1994, 0,
- 2001, 1995, 1999, 1996, 2001, 0, 0, 0, 0, 2002,
- 1999, 1996, 2003, 2002, 2005, 0, 2003, 0, 2005, 0,
- 0, 2006, 0, 2008, 0, 2006, 2025, 2008, 2001, 0,
- 2025, 2002, 0, 2001, 0, 2027, 2001, 2002, 0, 2027,
- 2003, 0, 2005, 2005, 2001, 2002, 0, 2002, 2003, 2006,
- 2005, 2008, 0, 2002, 2025, 0, 2003, 2006, 2005, 2008,
-
- 2029, 0, 2025, 2027, 2029, 2006, 0, 2008, 2025, 2031,
- 2025, 2027, 2032, 2031, 0, 0, 2032, 0, 2034, 2027,
- 2780, 0, 2034, 0, 2780, 0, 0, 2029, 2029, 2033,
- 2035, 0, 0, 2033, 2035, 0, 2029, 2031, 0, 2036,
- 2032, 2780, 2780, 2036, 2029, 2031, 2034, 0, 2032, 2032,
- 2031, 2033, 0, 2031, 2034, 2035, 2032, 2033, 2035, 2056,
- 2058, 2036, 2034, 2056, 2058, 2033, 2035, 2036, 0, 0,
- 2033, 0, 2060, 2033, 2035, 2036, 2060, 2196, 2063, 0,
- 2036, 2196, 2063, 2036, 0, 0, 0, 2056, 2058, 0,
- 2065, 0, 0, 0, 2065, 2056, 2058, 2067, 2196, 0,
-
- 2060, 2067, 2196, 2056, 2058, 0, 2063, 2068, 2060, 2060,
- 2065, 2068, 0, 2063, 2063, 0, 2060, 2063, 2065, 2077,
- 0, 2196, 2063, 2077, 2063, 2067, 2065, 2068, 2079, 0,
- 0, 0, 2079, 2067, 2065, 2068, 0, 0, 2067, 0,
- 2080, 2067, 0, 2068, 2080, 2068, 2083, 2077, 2811, 2082,
- 2083, 2068, 2811, 2082, 0, 2077, 2079, 0, 2086, 0,
- 0, 2088, 2086, 2077, 2079, 2088, 0, 2080, 2080, 2811,
- 2811, 2083, 2079, 0, 2083, 0, 2080, 2082, 0, 2082,
- 2086, 0, 2083, 2090, 2080, 2082, 2086, 2090, 0, 2088,
- 2083, 0, 2094, 2082, 2086, 0, 2094, 2088, 0, 0,
-
- 0, 0, 2086, 2097, 2100, 2088, 2102, 2097, 2100, 2103,
- 2102, 2090, 0, 2103, 0, 0, 2090, 0, 2105, 2090,
- 2094, 2106, 2105, 0, 0, 2106, 0, 2090, 2094, 0,
- 0, 2097, 2100, 0, 2102, 2094, 2094, 2103, 2103, 2097,
- 2100, 0, 2102, 0, 2097, 2103, 2105, 2097, 2100, 2106,
- 2102, 2109, 2110, 2103, 2105, 2109, 2110, 2106, 0, 0,
- 0, 2105, 2105, 2112, 2111, 2106, 0, 2112, 2111, 0,
- 2115, 0, 0, 2116, 2115, 0, 0, 2116, 0, 2109,
- 2110, 0, 2117, 0, 0, 2110, 2117, 2109, 2110, 2112,
- 0, 2112, 2111, 0, 2109, 2109, 2110, 2111, 2115, 2112,
-
- 2111, 2116, 0, 0, 0, 0, 2115, 2112, 2111, 2116,
- 2117, 2118, 0, 0, 2115, 2118, 2119, 2116, 2117, 2117,
- 2119, 2120, 0, 0, 0, 2120, 2117, 0, 2121, 0,
- 0, 2122, 2121, 2714, 0, 2122, 0, 2714, 0, 2118,
- 2118, 2123, 2124, 0, 2119, 2123, 2124, 2118, 0, 2120,
- 2714, 2714, 2119, 0, 2714, 2118, 2121, 2120, 2120, 2122,
- 2119, 0, 2119, 2121, 2121, 2120, 2123, 2122, 0, 2123,
- 2124, 2122, 2121, 0, 0, 2122, 2135, 2123, 2124, 2137,
- 2135, 2124, 0, 2137, 0, 2123, 2124, 0, 2138, 0,
- 2139, 2142, 2138, 0, 2139, 2142, 0, 2141, 0, 0,
-
- 0, 2141, 0, 0, 2135, 0, 0, 2137, 0, 0,
- 2135, 0, 2135, 2142, 0, 2137, 2138, 2138, 2139, 2142,
- 2135, 0, 0, 2137, 2138, 2141, 2139, 2142, 0, 2142,
- 2141, 0, 2138, 2141, 2139, 2142, 2143, 0, 2145, 2146,
- 2143, 2141, 2145, 2146, 0, 2149, 0, 0, 2155, 2149,
- 0, 0, 2155, 0, 0, 0, 2146, 2146, 0, 2152,
- 0, 0, 0, 2152, 2143, 2156, 2145, 2146, 0, 2156,
- 0, 0, 2143, 2149, 2145, 2146, 2155, 0, 0, 2143,
- 2143, 2149, 2145, 2146, 2155, 0, 2145, 2152, 0, 2149,
- 2157, 0, 2155, 2156, 2157, 2152, 2716, 2152, 2718, 0,
-
- 2716, 2156, 2718, 2152, 0, 0, 0, 2156, 0, 2156,
- 0, 0, 0, 2716, 2716, 2718, 2718, 2716, 2157, 2718,
- 0, 0, 0, 0, 0, 0, 2157, 0, 0, 0,
- 0, 0, 0, 0, 2157, 0, 2157, 2158, 2158, 2158,
- 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158,
- 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158,
- 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158,
- 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158,
- 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158,
- 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158,
-
- 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158,
- 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158,
- 2158, 2162, 2163, 2165, 2167, 2162, 2163, 2165, 2167, 0,
- 0, 0, 0, 2169, 0, 0, 0, 2169, 0, 0,
- 0, 0, 2178, 0, 0, 0, 2178, 0, 2165, 2162,
- 2163, 2165, 2167, 0, 0, 2167, 0, 2162, 2163, 2165,
- 2167, 2169, 0, 2163, 2162, 2162, 2163, 2165, 2167, 2169,
- 2178, 2180, 2182, 2178, 0, 2180, 2182, 2169, 2178, 0,
- 0, 2183, 0, 2185, 0, 2183, 2178, 2185, 0, 0,
- 0, 0, 2188, 2187, 2190, 0, 2188, 2187, 2190, 2180,
-
- 2182, 0, 0, 0, 0, 0, 2191, 2180, 2182, 2183,
- 2191, 2185, 0, 0, 2180, 2180, 2182, 2183, 2187, 2185,
- 2188, 2187, 2190, 0, 2183, 2183, 0, 2185, 2188, 2187,
- 2190, 0, 0, 0, 2191, 2188, 2188, 2187, 2190, 2191,
- 2192, 0, 2191, 2193, 2192, 2195, 0, 2193, 0, 2195,
- 2191, 0, 2197, 0, 2199, 0, 2197, 2202, 2199, 0,
- 0, 2202, 0, 2204, 0, 0, 0, 2204, 2192, 2192,
- 2195, 2193, 0, 2195, 2205, 0, 2192, 0, 2205, 2193,
- 2197, 2195, 2199, 2199, 2192, 2202, 0, 2193, 2197, 2195,
- 2199, 2204, 0, 2202, 0, 0, 2197, 0, 2199, 2204,
-
- 2202, 2202, 2205, 0, 2207, 2209, 2204, 2204, 2207, 2209,
- 2205, 0, 0, 0, 0, 2210, 2213, 2205, 2205, 2210,
- 2213, 0, 2215, 0, 0, 0, 2215, 0, 0, 2207,
- 0, 0, 2207, 2209, 0, 2214, 0, 0, 2209, 2214,
- 2207, 2209, 2210, 2210, 2213, 0, 0, 0, 2207, 2209,
- 2215, 2210, 2213, 0, 0, 0, 0, 2214, 2215, 2210,
- 2213, 2217, 0, 2214, 2219, 2217, 2215, 0, 2219, 0,
- 0, 2214, 0, 2221, 2222, 0, 2214, 2221, 2222, 2214,
- 0, 0, 0, 2224, 2223, 0, 2239, 2224, 2223, 2217,
- 2239, 0, 2219, 0, 0, 0, 0, 2217, 2219, 2223,
-
- 2219, 2221, 2222, 2222, 0, 2217, 2221, 0, 2219, 2221,
- 2222, 2224, 2223, 0, 2239, 0, 0, 2221, 2222, 2224,
- 2223, 2225, 2239, 0, 0, 2225, 0, 2224, 2223, 0,
- 2239, 2242, 0, 2244, 0, 2242, 2225, 2244, 2245, 0,
- 2246, 0, 2245, 0, 2246, 0, 0, 0, 0, 2225,
- 2247, 0, 0, 0, 2247, 0, 0, 2225, 0, 2242,
- 2242, 2244, 0, 2245, 0, 2225, 2245, 2242, 2246, 2244,
- 0, 2244, 0, 0, 2245, 2242, 2246, 2244, 2247, 2246,
- 0, 2248, 2245, 0, 2246, 2248, 2247, 2249, 2247, 0,
- 0, 2249, 2257, 0, 2247, 0, 2257, 0, 0, 2259,
-
- 0, 0, 2260, 2259, 0, 0, 2260, 2249, 0, 2248,
- 0, 0, 2263, 0, 2248, 2249, 2263, 2248, 2249, 0,
- 2257, 0, 0, 2249, 0, 2248, 2257, 2259, 2257, 2260,
- 2260, 2249, 2259, 2266, 0, 2259, 2257, 2266, 2260, 0,
- 2263, 0, 0, 2259, 0, 2267, 2260, 2270, 2263, 2267,
- 2273, 2270, 0, 0, 2273, 0, 2263, 0, 2266, 2275,
- 2267, 2266, 2276, 2275, 0, 0, 2276, 0, 2279, 2266,
- 0, 0, 2279, 2267, 0, 2270, 0, 2266, 2273, 0,
- 2270, 2267, 0, 2270, 0, 0, 2273, 2275, 2280, 2267,
- 2276, 2270, 2280, 2273, 2273, 2275, 2279, 2279, 2276, 0,
-
- 0, 0, 0, 2275, 2279, 2276, 2276, 2281, 0, 0,
- 2283, 2281, 2279, 2284, 2283, 0, 2280, 2284, 0, 0,
- 0, 0, 2286, 0, 2280, 2291, 2286, 0, 0, 2291,
- 0, 0, 2280, 0, 0, 2281, 0, 2280, 2283, 0,
- 0, 2284, 0, 2281, 2292, 0, 2283, 2283, 2292, 2284,
- 2286, 2281, 0, 2291, 2283, 2293, 0, 2284, 2286, 2293,
- 2286, 2291, 0, 0, 0, 0, 2286, 2294, 2291, 2291,
- 0, 2294, 2292, 0, 0, 0, 0, 2292, 0, 0,
- 2292, 2295, 2293, 2293, 0, 2295, 2296, 0, 2292, 0,
- 2296, 2293, 2297, 2298, 0, 2294, 2297, 2298, 2299, 2293,
-
- 0, 2296, 2299, 2294, 2294, 0, 0, 0, 0, 2295,
- 2295, 2294, 2300, 0, 2296, 2310, 2300, 2295, 0, 2310,
- 2297, 2298, 2296, 0, 0, 2295, 2299, 2297, 2297, 2298,
- 2296, 0, 0, 0, 2299, 2312, 2297, 2298, 0, 2312,
- 2300, 2300, 2299, 2310, 0, 2313, 2314, 0, 2300, 2313,
- 2314, 2310, 0, 0, 0, 0, 2300, 2316, 2315, 2310,
- 2319, 2316, 2315, 2312, 2319, 0, 0, 0, 0, 0,
- 0, 2312, 2323, 2313, 2314, 0, 2323, 0, 0, 2312,
- 2315, 2313, 2314, 2324, 0, 2316, 2315, 2324, 2319, 2313,
- 2314, 2319, 0, 2316, 2315, 0, 2319, 0, 0, 2315,
-
- 2323, 2316, 2315, 2328, 2319, 2330, 2332, 2328, 2323, 2330,
- 2332, 2324, 0, 0, 2323, 0, 2323, 2337, 2339, 2324,
- 2341, 2337, 2339, 0, 2341, 0, 0, 2324, 2328, 0,
- 0, 2328, 2330, 2330, 2332, 0, 0, 0, 0, 2328,
- 2343, 2330, 2332, 0, 2343, 2337, 2339, 2328, 2341, 2330,
- 2332, 2344, 0, 2337, 2339, 2344, 2341, 0, 0, 0,
- 0, 2337, 2339, 2346, 2341, 2347, 2348, 2346, 2343, 2347,
- 2348, 0, 0, 0, 0, 0, 2343, 0, 0, 2344,
- 0, 0, 0, 2343, 2343, 0, 0, 2344, 2349, 0,
- 0, 2346, 2349, 2347, 2348, 2344, 0, 0, 0, 2346,
-
- 2351, 2347, 2348, 0, 2351, 0, 0, 2346, 0, 2347,
- 2348, 2352, 0, 2349, 2347, 2352, 2349, 2355, 2358, 0,
- 0, 2355, 2358, 2360, 2349, 0, 0, 2360, 2351, 2361,
- 0, 0, 2349, 2361, 0, 0, 2351, 0, 2365, 2352,
- 0, 0, 2365, 0, 2351, 2355, 2358, 2352, 2352, 0,
- 0, 2360, 2363, 2355, 2358, 2352, 2363, 2361, 0, 2360,
- 0, 2355, 2358, 0, 2366, 2361, 2365, 2360, 2366, 0,
- 0, 0, 0, 2361, 2365, 2365, 0, 2369, 0, 2363,
- 2363, 2369, 2365, 0, 0, 0, 2371, 2370, 2363, 2373,
- 2371, 2370, 2366, 2373, 0, 0, 2363, 0, 2366, 2374,
-
- 2366, 0, 0, 2374, 0, 2369, 2369, 0, 2366, 2370,
- 0, 0, 2375, 2369, 2371, 2370, 2375, 2373, 0, 0,
- 0, 2369, 2371, 2370, 2376, 2373, 0, 2374, 2376, 0,
- 2371, 2370, 0, 2373, 2375, 2374, 2373, 2387, 2389, 0,
- 2375, 2387, 2389, 2374, 0, 0, 2376, 2390, 2375, 2391,
- 2393, 2390, 2376, 2391, 2393, 0, 2375, 0, 2375, 2392,
- 2376, 0, 0, 2392, 0, 2387, 2389, 0, 2376, 0,
- 2376, 0, 0, 2387, 2389, 2390, 0, 2391, 2393, 0,
- 2390, 2387, 2389, 2390, 2389, 2391, 2393, 2392, 2391, 0,
- 0, 2390, 2394, 2391, 2393, 2392, 2394, 2393, 2398, 0,
-
- 2392, 2400, 2398, 2392, 2404, 2400, 0, 2401, 2404, 0,
- 0, 2401, 2394, 0, 0, 0, 0, 0, 2405, 2406,
- 2394, 2408, 2405, 2406, 0, 2408, 2398, 0, 2394, 2400,
- 0, 0, 2404, 0, 2398, 2401, 2394, 2400, 2400, 2404,
- 2404, 2401, 2398, 2401, 2406, 2400, 2405, 2406, 2404, 2408,
- 0, 2401, 2410, 0, 2405, 2406, 2410, 2408, 0, 0,
- 2405, 0, 2405, 2406, 2412, 2408, 2413, 0, 2412, 2414,
- 2413, 2415, 0, 2414, 0, 2415, 0, 0, 0, 0,
- 2410, 0, 0, 0, 0, 0, 2419, 0, 2410, 2418,
- 2419, 0, 2412, 2418, 2413, 2413, 2410, 2414, 2414, 2415,
-
- 2412, 0, 2413, 0, 0, 2414, 2415, 2415, 2412, 0,
- 2413, 2412, 2420, 2414, 2419, 2415, 2420, 2418, 2418, 0,
- 0, 0, 2419, 2421, 2422, 2418, 0, 2421, 2422, 0,
- 2419, 2419, 0, 2418, 2423, 0, 0, 0, 2423, 0,
- 2420, 0, 0, 0, 0, 2424, 0, 0, 2420, 2424,
- 0, 2421, 2422, 0, 2425, 2420, 2420, 2436, 2425, 2421,
- 2422, 2436, 2423, 0, 0, 2422, 2421, 2421, 2422, 0,
- 2423, 2423, 2426, 2424, 2424, 2438, 2426, 0, 2423, 2438,
- 2425, 2424, 2425, 0, 2439, 2436, 0, 2436, 2439, 2424,
- 2425, 0, 0, 2436, 0, 2441, 0, 0, 2425, 2441,
-
- 2426, 2436, 0, 2438, 2438, 2442, 2439, 0, 2426, 2442,
- 2446, 2438, 2439, 2426, 2446, 0, 2426, 2447, 0, 2438,
- 2439, 2447, 2449, 2441, 0, 0, 2449, 0, 2439, 0,
- 0, 2441, 2452, 2442, 0, 0, 2452, 2446, 2446, 2441,
- 0, 2442, 0, 0, 2454, 2447, 2446, 2449, 2454, 2442,
- 2449, 2447, 0, 2447, 2446, 2455, 0, 2452, 2449, 2455,
- 2452, 2447, 0, 0, 0, 2456, 2449, 2457, 2452, 2456,
- 2459, 2457, 2454, 2454, 2459, 0, 2452, 0, 0, 0,
- 2454, 0, 2455, 2455, 0, 0, 0, 0, 2454, 0,
- 0, 2455, 2461, 2456, 2456, 2457, 2461, 2457, 2459, 2455,
-
- 0, 2456, 0, 2457, 2462, 0, 2459, 0, 2462, 2456,
- 0, 2457, 0, 2459, 2459, 2467, 0, 2469, 2461, 2467,
- 2461, 2469, 2472, 0, 0, 0, 2472, 2474, 2461, 2462,
- 0, 2474, 2462, 0, 2475, 0, 2461, 2478, 2475, 0,
- 2462, 2478, 0, 2467, 0, 2469, 0, 0, 2462, 0,
- 2472, 2467, 2467, 2469, 2472, 2474, 2474, 0, 2472, 2467,
- 2479, 2469, 2475, 2474, 2479, 2478, 2472, 0, 0, 0,
- 2475, 2474, 2480, 2478, 0, 2488, 2480, 0, 2475, 2488,
- 0, 2478, 2479, 0, 0, 2490, 0, 2491, 2479, 2490,
- 0, 2491, 0, 0, 2493, 0, 2479, 2492, 2493, 0,
-
- 2480, 2492, 2479, 2488, 2479, 0, 2497, 0, 2480, 0,
- 2497, 2488, 0, 2490, 2490, 2491, 2480, 0, 0, 2488,
- 2501, 2490, 2493, 2491, 2501, 2492, 2492, 0, 2491, 2490,
- 2493, 2491, 2498, 2492, 2497, 2497, 2498, 0, 2493, 2502,
- 0, 2492, 2497, 2502, 0, 2503, 0, 0, 2501, 2503,
- 2497, 2505, 0, 2501, 2498, 2505, 2501, 2508, 2509, 0,
- 2498, 2508, 2509, 0, 2501, 0, 0, 2502, 2498, 0,
- 2510, 2503, 0, 2503, 2510, 2502, 2498, 0, 0, 2505,
- 2511, 2503, 2502, 2502, 2511, 2508, 2509, 2505, 0, 2503,
- 0, 0, 2514, 2508, 2509, 2505, 2514, 0, 2510, 2516,
-
- 2508, 2508, 2509, 2516, 0, 2515, 2510, 0, 2511, 2515,
- 0, 2517, 0, 0, 2510, 2517, 2511, 2519, 0, 0,
- 2514, 2519, 2518, 0, 2511, 0, 2518, 2516, 2514, 0,
- 2515, 0, 2516, 2515, 2520, 2516, 2514, 2518, 2520, 2517,
- 0, 2515, 0, 2516, 2517, 2519, 0, 2517, 2521, 2515,
- 2518, 0, 2521, 2519, 0, 2517, 0, 0, 2518, 2522,
- 2533, 2519, 2520, 2522, 2533, 2531, 2518, 0, 0, 2531,
- 2520, 2520, 0, 0, 2522, 0, 2521, 0, 2520, 0,
- 0, 2521, 0, 0, 2521, 2534, 2536, 2522, 2533, 2534,
- 2536, 0, 2521, 2531, 0, 2522, 2533, 0, 2539, 0,
-
- 2531, 2531, 2539, 2522, 2533, 0, 0, 2534, 2540, 2531,
- 0, 2542, 2540, 2534, 2536, 2542, 0, 0, 0, 0,
- 2544, 2534, 2536, 0, 2544, 2536, 2539, 2534, 0, 2534,
- 2536, 2546, 2539, 2547, 2539, 2546, 2540, 2547, 0, 2542,
- 0, 0, 2539, 0, 2540, 2548, 0, 2542, 2544, 2548,
- 2549, 0, 2540, 0, 2549, 2542, 2544, 0, 0, 2546,
- 0, 2547, 2551, 0, 2544, 0, 2551, 2546, 2553, 2547,
- 0, 0, 2553, 2548, 0, 2546, 2547, 2547, 2549, 2554,
- 2559, 2548, 0, 2554, 2559, 2549, 2549, 0, 0, 2548,
- 2551, 2563, 0, 2561, 2549, 2563, 2553, 2561, 2551, 0,
-
- 0, 2553, 0, 2551, 2553, 0, 2551, 2554, 2559, 2559,
- 2564, 0, 2553, 2568, 2564, 2554, 2559, 2568, 0, 2563,
- 0, 2561, 0, 2554, 2559, 0, 2561, 2563, 2579, 2561,
- 2570, 0, 2579, 0, 2570, 2563, 0, 2561, 2564, 2580,
- 2582, 2568, 0, 2580, 2582, 0, 2564, 0, 0, 2568,
- 2568, 2581, 2570, 0, 2564, 2581, 2579, 2568, 2570, 0,
- 0, 2585, 0, 0, 2579, 2585, 2570, 2580, 2582, 2579,
- 0, 0, 2579, 0, 2570, 2580, 2582, 0, 0, 2581,
- 0, 2580, 2582, 2580, 2582, 0, 2586, 2581, 0, 2585,
- 2586, 0, 2581, 0, 0, 2581, 2590, 2585, 2591, 2594,
-
- 2590, 0, 2591, 2594, 0, 2585, 0, 0, 2586, 0,
- 0, 0, 0, 2592, 2586, 0, 2595, 2592, 0, 2594,
- 2595, 0, 2586, 0, 2590, 0, 2591, 2594, 0, 0,
- 2586, 0, 2590, 2597, 2591, 2594, 0, 2597, 0, 2591,
- 2590, 2592, 2591, 2594, 2595, 2590, 2598, 0, 2592, 2592,
- 2598, 2599, 2595, 0, 0, 2599, 0, 2592, 0, 0,
- 2595, 2597, 0, 2600, 0, 0, 0, 2600, 2597, 2597,
- 0, 0, 0, 0, 2598, 0, 2601, 2597, 0, 2599,
- 2601, 2602, 2598, 0, 2598, 2602, 0, 2599, 0, 0,
- 2598, 2600, 0, 0, 2599, 2599, 0, 0, 2603, 2600,
-
- 0, 0, 2603, 0, 2601, 0, 0, 2600, 0, 2602,
- 2604, 2601, 2601, 0, 2604, 2613, 0, 2602, 0, 2613,
- 2601, 0, 0, 0, 2602, 2602, 2603, 2615, 0, 2603,
- 0, 2615, 0, 0, 2603, 0, 0, 2620, 2604, 0,
- 2613, 2620, 2603, 2613, 2617, 0, 2604, 0, 2617, 2622,
- 0, 2613, 0, 2622, 2604, 2615, 2623, 0, 2624, 2613,
- 2623, 0, 2624, 2615, 2615, 2620, 2617, 0, 0, 0,
- 2626, 2615, 2617, 2620, 2626, 0, 2620, 2622, 0, 0,
- 2617, 2620, 0, 2624, 2623, 2622, 2624, 2628, 2617, 0,
- 0, 2628, 2623, 2622, 2624, 0, 0, 0, 2626, 0,
-
- 2623, 2629, 2624, 2626, 2634, 2629, 2626, 2636, 2634, 0,
- 0, 2636, 0, 2638, 2626, 2628, 0, 2638, 2628, 0,
- 0, 0, 2650, 2628, 2640, 0, 2650, 2629, 2640, 2629,
- 0, 2628, 2634, 0, 2651, 2636, 0, 2629, 2651, 0,
- 2634, 2638, 0, 2636, 2636, 2629, 0, 2634, 2634, 2638,
- 2650, 2636, 2640, 2640, 0, 2652, 0, 2638, 2650, 2652,
- 2640, 2653, 2651, 0, 0, 2653, 2650, 2655, 2640, 0,
- 2651, 2655, 2657, 0, 0, 0, 2657, 0, 2651, 0,
- 0, 2658, 2661, 2652, 2659, 2658, 2661, 0, 2659, 2653,
- 0, 2652, 0, 0, 2663, 2655, 0, 2653, 2663, 2652,
-
- 2657, 2657, 2661, 2655, 0, 2653, 0, 0, 2657, 2658,
- 2661, 2655, 2659, 2659, 2658, 2664, 2657, 2658, 2661, 2664,
- 2659, 0, 2663, 0, 0, 2658, 2661, 2665, 2659, 2666,
- 2663, 2665, 2667, 2666, 0, 0, 2667, 0, 2663, 0,
- 0, 0, 0, 2664, 2664, 0, 2668, 0, 0, 0,
- 2668, 2664, 2669, 0, 0, 2665, 2669, 2666, 2670, 2664,
- 2667, 0, 2670, 2665, 2665, 2666, 0, 0, 2667, 2667,
- 0, 2665, 0, 2666, 2668, 2671, 2667, 2666, 0, 2671,
- 2669, 2668, 2668, 0, 0, 2669, 2670, 2680, 2669, 0,
- 2668, 2680, 0, 2682, 2670, 0, 2669, 2682, 0, 0,
-
- 0, 0, 2670, 2671, 2684, 0, 2686, 0, 2684, 0,
- 2686, 2671, 2671, 2688, 0, 2680, 0, 2688, 2680, 2671,
- 0, 2682, 2682, 2680, 2690, 0, 2692, 0, 2690, 2682,
- 2692, 2680, 2684, 2684, 2686, 0, 0, 2682, 0, 2693,
- 2684, 2688, 2686, 2693, 2688, 0, 0, 0, 2684, 2688,
- 2686, 2696, 2690, 0, 2692, 2696, 0, 2688, 0, 0,
- 2690, 2698, 2692, 0, 0, 2698, 0, 2693, 2690, 0,
- 2692, 2690, 2693, 2700, 2713, 2693, 2703, 2700, 2713, 2696,
- 2703, 0, 0, 2693, 0, 0, 0, 2696, 0, 2698,
- 2698, 2713, 2713, 0, 2720, 2696, 2715, 2698, 2720, 2723,
-
- 2715, 2700, 2713, 2723, 2703, 2698, 0, 0, 0, 2700,
- 2713, 2717, 2703, 2715, 2715, 2717, 2720, 2700, 2713, 2703,
- 2703, 2724, 2720, 0, 2715, 2724, 0, 2723, 2717, 2717,
- 2720, 0, 2715, 2725, 0, 2723, 0, 2725, 2720, 2717,
- 2715, 0, 2728, 2723, 2730, 0, 2728, 2717, 2730, 2724,
- 0, 0, 0, 0, 0, 2717, 0, 2724, 2724, 2731,
- 0, 2725, 0, 2731, 2732, 2724, 0, 0, 2732, 2725,
- 2728, 0, 2730, 0, 0, 2725, 0, 2725, 2728, 2733,
- 2730, 2730, 0, 2733, 2731, 0, 2728, 2731, 2730, 2734,
- 0, 0, 2732, 2734, 0, 2731, 0, 2732, 2735, 0,
-
- 2732, 2744, 2735, 2731, 2733, 2744, 0, 2733, 2732, 0,
- 2746, 0, 0, 0, 2746, 2733, 2744, 2734, 0, 0,
- 2748, 0, 0, 2733, 2748, 2734, 2735, 2734, 0, 2744,
- 0, 2735, 2750, 2734, 2735, 0, 2750, 2744, 2746, 0,
- 0, 0, 2735, 0, 0, 2744, 2746, 0, 2748, 0,
- 0, 0, 0, 2746, 2746, 0, 2748, 0, 0, 0,
- 2750, 2748, 0, 0, 2748, 0, 0, 0, 2750, 0,
- 0, 0, 0, 0, 2750, 0, 2750, 2752, 2752, 2752,
- 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752,
- 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752,
-
- 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752,
- 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752,
- 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752,
- 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752,
- 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752,
- 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752, 2752,
- 2752, 2754, 2755, 0, 2758, 2754, 2755, 2760, 2758, 0,
- 0, 2760, 0, 2763, 0, 0, 0, 2763, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 2754,
- 2755, 0, 2758, 2755, 0, 2760, 0, 2754, 2755, 2754,
-
- 2758, 2763, 0, 2760, 0, 2754, 2755, 0, 2758, 2763,
- 2772, 2760, 2774, 0, 2772, 2776, 2774, 2763, 0, 2776,
- 0, 2782, 2779, 0, 0, 2782, 2779, 2772, 2772, 2774,
- 2774, 0, 2776, 2776, 0, 2783, 0, 0, 2772, 2783,
- 2774, 2786, 0, 2776, 2779, 2786, 2772, 0, 2774, 2782,
- 2779, 2776, 2782, 0, 2772, 0, 2774, 2782, 2779, 2776,
- 2787, 0, 0, 2783, 2787, 2782, 2779, 0, 0, 2786,
- 2788, 2783, 2789, 0, 2788, 0, 2789, 2786, 0, 2783,
- 0, 0, 2790, 2791, 2786, 2786, 2790, 2791, 2787, 2798,
- 0, 0, 0, 2798, 0, 2787, 2787, 0, 2788, 0,
-
- 2789, 2800, 0, 0, 2787, 2800, 2788, 2789, 2789, 0,
- 2790, 2791, 0, 0, 2788, 2788, 2789, 2798, 2790, 2791,
- 2802, 0, 0, 0, 2802, 2798, 2790, 2791, 2791, 2800,
- 2804, 2806, 2809, 2798, 2804, 2806, 2809, 2800, 0, 0,
- 0, 0, 2820, 2821, 0, 2800, 2820, 2821, 2802, 2824,
- 0, 0, 0, 2824, 0, 0, 2802, 0, 2804, 2806,
- 2809, 0, 0, 0, 2802, 0, 2804, 2806, 2809, 2833,
- 2820, 2821, 0, 2833, 2804, 2806, 2809, 2824, 2820, 2821,
- 0, 0, 0, 0, 2833, 2824, 2820, 2821, 2834, 0,
- 0, 2835, 2834, 2824, 2836, 2835, 0, 2833, 2836, 0,
-
- 0, 0, 0, 2843, 0, 2833, 2845, 2843, 0, 0,
- 2845, 0, 0, 2833, 0, 0, 2834, 2834, 0, 2835,
- 0, 2851, 2836, 2836, 2834, 2851, 0, 2835, 0, 0,
- 2836, 2843, 2834, 2835, 2845, 2835, 2843, 0, 2836, 2843,
- 2852, 2859, 2845, 0, 2852, 2859, 0, 2843, 0, 2851,
- 2845, 2860, 2861, 0, 0, 2860, 2861, 2851, 0, 0,
- 0, 0, 2867, 2851, 0, 2851, 2867, 0, 2852, 2859,
- 0, 0, 0, 0, 0, 0, 2852, 2859, 2852, 2860,
- 2861, 0, 2865, 2859, 2852, 2859, 2865, 2860, 2861, 2871,
- 2867, 0, 2860, 2871, 2861, 2860, 2861, 0, 2867, 2872,
-
- 0, 0, 0, 2872, 2873, 0, 2867, 2865, 2873, 2876,
- 2865, 0, 0, 2876, 0, 0, 2878, 2871, 2865, 2882,
- 2878, 0, 2871, 2882, 0, 2871, 2865, 2872, 0, 2873,
- 0, 0, 2873, 2871, 2883, 2872, 2876, 2876, 2883, 0,
- 2873, 2872, 0, 2872, 2878, 2876, 2884, 2882, 2873, 0,
- 2884, 0, 2878, 2876, 0, 2882, 0, 2887, 2878, 2883,
- 2878, 2887, 2883, 2882, 0, 0, 0, 2891, 2882, 2892,
- 2883, 2891, 0, 2892, 2884, 0, 0, 0, 2883, 2895,
- 0, 2884, 2884, 2895, 0, 2887, 0, 0, 2900, 0,
- 2884, 0, 2900, 2887, 0, 2891, 2891, 2892, 2887, 0,
-
- 0, 2887, 2903, 2891, 2892, 2892, 2903, 2895, 0, 0,
- 0, 2891, 2895, 2892, 2906, 2895, 2900, 2909, 2906, 0,
- 0, 2909, 0, 2895, 2900, 0, 0, 0, 0, 0,
- 2903, 2900, 2900, 0, 0, 2903, 0, 0, 2903, 0,
- 0, 2906, 2906, 0, 0, 2909, 2903, 0, 0, 0,
- 2906, 0, 0, 2909, 0, 0, 0, 0, 2906, 2909,
- 0, 2909, 2913, 2913, 2913, 2913, 2913, 2913, 2913, 2913,
- 2913, 2913, 2913, 2913, 2913, 2914, 2914, 2914, 2914, 2914,
- 2914, 2914, 2914, 2914, 2914, 2914, 2914, 2914, 2915, 2915,
- 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
-
- 2915, 2916, 2916, 2916, 2916, 2916, 2916, 2916, 2916, 2916,
- 2916, 2916, 2916, 2916, 2917, 2917, 2917, 2917, 2917, 2917,
- 2917, 2917, 2917, 2917, 2917, 2917, 2917, 2918, 2918, 2918,
+ 1738, 1738, 1738, 1738, 1738, 1738, 1738, 1738, 1782, 2255,
+ 0, 2240, 1782, 2255, 0, 2240, 0, 2242, 1738, 1738,
+ 2284, 2242, 2287, 2322, 2284, 0, 2287, 2322, 0, 2255,
+ 1782, 2240, 2240, 2240, 0, 0, 1782, 2242, 2242, 2242,
+ 2255, 2284, 0, 2287, 1782, 2322, 0, 0, 0, 0,
+ 0, 0, 1782, 1784, 1784, 1784, 1784, 1784, 1784, 1784,
+ 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784,
+ 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784,
+
+ 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784,
+ 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784,
+ 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784,
+ 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784,
+ 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1784,
+ 1784, 1784, 1784, 1784, 1784, 1784, 1784, 1787, 2323, 0,
+ 1795, 1787, 2323, 2338, 1795, 0, 0, 2338, 0, 1797,
+ 2383, 0, 1798, 1797, 2383, 0, 1798, 2471, 0, 1787,
+ 2323, 2471, 1800, 0, 2338, 1787, 1800, 0, 1795, 0,
+ 0, 1803, 2383, 1787, 1798, 1803, 1795, 1797, 2471, 0,
+
+ 1798, 1787, 1789, 1789, 1795, 1797, 1789, 1789, 1798, 1789,
+ 1800, 2389, 0, 1797, 2471, 2389, 1798, 2440, 1800, 1803,
+ 2461, 2440, 0, 1789, 2461, 0, 1800, 1803, 2389, 2389,
+ 0, 0, 2473, 2389, 1803, 1803, 2473, 0, 2440, 0,
+ 0, 2461, 0, 0, 0, 0, 0, 2440, 0, 0,
+ 2461, 0, 0, 2473, 1789, 1789, 1789, 1789, 1789, 1789,
+ 1789, 1789, 1789, 1789, 1789, 1789, 1789, 1789, 1789, 1789,
+ 1789, 1789, 1789, 1789, 1789, 1789, 1789, 1789, 1789, 1789,
+ 1789, 1789, 1789, 1789, 1789, 1801, 1804, 1805, 0, 1801,
+ 1804, 1805, 0, 0, 0, 0, 1806, 2446, 1807, 1808,
+
+ 1806, 2446, 1807, 1808, 0, 0, 0, 2509, 1825, 1805,
+ 1801, 2509, 1825, 1801, 1804, 1805, 1805, 0, 2446, 1804,
+ 0, 1801, 1804, 1805, 1806, 1806, 1807, 1808, 2509, 1801,
+ 1804, 1805, 1806, 1809, 1807, 1808, 1825, 1809, 0, 1807,
+ 1806, 2446, 1807, 1808, 1825, 1827, 0, 1828, 0, 1827,
+ 1834, 1828, 1825, 0, 1834, 1809, 0, 0, 0, 1830,
+ 1836, 1809, 1809, 1830, 1836, 1827, 0, 0, 0, 1809,
+ 1832, 0, 1837, 1827, 1832, 1828, 1837, 1809, 1834, 0,
+ 1836, 1827, 0, 1828, 0, 1832, 1834, 1830, 1836, 1827,
+ 1828, 1828, 1830, 1838, 1834, 1830, 1836, 1838, 1832, 0,
+
+ 1837, 0, 0, 1830, 1836, 1847, 1832, 1849, 1837, 1847,
+ 0, 1849, 0, 0, 1832, 1837, 1837, 0, 1852, 1850,
+ 0, 1838, 1852, 1850, 0, 0, 0, 0, 1838, 1838,
+ 1854, 0, 0, 1847, 1854, 1849, 1849, 1838, 2382, 0,
+ 0, 1847, 2382, 1849, 1850, 0, 1852, 1850, 1847, 1847,
+ 0, 1849, 1855, 1858, 1852, 1850, 1855, 1858, 1854, 1854,
+ 2382, 0, 1852, 1850, 1860, 2388, 1854, 0, 1860, 2388,
+ 0, 0, 2486, 1863, 1854, 1858, 2486, 1863, 1864, 0,
+ 1855, 1858, 1864, 0, 2382, 0, 0, 2388, 1855, 1858,
+ 0, 0, 1860, 0, 2486, 1855, 1855, 1858, 2503, 2538,
+
+ 1860, 1863, 2503, 2538, 0, 0, 1864, 1860, 1860, 1863,
+ 2486, 2388, 0, 0, 1864, 1863, 0, 1863, 0, 0,
+ 2503, 2538, 1864, 1865, 1865, 1865, 1865, 1865, 1865, 1865,
+ 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865,
+ 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865,
+ 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865,
+ 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865,
+ 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865,
+ 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865,
+ 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1865,
+
+ 1865, 1865, 1865, 1865, 1865, 1865, 1865, 1868, 0, 1874,
+ 1876, 1868, 1877, 1874, 1876, 0, 1877, 0, 0, 1879,
+ 2568, 2690, 1880, 1879, 2568, 2690, 1880, 0, 0, 1881,
+ 0, 0, 1868, 1881, 0, 1868, 0, 1874, 1876, 1876,
+ 1877, 2568, 2690, 1868, 0, 1874, 1876, 1879, 1877, 1877,
+ 1880, 1868, 1874, 1874, 1876, 1879, 1877, 1881, 1880, 0,
+ 1885, 0, 0, 1879, 1885, 1881, 1880, 1886, 1881, 0,
+ 1887, 1886, 1888, 1881, 1887, 0, 1888, 1889, 0, 1892,
+ 0, 1889, 1886, 1892, 0, 0, 0, 2574, 1885, 1893,
+ 1894, 2574, 0, 1893, 1894, 1886, 1885, 2490, 1887, 0,
+
+ 1888, 2490, 1888, 1886, 1885, 1889, 1887, 1892, 1888, 2574,
+ 0, 1886, 1887, 1889, 1887, 1892, 1888, 1893, 1894, 2490,
+ 1895, 1889, 1892, 1892, 1895, 1893, 1894, 1896, 2574, 0,
+ 1894, 1896, 1897, 1893, 1894, 2490, 1897, 1898, 1899, 0,
+ 2590, 1898, 1899, 0, 2590, 0, 0, 2592, 1895, 1900,
+ 2621, 2592, 0, 1900, 2621, 1896, 1895, 0, 0, 0,
+ 1897, 2590, 2590, 1896, 1895, 1898, 1899, 1899, 1897, 2592,
+ 1897, 1896, 2621, 1898, 1899, 1898, 1897, 1900, 0, 1909,
+ 1913, 1898, 1899, 1909, 1913, 1900, 1900, 0, 1900, 0,
+ 1911, 1914, 2572, 1900, 1911, 1914, 2572, 1915, 0, 2704,
+
+ 2619, 1915, 1921, 2704, 2619, 0, 1921, 1909, 1913, 1913,
+ 0, 0, 0, 2572, 0, 1909, 1913, 1911, 1911, 1914,
+ 2704, 2619, 0, 1909, 1913, 1915, 1911, 1914, 1914, 2572,
+ 1921, 1921, 1918, 1915, 1911, 1914, 1918, 2619, 1921, 1920,
+ 1915, 1915, 0, 1920, 1923, 0, 1921, 2622, 1923, 1918,
+ 1918, 2622, 0, 0, 0, 0, 0, 0, 1929, 0,
+ 1918, 1920, 1929, 0, 0, 0, 0, 1920, 1918, 2622,
+ 1934, 0, 1923, 1927, 1934, 1920, 1918, 1927, 2724, 0,
+ 1923, 1920, 2724, 1920, 0, 1920, 1929, 1923, 1923, 2725,
+ 1927, 1927, 2665, 2725, 1929, 1927, 2665, 0, 1934, 2724,
+
+ 2724, 1927, 1929, 0, 0, 1934, 1934, 0, 0, 1927,
+ 0, 2725, 2665, 2665, 1934, 0, 0, 1927, 1930, 1930,
+ 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930,
+ 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930,
+ 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930,
+ 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930,
+ 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930,
+ 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930,
+ 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930,
+ 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930, 1930,
+
+ 1930, 1930, 1937, 0, 1939, 1940, 1937, 1941, 1939, 1940,
+ 0, 1941, 0, 2732, 2645, 0, 2764, 2732, 2645, 1943,
+ 2764, 0, 2708, 1943, 1942, 0, 2708, 0, 1942, 0,
+ 1937, 0, 1939, 1940, 2732, 1941, 2645, 2764, 1937, 1939,
+ 1939, 1940, 2645, 1941, 2708, 1941, 1937, 1943, 1939, 1940,
+ 1947, 1941, 1942, 1942, 1947, 1943, 0, 1948, 1943, 0,
+ 1942, 1948, 1950, 1943, 2708, 0, 1950, 2756, 1942, 1952,
+ 2768, 2756, 0, 1952, 2768, 1954, 0, 0, 1947, 1954,
+ 2774, 0, 0, 0, 2774, 1948, 1947, 2652, 0, 2756,
+ 1950, 2652, 2768, 1948, 1947, 1950, 2783, 1952, 1950, 1948,
+
+ 2783, 1948, 2774, 1954, 0, 1952, 1950, 0, 1954, 2652,
+ 0, 1954, 1952, 1952, 0, 2652, 0, 2783, 2783, 1954,
+ 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956,
+ 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956,
+ 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956,
+ 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956,
+ 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956,
+ 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956,
+ 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956,
+ 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956, 1956,
+
+ 1956, 1956, 1956, 1956, 1957, 1959, 1960, 2784, 1957, 1959,
+ 1960, 2784, 0, 0, 2715, 1962, 2581, 1963, 2715, 1962,
+ 2581, 1963, 0, 0, 1965, 0, 1966, 1968, 1965, 2784,
+ 1966, 1968, 1957, 1959, 1960, 1960, 2715, 1957, 2581, 0,
+ 1957, 1959, 1960, 1962, 1962, 1963, 1963, 0, 1957, 1959,
+ 1960, 1962, 1965, 1963, 1966, 1968, 2715, 2581, 0, 1962,
+ 1965, 1963, 1966, 1968, 1968, 0, 0, 0, 1965, 1966,
+ 1966, 1968, 1969, 0, 1971, 1972, 1969, 1973, 1971, 1972,
+ 0, 1973, 0, 2808, 2817, 0, 1975, 2808, 2817, 1977,
+ 1975, 0, 0, 1977, 2828, 0, 0, 0, 2828, 0,
+
+ 1969, 1969, 1971, 1972, 2808, 1973, 2817, 0, 1969, 0,
+ 1971, 1972, 1971, 1973, 1975, 2828, 1969, 1977, 1971, 1972,
+ 1978, 1973, 1975, 1975, 1978, 1977, 0, 1980, 0, 0,
+ 1975, 1980, 1977, 1977, 1981, 0, 0, 1982, 1981, 1985,
+ 1986, 1982, 0, 1985, 1986, 2849, 1989, 1978, 1978, 2849,
+ 1989, 0, 0, 0, 0, 1980, 1978, 2821, 0, 0,
+ 0, 2821, 1981, 1980, 1978, 1982, 2849, 1985, 1986, 1986,
+ 1981, 1980, 1985, 1982, 1989, 1985, 1986, 0, 1981, 2821,
+ 0, 1982, 1989, 1985, 1986, 1988, 1982, 1991, 0, 1988,
+ 1989, 1991, 1993, 0, 1996, 2851, 1993, 1997, 1996, 2851,
+
+ 0, 1997, 0, 0, 0, 2852, 2001, 1998, 0, 2852,
+ 2001, 1998, 1988, 1988, 0, 1991, 1991, 2851, 0, 0,
+ 1993, 1988, 1996, 1991, 0, 1997, 2852, 1997, 1993, 1988,
+ 1996, 1991, 1998, 1997, 2001, 1998, 1993, 0, 1996, 0,
+ 2003, 1997, 2001, 1998, 2003, 0, 0, 0, 0, 2004,
+ 2001, 1998, 2005, 2004, 2007, 0, 2005, 0, 2007, 0,
+ 0, 2008, 0, 2010, 0, 2008, 2027, 2010, 2003, 0,
+ 2027, 2004, 0, 2003, 0, 2029, 2003, 2004, 0, 2029,
+ 2005, 0, 2007, 2007, 2003, 2004, 0, 2004, 2005, 2008,
+ 2007, 2010, 0, 2004, 2027, 0, 2005, 2008, 2007, 2010,
+
+ 2031, 0, 2027, 2029, 2031, 2008, 0, 2010, 2027, 2033,
+ 2027, 2029, 2034, 2033, 0, 0, 2034, 0, 2036, 2029,
+ 2814, 0, 2036, 0, 2814, 0, 0, 2031, 2031, 2035,
+ 2037, 0, 0, 2035, 2037, 0, 2031, 2033, 0, 2038,
+ 2034, 2814, 2814, 2038, 2031, 2033, 2036, 0, 2034, 2034,
+ 2033, 2035, 0, 2033, 2036, 2037, 2034, 2035, 2037, 2058,
+ 2060, 2038, 2036, 2058, 2060, 2035, 2037, 2038, 0, 0,
+ 2035, 0, 2062, 2035, 2037, 2038, 2062, 2198, 2065, 0,
+ 2038, 2198, 2065, 2038, 0, 0, 0, 2058, 2060, 0,
+ 2067, 0, 0, 0, 2067, 2058, 2060, 2069, 2198, 0,
+
+ 2062, 2069, 2198, 2058, 2060, 0, 2065, 2070, 2062, 2062,
+ 2067, 2070, 0, 2065, 2065, 0, 2062, 2065, 2067, 2079,
+ 0, 2198, 2065, 2079, 2065, 2069, 2067, 2070, 2081, 0,
+ 0, 0, 2081, 2069, 2067, 2070, 0, 0, 2069, 0,
+ 2082, 2069, 0, 2070, 2082, 2070, 2085, 2079, 0, 2084,
+ 2085, 2070, 0, 2084, 0, 2079, 2081, 0, 2088, 0,
+ 0, 2090, 2088, 2079, 2081, 2090, 0, 2082, 2082, 0,
+ 0, 2085, 2081, 0, 2085, 0, 2082, 2084, 0, 2084,
+ 2088, 0, 2085, 2092, 2082, 2084, 2088, 2092, 0, 2090,
+ 2085, 0, 2096, 2084, 2088, 0, 2096, 2090, 0, 0,
+
+ 0, 0, 2088, 2099, 2102, 2090, 2104, 2099, 2102, 2105,
+ 2104, 2092, 0, 2105, 0, 0, 2092, 0, 2107, 2092,
+ 2096, 2108, 2107, 0, 0, 2108, 0, 2092, 2096, 0,
+ 0, 2099, 2102, 0, 2104, 2096, 2096, 2105, 2105, 2099,
+ 2102, 0, 2104, 0, 2099, 2105, 2107, 2099, 2102, 2108,
+ 2104, 2111, 2112, 2105, 2107, 2111, 2112, 2108, 0, 0,
+ 0, 2107, 2107, 2114, 2113, 2108, 0, 2114, 2113, 0,
+ 2117, 0, 0, 2118, 2117, 0, 0, 2118, 0, 2111,
+ 2112, 0, 2119, 0, 0, 2112, 2119, 2111, 2112, 2114,
+ 0, 2114, 2113, 0, 2111, 2111, 2112, 2113, 2117, 2114,
+
+ 2113, 2118, 0, 0, 0, 0, 2117, 2114, 2113, 2118,
+ 2119, 2120, 0, 0, 2117, 2120, 2121, 2118, 2119, 2119,
+ 2121, 2122, 0, 0, 0, 2122, 2119, 0, 2123, 0,
+ 0, 2124, 2123, 2717, 0, 2124, 0, 2717, 0, 2120,
+ 2120, 2125, 2126, 0, 2121, 2125, 2126, 2120, 0, 2122,
+ 2717, 2717, 2121, 0, 2717, 2120, 2123, 2122, 2122, 2124,
+ 2121, 0, 2121, 2123, 2123, 2122, 2125, 2124, 0, 2125,
+ 2126, 2124, 2123, 0, 0, 2124, 2137, 2125, 2126, 2139,
+ 2137, 2126, 0, 2139, 0, 2125, 2126, 0, 2140, 0,
+ 2141, 2144, 2140, 0, 2141, 2144, 0, 2143, 0, 0,
+
+ 0, 2143, 0, 0, 2137, 0, 0, 2139, 0, 0,
+ 2137, 0, 2137, 2144, 0, 2139, 2140, 2140, 2141, 2144,
+ 2137, 0, 0, 2139, 2140, 2143, 2141, 2144, 0, 2144,
+ 2143, 0, 2140, 2143, 2141, 2144, 2145, 0, 2147, 2148,
+ 2145, 2143, 2147, 2148, 0, 2151, 0, 0, 2157, 2151,
+ 0, 0, 2157, 0, 0, 0, 2148, 2148, 0, 2154,
+ 0, 0, 0, 2154, 2145, 2158, 2147, 2148, 0, 2158,
+ 0, 0, 2145, 2151, 2147, 2148, 2157, 0, 0, 2145,
+ 2145, 2151, 2147, 2148, 2157, 0, 2147, 2154, 0, 2151,
+ 2159, 0, 2157, 2158, 2159, 2154, 2719, 2154, 2721, 0,
+
+ 2719, 2158, 2721, 2154, 0, 0, 0, 2158, 0, 2158,
+ 0, 0, 0, 2719, 2719, 2721, 2721, 2719, 2159, 2721,
+ 0, 0, 0, 0, 0, 0, 2159, 0, 0, 0,
+ 0, 0, 0, 0, 2159, 0, 2159, 2160, 2160, 2160,
+ 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160,
+ 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160,
+ 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160,
+ 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160,
+ 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160,
+ 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160,
+
+ 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160,
+ 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160,
+ 2160, 2164, 2165, 2167, 2169, 2164, 2165, 2167, 2169, 0,
+ 0, 0, 0, 2171, 0, 0, 0, 2171, 0, 0,
+ 0, 0, 2180, 0, 0, 0, 2180, 0, 2167, 2164,
+ 2165, 2167, 2169, 0, 0, 2169, 0, 2164, 2165, 2167,
+ 2169, 2171, 0, 2165, 2164, 2164, 2165, 2167, 2169, 2171,
+ 2180, 2182, 2184, 2180, 0, 2182, 2184, 2171, 2180, 0,
+ 0, 2185, 0, 2187, 0, 2185, 2180, 2187, 0, 0,
+ 0, 0, 2190, 2189, 2192, 0, 2190, 2189, 2192, 2182,
+
+ 2184, 0, 0, 0, 0, 0, 2193, 2182, 2184, 2185,
+ 2193, 2187, 0, 0, 2182, 2182, 2184, 2185, 2189, 2187,
+ 2190, 2189, 2192, 0, 2185, 2185, 0, 2187, 2190, 2189,
+ 2192, 0, 0, 0, 2193, 2190, 2190, 2189, 2192, 2193,
+ 2194, 0, 2193, 2195, 2194, 2197, 0, 2195, 0, 2197,
+ 2193, 0, 2199, 0, 2201, 0, 2199, 2204, 2201, 0,
+ 0, 2204, 0, 2206, 0, 0, 0, 2206, 2194, 2194,
+ 2197, 2195, 0, 2197, 2207, 0, 2194, 0, 2207, 2195,
+ 2199, 2197, 2201, 2201, 2194, 2204, 0, 2195, 2199, 2197,
+ 2201, 2206, 0, 2204, 0, 0, 2199, 0, 2201, 2206,
+
+ 2204, 2204, 2207, 0, 2209, 2211, 2206, 2206, 2209, 2211,
+ 2207, 0, 0, 0, 0, 2212, 2215, 2207, 2207, 2212,
+ 2215, 0, 2217, 0, 0, 0, 2217, 0, 0, 2209,
+ 0, 0, 2209, 2211, 0, 2216, 0, 0, 2211, 2216,
+ 2209, 2211, 2212, 2212, 2215, 0, 0, 0, 2209, 2211,
+ 2217, 2212, 2215, 0, 0, 0, 0, 2216, 2217, 2212,
+ 2215, 2219, 0, 2216, 2221, 2219, 2217, 0, 2221, 0,
+ 0, 2216, 0, 2223, 2224, 0, 2216, 2223, 2224, 2216,
+ 0, 0, 0, 2226, 2225, 0, 2241, 2226, 2225, 2219,
+ 2241, 0, 2221, 0, 0, 0, 0, 2219, 2221, 2225,
+
+ 2221, 2223, 2224, 2224, 0, 2219, 2223, 0, 2221, 2223,
+ 2224, 2226, 2225, 0, 2241, 0, 0, 2223, 2224, 2226,
+ 2225, 2227, 2241, 0, 0, 2227, 0, 2226, 2225, 0,
+ 2241, 2244, 0, 2246, 0, 2244, 2227, 2246, 2247, 0,
+ 2248, 0, 2247, 0, 2248, 0, 0, 0, 0, 2227,
+ 2249, 0, 0, 0, 2249, 0, 0, 2227, 0, 2244,
+ 2244, 2246, 0, 2247, 0, 2227, 2247, 2244, 2248, 2246,
+ 0, 2246, 0, 0, 2247, 2244, 2248, 2246, 2249, 2248,
+ 0, 2250, 2247, 0, 2248, 2250, 2249, 2251, 2249, 0,
+ 0, 2251, 2259, 0, 2249, 0, 2259, 0, 0, 2261,
+
+ 0, 0, 2262, 2261, 0, 0, 2262, 2251, 0, 2250,
+ 0, 0, 2265, 0, 2250, 2251, 2265, 2250, 2251, 0,
+ 2259, 0, 0, 2251, 0, 2250, 2259, 2261, 2259, 2262,
+ 2262, 2251, 2261, 2268, 0, 2261, 2259, 2268, 2262, 0,
+ 2265, 0, 0, 2261, 0, 2269, 2262, 2272, 2265, 2269,
+ 2275, 2272, 0, 0, 2275, 0, 2265, 0, 2268, 2277,
+ 2269, 2268, 2278, 2277, 0, 0, 2278, 0, 2281, 2268,
+ 0, 0, 2281, 2269, 0, 2272, 0, 2268, 2275, 0,
+ 2272, 2269, 0, 2272, 0, 0, 2275, 2277, 2282, 2269,
+ 2278, 2272, 2282, 2275, 2275, 2277, 2281, 2281, 2278, 0,
+
+ 0, 0, 0, 2277, 2281, 2278, 2278, 2283, 0, 0,
+ 2285, 2283, 2281, 2286, 2285, 0, 2282, 2286, 0, 0,
+ 0, 0, 2288, 0, 2282, 2293, 2288, 0, 0, 2293,
+ 0, 0, 2282, 0, 0, 2283, 0, 2282, 2285, 0,
+ 0, 2286, 0, 2283, 2294, 0, 2285, 2285, 2294, 2286,
+ 2288, 2283, 0, 2293, 2285, 2295, 0, 2286, 2288, 2295,
+ 2288, 2293, 0, 0, 0, 0, 2288, 2296, 2293, 2293,
+ 0, 2296, 2294, 0, 0, 0, 0, 2294, 0, 0,
+ 2294, 2297, 2295, 2295, 0, 2297, 2298, 0, 2294, 0,
+ 2298, 2295, 2299, 2300, 0, 2296, 2299, 2300, 2301, 2295,
+
+ 0, 2298, 2301, 2296, 2296, 0, 0, 0, 0, 2297,
+ 2297, 2296, 2302, 0, 2298, 2312, 2302, 2297, 0, 2312,
+ 2299, 2300, 2298, 0, 0, 2297, 2301, 2299, 2299, 2300,
+ 2298, 0, 0, 0, 2301, 2314, 2299, 2300, 0, 2314,
+ 2302, 2302, 2301, 2312, 0, 2315, 2316, 0, 2302, 2315,
+ 2316, 2312, 0, 0, 0, 0, 2302, 2318, 2317, 2312,
+ 2321, 2318, 2317, 2314, 2321, 0, 0, 0, 0, 0,
+ 0, 2314, 2325, 2315, 2316, 0, 2325, 0, 0, 2314,
+ 2317, 2315, 2316, 2326, 0, 2318, 2317, 2326, 2321, 2315,
+ 2316, 2321, 0, 2318, 2317, 0, 2321, 0, 0, 2317,
+
+ 2325, 2318, 2317, 2330, 2321, 2332, 2334, 2330, 2325, 2332,
+ 2334, 2326, 0, 0, 2325, 0, 2325, 2339, 2341, 2326,
+ 2343, 2339, 2341, 0, 2343, 0, 0, 2326, 2330, 0,
+ 0, 2330, 2332, 2332, 2334, 0, 0, 0, 0, 2330,
+ 2345, 2332, 2334, 0, 2345, 2339, 2341, 2330, 2343, 2332,
+ 2334, 2346, 0, 2339, 2341, 2346, 2343, 0, 0, 0,
+ 0, 2339, 2341, 2348, 2343, 2349, 2350, 2348, 2345, 2349,
+ 2350, 0, 0, 0, 0, 0, 2345, 0, 0, 2346,
+ 0, 0, 0, 2345, 2345, 0, 0, 2346, 2351, 0,
+ 0, 2348, 2351, 2349, 2350, 2346, 0, 0, 0, 2348,
+
+ 2353, 2349, 2350, 0, 2353, 0, 0, 2348, 0, 2349,
+ 2350, 2354, 0, 2351, 2349, 2354, 2351, 2357, 2360, 0,
+ 0, 2357, 2360, 2362, 2351, 0, 0, 2362, 2353, 2363,
+ 0, 0, 2351, 2363, 0, 0, 2353, 0, 2367, 2354,
+ 0, 0, 2367, 0, 2353, 2357, 2360, 2354, 2354, 0,
+ 0, 2362, 2365, 2357, 2360, 2354, 2365, 2363, 0, 2362,
+ 0, 2357, 2360, 0, 2368, 2363, 2367, 2362, 2368, 0,
+ 0, 0, 0, 2363, 2367, 2367, 0, 2371, 0, 2365,
+ 2365, 2371, 2367, 0, 0, 0, 2373, 2372, 2365, 2375,
+ 2373, 2372, 2368, 2375, 0, 0, 2365, 0, 2368, 2376,
+
+ 2368, 0, 0, 2376, 0, 2371, 2371, 0, 2368, 2372,
+ 0, 0, 2377, 2371, 2373, 2372, 2377, 2375, 0, 0,
+ 0, 2371, 2373, 2372, 2378, 2375, 0, 2376, 2378, 0,
+ 2373, 2372, 0, 2375, 2377, 2376, 2375, 2390, 2392, 0,
+ 2377, 2390, 2392, 2376, 0, 0, 2378, 2393, 2377, 2394,
+ 2396, 2393, 2378, 2394, 2396, 0, 2377, 0, 2377, 2395,
+ 2378, 0, 0, 2395, 0, 2390, 2392, 0, 2378, 0,
+ 2378, 0, 0, 2390, 2392, 2393, 0, 2394, 2396, 0,
+ 2393, 2390, 2392, 2393, 2392, 2394, 2396, 2395, 2394, 0,
+ 0, 2393, 2397, 2394, 2396, 2395, 2397, 2396, 2401, 0,
+
+ 2395, 2403, 2401, 2395, 2407, 2403, 0, 2404, 2407, 0,
+ 0, 2404, 2397, 0, 0, 0, 0, 0, 2408, 2409,
+ 2397, 2411, 2408, 2409, 0, 2411, 2401, 0, 2397, 2403,
+ 0, 0, 2407, 0, 2401, 2404, 2397, 2403, 2403, 2407,
+ 2407, 2404, 2401, 2404, 2409, 2403, 2408, 2409, 2407, 2411,
+ 0, 2404, 2413, 0, 2408, 2409, 2413, 2411, 0, 0,
+ 2408, 0, 2408, 2409, 2415, 2411, 2416, 0, 2415, 2417,
+ 2416, 2418, 0, 2417, 0, 2418, 0, 0, 0, 0,
+ 2413, 0, 0, 0, 0, 0, 2422, 0, 2413, 2421,
+ 2422, 0, 2415, 2421, 2416, 2416, 2413, 2417, 2417, 2418,
+
+ 2415, 0, 2416, 0, 0, 2417, 2418, 2418, 2415, 0,
+ 2416, 2415, 2423, 2417, 2422, 2418, 2423, 2421, 2421, 0,
+ 0, 0, 2422, 2424, 2425, 2421, 0, 2424, 2425, 0,
+ 2422, 2422, 0, 2421, 2426, 0, 0, 0, 2426, 0,
+ 2423, 0, 0, 0, 0, 2427, 0, 0, 2423, 2427,
+ 0, 2424, 2425, 0, 2428, 2423, 2423, 2439, 2428, 2424,
+ 2425, 2439, 2426, 0, 0, 2425, 2424, 2424, 2425, 0,
+ 2426, 2426, 2429, 2427, 2427, 2441, 2429, 0, 2426, 2441,
+ 2428, 2427, 2428, 0, 2442, 2439, 0, 2439, 2442, 2427,
+ 2428, 0, 0, 2439, 0, 2444, 0, 0, 2428, 2444,
+
+ 2429, 2439, 0, 2441, 2441, 2445, 2442, 0, 2429, 2445,
+ 2449, 2441, 2442, 2429, 2449, 0, 2429, 2450, 0, 2441,
+ 2442, 2450, 2452, 2444, 0, 0, 2452, 0, 2442, 0,
+ 0, 2444, 2455, 2445, 0, 0, 2455, 2449, 2449, 2444,
+ 0, 2445, 0, 0, 2457, 2450, 2449, 2452, 2457, 2445,
+ 2452, 2450, 0, 2450, 2449, 2458, 0, 2455, 2452, 2458,
+ 2455, 2450, 0, 0, 0, 2459, 2452, 2460, 2455, 2459,
+ 2462, 2460, 2457, 2457, 2462, 0, 2455, 0, 0, 0,
+ 2457, 0, 2458, 2458, 0, 0, 0, 0, 2457, 0,
+ 0, 2458, 2464, 2459, 2459, 2460, 2464, 2460, 2462, 2458,
+
+ 0, 2459, 0, 2460, 2465, 0, 2462, 0, 2465, 2459,
+ 0, 2460, 0, 2462, 2462, 2470, 0, 2472, 2464, 2470,
+ 2464, 2472, 2475, 0, 0, 0, 2475, 2477, 2464, 2465,
+ 0, 2477, 2465, 0, 2478, 0, 2464, 2481, 2478, 0,
+ 2465, 2481, 0, 2470, 0, 2472, 0, 0, 2465, 0,
+ 2475, 2470, 2470, 2472, 2475, 2477, 2477, 0, 2475, 2470,
+ 2482, 2472, 2478, 2477, 2482, 2481, 2475, 0, 0, 0,
+ 2478, 2477, 2483, 2481, 0, 2491, 2483, 0, 2478, 2491,
+ 0, 2481, 2482, 0, 0, 2493, 0, 2494, 2482, 2493,
+ 0, 2494, 0, 0, 2496, 0, 2482, 2495, 2496, 0,
+
+ 2483, 2495, 2482, 2491, 2482, 0, 2500, 0, 2483, 0,
+ 2500, 2491, 0, 2493, 2493, 2494, 2483, 0, 0, 2491,
+ 2504, 2493, 2496, 2494, 2504, 2495, 2495, 0, 2494, 2493,
+ 2496, 2494, 2501, 2495, 2500, 2500, 2501, 0, 2496, 2505,
+ 0, 2495, 2500, 2505, 0, 2506, 0, 0, 2504, 2506,
+ 2500, 2508, 0, 2504, 2501, 2508, 2504, 2511, 2512, 0,
+ 2501, 2511, 2512, 0, 2504, 0, 0, 2505, 2501, 0,
+ 2513, 2506, 0, 2506, 2513, 2505, 2501, 0, 0, 2508,
+ 2514, 2506, 2505, 2505, 2514, 2511, 2512, 2508, 0, 2506,
+ 0, 0, 2517, 2511, 2512, 2508, 2517, 0, 2513, 2519,
+
+ 2511, 2511, 2512, 2519, 0, 2518, 2513, 0, 2514, 2518,
+ 0, 2520, 0, 0, 2513, 2520, 2514, 2522, 0, 0,
+ 2517, 2522, 2521, 0, 2514, 0, 2521, 2519, 2517, 0,
+ 2518, 0, 2519, 2518, 2523, 2519, 2517, 2521, 2523, 2520,
+ 0, 2518, 0, 2519, 2520, 2522, 0, 2520, 2524, 2518,
+ 2521, 0, 2524, 2522, 0, 2520, 0, 0, 2521, 2525,
+ 2536, 2522, 2523, 2525, 2536, 2534, 2521, 0, 0, 2534,
+ 2523, 2523, 0, 0, 2525, 0, 2524, 0, 2523, 0,
+ 0, 2524, 0, 0, 2524, 2537, 2539, 2525, 2536, 2537,
+ 2539, 0, 2524, 2534, 0, 2525, 2536, 0, 2542, 0,
+
+ 2534, 2534, 2542, 2525, 2536, 0, 0, 2537, 2543, 2534,
+ 0, 2545, 2543, 2537, 2539, 2545, 0, 0, 0, 0,
+ 2547, 2537, 2539, 0, 2547, 2539, 2542, 2537, 0, 2537,
+ 2539, 2549, 2542, 2550, 2542, 2549, 2543, 2550, 0, 2545,
+ 0, 0, 2542, 0, 2543, 2551, 0, 2545, 2547, 2551,
+ 2552, 0, 2543, 0, 2552, 2545, 2547, 0, 0, 2549,
+ 0, 2550, 2554, 0, 2547, 0, 2554, 2549, 2556, 2550,
+ 0, 0, 2556, 2551, 0, 2549, 2550, 2550, 2552, 2557,
+ 2562, 2551, 0, 2557, 2562, 2552, 2552, 0, 0, 2551,
+ 2554, 2566, 0, 2564, 2552, 2566, 2556, 2564, 2554, 0,
+
+ 0, 2556, 0, 2554, 2556, 0, 2554, 2557, 2562, 2562,
+ 2567, 0, 2556, 2571, 2567, 2557, 2562, 2571, 0, 2566,
+ 0, 2564, 0, 2557, 2562, 0, 2564, 2566, 2582, 2564,
+ 2573, 0, 2582, 0, 2573, 2566, 0, 2564, 2567, 2583,
+ 2585, 2571, 0, 2583, 2585, 0, 2567, 0, 0, 2571,
+ 2571, 2584, 2573, 0, 2567, 2584, 2582, 2571, 2573, 0,
+ 0, 2588, 0, 0, 2582, 2588, 2573, 2583, 2585, 2582,
+ 0, 0, 2582, 0, 2573, 2583, 2585, 0, 0, 2584,
+ 0, 2583, 2585, 2583, 2585, 0, 2589, 2584, 0, 2588,
+ 2589, 0, 2584, 0, 0, 2584, 2593, 2588, 2594, 2597,
+
+ 2593, 0, 2594, 2597, 0, 2588, 0, 0, 2589, 0,
+ 0, 0, 0, 2595, 2589, 0, 2598, 2595, 0, 2597,
+ 2598, 0, 2589, 0, 2593, 0, 2594, 2597, 0, 0,
+ 2589, 0, 2593, 2600, 2594, 2597, 0, 2600, 0, 2594,
+ 2593, 2595, 2594, 2597, 2598, 2593, 2601, 0, 2595, 2595,
+ 2601, 2602, 2598, 0, 0, 2602, 0, 2595, 0, 0,
+ 2598, 2600, 0, 2603, 0, 0, 0, 2603, 2600, 2600,
+ 0, 0, 0, 0, 2601, 0, 2604, 2600, 0, 2602,
+ 2604, 2605, 2601, 0, 2601, 2605, 0, 2602, 0, 0,
+ 2601, 2603, 0, 0, 2602, 2602, 0, 0, 2606, 2603,
+
+ 0, 0, 2606, 0, 2604, 0, 0, 2603, 0, 2605,
+ 2607, 2604, 2604, 0, 2607, 2616, 0, 2605, 0, 2616,
+ 2604, 0, 0, 0, 2605, 2605, 2606, 2618, 0, 2606,
+ 0, 2618, 0, 0, 2606, 0, 0, 2623, 2607, 0,
+ 2616, 2623, 2606, 2616, 2620, 0, 2607, 0, 2620, 2625,
+ 0, 2616, 0, 2625, 2607, 2618, 2626, 0, 2627, 2616,
+ 2626, 0, 2627, 2618, 2618, 2623, 2620, 0, 0, 0,
+ 2629, 2618, 2620, 2623, 2629, 0, 2623, 2625, 0, 0,
+ 2620, 2623, 0, 2627, 2626, 2625, 2627, 2631, 2620, 0,
+ 0, 2631, 2626, 2625, 2627, 0, 0, 0, 2629, 0,
+
+ 2626, 2632, 2627, 2629, 2637, 2632, 2629, 2639, 2637, 0,
+ 0, 2639, 0, 2641, 2629, 2631, 0, 2641, 2631, 0,
+ 0, 0, 2653, 2631, 2643, 0, 2653, 2632, 2643, 2632,
+ 0, 2631, 2637, 0, 2654, 2639, 0, 2632, 2654, 0,
+ 2637, 2641, 0, 2639, 2639, 2632, 0, 2637, 2637, 2641,
+ 2653, 2639, 2643, 2643, 0, 2655, 0, 2641, 2653, 2655,
+ 2643, 2656, 2654, 0, 0, 2656, 2653, 2658, 2643, 0,
+ 2654, 2658, 2660, 0, 0, 0, 2660, 0, 2654, 0,
+ 0, 2661, 2664, 2655, 2662, 2661, 2664, 0, 2662, 2656,
+ 0, 2655, 0, 0, 2666, 2658, 0, 2656, 2666, 2655,
+
+ 2660, 2660, 2664, 2658, 0, 2656, 0, 0, 2660, 2661,
+ 2664, 2658, 2662, 2662, 2661, 2667, 2660, 2661, 2664, 2667,
+ 2662, 0, 2666, 0, 0, 2661, 2664, 2668, 2662, 2669,
+ 2666, 2668, 2670, 2669, 0, 0, 2670, 0, 2666, 0,
+ 0, 0, 0, 2667, 2667, 0, 2671, 0, 0, 0,
+ 2671, 2667, 2672, 0, 0, 2668, 2672, 2669, 2673, 2667,
+ 2670, 0, 2673, 2668, 2668, 2669, 0, 0, 2670, 2670,
+ 0, 2668, 0, 2669, 2671, 2674, 2670, 2669, 0, 2674,
+ 2672, 2671, 2671, 0, 0, 2672, 2673, 2683, 2672, 0,
+ 2671, 2683, 0, 2685, 2673, 0, 2672, 2685, 0, 0,
+
+ 0, 0, 2673, 2674, 2687, 0, 2689, 0, 2687, 0,
+ 2689, 2674, 2674, 2691, 0, 2683, 0, 2691, 2683, 2674,
+ 0, 2685, 2685, 2683, 2693, 0, 2695, 0, 2693, 2685,
+ 2695, 2683, 2687, 2687, 2689, 0, 0, 2685, 0, 2696,
+ 2687, 2691, 2689, 2696, 2691, 0, 0, 0, 2687, 2691,
+ 2689, 2699, 2693, 0, 2695, 2699, 0, 2691, 0, 0,
+ 2693, 2701, 2695, 0, 0, 2701, 0, 2696, 2693, 0,
+ 2695, 2693, 2696, 2703, 2716, 2696, 2706, 2703, 2716, 2699,
+ 2706, 0, 0, 2696, 0, 0, 0, 2699, 0, 2701,
+ 2701, 2716, 2716, 0, 2723, 2699, 2718, 2701, 2723, 2726,
+
+ 2718, 2703, 2716, 2726, 2706, 2701, 0, 0, 0, 2703,
+ 2716, 2720, 2706, 2718, 2718, 2720, 2723, 2703, 2716, 2706,
+ 2706, 2727, 2723, 0, 2718, 2727, 0, 2726, 2720, 2720,
+ 2723, 0, 2718, 2728, 0, 2726, 0, 2728, 2723, 2720,
+ 2718, 0, 2731, 2726, 2733, 0, 2731, 2720, 2733, 2727,
+ 0, 0, 0, 0, 0, 2720, 0, 2727, 2727, 2734,
+ 0, 2728, 0, 2734, 2735, 2727, 0, 0, 2735, 2728,
+ 2731, 0, 2733, 0, 0, 2728, 0, 2728, 2731, 2736,
+ 2733, 2733, 0, 2736, 2734, 0, 2731, 2734, 2733, 2737,
+ 0, 0, 2735, 2737, 0, 2734, 0, 2735, 2738, 0,
+
+ 2735, 2747, 2738, 2734, 2736, 2747, 0, 2736, 2735, 0,
+ 2749, 0, 0, 0, 2749, 2736, 2747, 2737, 0, 0,
+ 2751, 0, 0, 2736, 2751, 2737, 2738, 2737, 0, 2747,
+ 0, 2738, 2753, 2737, 2738, 0, 2753, 2747, 2749, 0,
+ 0, 0, 2738, 0, 0, 2747, 2749, 0, 2751, 0,
+ 0, 0, 0, 2749, 2749, 0, 2751, 0, 0, 0,
+ 2753, 2751, 0, 0, 2751, 0, 0, 0, 2753, 0,
+ 0, 0, 0, 0, 2753, 0, 2753, 2755, 2755, 2755,
+ 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755,
+ 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755,
+
+ 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755,
+ 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755,
+ 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755,
+ 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755,
+ 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755,
+ 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755, 2755,
+ 2755, 2757, 2758, 0, 2761, 2757, 2758, 2763, 2761, 0,
+ 0, 2763, 0, 2766, 0, 0, 0, 2766, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 2757,
+ 2758, 0, 2761, 2758, 0, 2763, 0, 2757, 2758, 2757,
+
+ 2761, 2766, 0, 2763, 0, 2757, 2758, 0, 2761, 2766,
+ 2775, 2763, 2777, 0, 2775, 2779, 2777, 2766, 0, 2779,
+ 0, 2785, 2782, 0, 0, 2785, 2782, 2775, 2775, 2777,
+ 2777, 0, 2779, 2779, 0, 2786, 0, 0, 2775, 2786,
+ 2777, 2789, 0, 2779, 2782, 2789, 2775, 0, 2777, 2785,
+ 2782, 2779, 2785, 0, 2775, 0, 2777, 2785, 2782, 2779,
+ 2790, 0, 0, 2786, 2790, 2785, 2782, 0, 0, 2789,
+ 2791, 2786, 2792, 0, 2791, 0, 2792, 2789, 0, 2786,
+ 0, 0, 2793, 2794, 2789, 2789, 2793, 2794, 2790, 2801,
+ 0, 0, 0, 2801, 0, 2790, 2790, 0, 2791, 0,
+
+ 2792, 2803, 0, 0, 2790, 2803, 2791, 2792, 2792, 0,
+ 2793, 2794, 0, 0, 2791, 2791, 2792, 2801, 2793, 2794,
+ 2805, 0, 0, 0, 2805, 2801, 2793, 2794, 2794, 2803,
+ 2807, 2809, 2812, 2801, 2807, 2809, 2812, 2803, 0, 0,
+ 0, 0, 2823, 2824, 0, 2803, 2823, 2824, 2805, 2827,
+ 0, 0, 0, 2827, 0, 0, 2805, 0, 2807, 2809,
+ 2812, 0, 0, 0, 2805, 0, 2807, 2809, 2812, 2836,
+ 2823, 2824, 0, 2836, 2807, 2809, 2812, 2827, 2823, 2824,
+ 0, 0, 0, 0, 2836, 2827, 2823, 2824, 2837, 0,
+ 0, 2838, 2837, 2827, 2839, 2838, 0, 2836, 2839, 0,
+
+ 0, 0, 0, 2846, 0, 2836, 2848, 2846, 0, 0,
+ 2848, 0, 0, 2836, 0, 0, 2837, 2837, 0, 2838,
+ 0, 2854, 2839, 2839, 2837, 2854, 0, 2838, 0, 0,
+ 2839, 2846, 2837, 2838, 2848, 2838, 2846, 0, 2839, 2846,
+ 2855, 2862, 2848, 0, 2855, 2862, 0, 2846, 0, 2854,
+ 2848, 2863, 2864, 0, 0, 2863, 2864, 2854, 0, 0,
+ 0, 0, 2870, 2854, 0, 2854, 2870, 0, 2855, 2862,
+ 0, 0, 0, 0, 0, 0, 2855, 2862, 2855, 2863,
+ 2864, 0, 2868, 2862, 2855, 2862, 2868, 2863, 2864, 2874,
+ 2870, 0, 2863, 2874, 2864, 2863, 2864, 0, 2870, 2875,
+
+ 0, 0, 0, 2875, 2876, 0, 2870, 2868, 2876, 2879,
+ 2868, 0, 0, 2879, 0, 0, 2881, 2874, 2868, 2885,
+ 2881, 0, 2874, 2885, 0, 2874, 2868, 2875, 0, 2876,
+ 0, 0, 2876, 2874, 2886, 2875, 2879, 2879, 2886, 0,
+ 2876, 2875, 0, 2875, 2881, 2879, 2887, 2885, 2876, 0,
+ 2887, 0, 2881, 2879, 0, 2885, 0, 2890, 2881, 2886,
+ 2881, 2890, 2886, 2885, 0, 0, 0, 2894, 2885, 2895,
+ 2886, 2894, 0, 2895, 2887, 0, 0, 0, 2886, 2898,
+ 0, 2887, 2887, 2898, 0, 2890, 0, 0, 2903, 0,
+ 2887, 0, 2903, 2890, 0, 2894, 2894, 2895, 2890, 0,
+
+ 0, 2890, 2906, 2894, 2895, 2895, 2906, 2898, 0, 0,
+ 0, 2894, 2898, 2895, 2909, 2898, 2903, 2912, 2909, 0,
+ 0, 2912, 0, 2898, 2903, 0, 0, 0, 0, 0,
+ 2906, 2903, 2903, 0, 0, 2906, 0, 0, 2906, 0,
+ 0, 2909, 2909, 0, 0, 2912, 2906, 0, 0, 0,
+ 2909, 0, 0, 2912, 0, 0, 0, 0, 2909, 2912,
+ 0, 2912, 2916, 2916, 2916, 2916, 2916, 2916, 2916, 2916,
+ 2916, 2916, 2916, 2916, 2916, 2917, 2917, 2917, 2917, 2917,
+ 2917, 2917, 2917, 2917, 2917, 2917, 2917, 2917, 2918, 2918,
2918, 2918, 2918, 2918, 2918, 2918, 2918, 2918, 2918, 2918,
- 2919, 2919, 2919, 2919, 2919, 2919, 2919, 2919, 2919, 2919,
- 2919, 2919, 2919, 2920, 2920, 2920, 2920, 2920, 2920, 2920,
- 2920, 2920, 2920, 2920, 2920, 2920, 2921, 2921, 2921, 2921,
- 2921, 2921, 2921, 2921, 2921, 2921, 2921, 2921, 2921, 2922,
- 2922, 2922, 2922, 2922, 2922, 2922, 2922, 2922, 2922, 2922,
- 2922, 2922, 2923, 2923, 2923, 2923, 2923, 2923, 2923, 2923,
- 2923, 2923, 2923, 2923, 2923, 2924, 2924, 2924, 2924, 2924,
- 2924, 2924, 2924, 2924, 2924, 2924, 2924, 2924, 2925, 2925,
+ 2918, 2919, 2919, 2919, 2919, 2919, 2919, 2919, 2919, 2919,
+ 2919, 2919, 2919, 2919, 2920, 2920, 2920, 2920, 2920, 2920,
+ 2920, 2920, 2920, 2920, 2920, 2920, 2920, 2921, 2921, 2921,
+ 2921, 2921, 2921, 2921, 2921, 2921, 2921, 2921, 2921, 2921,
+ 2922, 2922, 2922, 2922, 2922, 2922, 2922, 2922, 2922, 2922,
+ 2922, 2922, 2922, 2923, 2923, 2923, 2923, 2923, 2923, 2923,
+ 2923, 2923, 2923, 2923, 2923, 2923, 2924, 2924, 2924, 2924,
+ 2924, 2924, 2924, 2924, 2924, 2924, 2924, 2924, 2924, 2925,
2925, 2925, 2925, 2925, 2925, 2925, 2925, 2925, 2925, 2925,
- 2925, 2926, 2926, 0, 2926, 2926, 2926, 2926, 2926, 2926,
- 2926, 2926, 2926, 2926, 2927, 2927, 2927, 2927, 2927, 0,
- 2927, 2927, 2927, 2927, 2928, 2928, 0, 2928, 2928, 2928,
- 2928, 2928, 2928, 2928, 2928, 2928, 2928, 2929, 2929, 0,
- 2929, 2929, 2929, 2929, 2929, 2929, 2929, 2929, 2929, 2929,
- 2930, 2930, 2930, 2930, 2930, 2930, 2930, 2930, 2930, 2930,
- 2931, 2931, 0, 2931, 2931, 2931, 2931, 2931, 2931, 2931,
-
- 2931, 2931, 2931, 2932, 2932, 0, 2932, 2932, 2932, 2932,
- 2932, 2932, 2932, 2932, 2932, 2932, 2933, 2933, 0, 2933,
- 2933, 2933, 2933, 2933, 2933, 2933, 2933, 2933, 2933, 2934,
- 0, 2934, 2934, 0, 2934, 2934, 2934, 2934, 2934, 2934,
- 2934, 2934, 2935, 2935, 0, 0, 2935, 2935, 2935, 2935,
- 2935, 2935, 2936, 2936, 0, 2936, 2936, 2936, 2936, 2936,
- 2936, 2936, 2936, 2936, 2936, 2937, 2937, 0, 2937, 2937,
- 2937, 2937, 2937, 2937, 2937, 2937, 2937, 2937, 2938, 2938,
- 0, 0, 2938, 2938, 2938, 2938, 2938, 2938, 2939, 2939,
- 0, 2939, 2939, 2939, 2939, 2939, 2939, 2939, 2939, 2939,
-
- 2939, 2940, 2940, 0, 2940, 2940, 2940, 2940, 2940, 2940,
- 2940, 2940, 2940, 2940, 2941, 2941, 2941, 2941, 2941, 2941,
- 2941, 2941, 0, 2941, 2941, 2941, 2941, 2942, 2942, 0,
- 0, 2942, 2942, 2942, 2942, 2942, 2942, 2943, 2943, 2943,
- 2943, 2943, 2943, 2943, 2943, 2943, 2943, 2943, 2943, 2943,
- 2944, 2944, 2944, 2944, 2944, 2944, 2944, 2944, 2944, 2944,
- 2944, 2944, 2944, 2945, 2945, 0, 0, 2945, 2945, 2945,
- 2945, 2945, 2945, 2946, 2946, 0, 2946, 2946, 2946, 2946,
- 2946, 2946, 2946, 2946, 2946, 2946, 2947, 2947, 0, 2947,
- 2947, 2947, 2947, 2947, 2947, 2947, 2947, 2947, 2947, 2948,
-
- 2948, 0, 0, 2948, 2948, 2948, 2948, 2948, 2948, 2949,
- 2949, 0, 2949, 2949, 2949, 2949, 2949, 2949, 2949, 2949,
- 2949, 2949, 2950, 2950, 0, 2950, 2950, 2950, 2950, 2950,
- 2950, 2950, 2950, 2950, 2950, 2951, 2951, 0, 0, 2951,
- 2951, 2951, 2951, 2951, 2951, 2952, 2952, 0, 2952, 2952,
- 2952, 2952, 2952, 2952, 2952, 2952, 2952, 2952, 2953, 2953,
- 0, 2953, 2953, 2953, 2953, 2953, 2953, 2953, 2953, 2953,
- 2953, 2954, 2954, 0, 0, 2954, 2954, 2954, 2954, 2954,
- 2954, 2955, 2955, 0, 2955, 2955, 2955, 2955, 2955, 2955,
- 2955, 2955, 2955, 2955, 2956, 2956, 0, 2956, 2956, 2956,
-
- 2956, 2956, 2956, 2956, 2956, 2956, 2956, 2957, 2957, 0,
- 0, 2957, 2957, 2957, 2957, 2957, 2957, 2958, 2958, 0,
- 2958, 2958, 2958, 2958, 2958, 2958, 2958, 2958, 2958, 2958,
- 2959, 2959, 0, 2959, 2959, 2959, 2959, 2959, 2959, 2959,
- 2959, 2959, 2959, 2960, 2960, 0, 0, 2960, 2960, 2960,
- 2960, 2960, 2960, 2961, 2961, 0, 2961, 2961, 2961, 2961,
- 2961, 2961, 2961, 2961, 2961, 2961, 2962, 2962, 0, 2962,
- 2962, 2962, 2962, 2962, 2962, 2962, 2962, 2962, 2962, 2963,
- 2963, 0, 0, 2963, 2963, 2963, 2963, 2963, 2963, 2964,
- 2964, 0, 2964, 2964, 2964, 2964, 2964, 2964, 2964, 2964,
-
- 2964, 2964, 2965, 2965, 0, 2965, 2965, 2965, 2965, 2965,
- 2965, 2965, 2965, 2965, 2965, 2966, 2966, 0, 0, 2966,
- 2966, 2966, 2966, 2966, 2966, 2967, 2967, 0, 2967, 2967,
- 2967, 2967, 2967, 2967, 2967, 2967, 2967, 2967, 2968, 2968,
- 0, 2968, 2968, 2968, 2968, 2968, 2968, 2968, 2968, 2968,
- 2968, 2969, 2969, 0, 0, 2969, 2969, 2969, 2969, 2969,
- 2969, 2970, 2970, 0, 2970, 2970, 2970, 2970, 2970, 2970,
- 2970, 2970, 2970, 2970, 2971, 2971, 0, 2971, 2971, 2971,
- 2971, 2971, 2971, 2971, 2971, 2971, 2971, 2972, 2972, 0,
- 0, 2972, 2972, 2972, 2972, 2972, 2972, 2973, 2973, 0,
-
- 2973, 2973, 2973, 2973, 2973, 2973, 2973, 2973, 2973, 2973,
- 2974, 2974, 0, 2974, 2974, 2974, 2974, 2974, 2974, 2974,
- 2974, 2974, 2974, 2975, 2975, 0, 0, 2975, 2975, 2975,
- 2975, 2975, 2975, 2976, 2976, 0, 2976, 2976, 2976, 2976,
- 2976, 2976, 2976, 2976, 2976, 2976, 2977, 2977, 0, 2977,
- 2977, 2977, 2977, 2977, 2977, 2977, 2977, 2977, 2977, 2978,
- 2978, 2978, 2978, 0, 2978, 2978, 2978, 2978, 2978, 2978,
- 2978, 2978, 2979, 2979, 0, 2979, 0, 2979, 2979, 2979,
- 2979, 2979, 2979, 2979, 2979, 2980, 2980, 0, 0, 2980,
- 2980, 2980, 2980, 2980, 2980, 2981, 2981, 0, 2981, 2981,
-
- 2981, 2981, 2981, 2981, 2981, 2981, 2981, 2981, 2982, 2982,
- 0, 2982, 2982, 2982, 2982, 2982, 2982, 2982, 2982, 2982,
- 2982, 2983, 2983, 0, 0, 2983, 2983, 2983, 2983, 2983,
- 2983, 2984, 2984, 0, 2984, 2984, 2984, 2984, 2984, 2984,
- 2984, 2984, 2984, 2984, 2985, 2985, 0, 2985, 2985, 2985,
- 2985, 2985, 2985, 2985, 2985, 2985, 2985, 2986, 2986, 0,
- 0, 2986, 2986, 2986, 2986, 2986, 2986, 2987, 2987, 0,
- 2987, 2987, 2987, 2987, 2987, 2987, 2987, 2987, 2987, 2987,
- 2988, 2988, 0, 2988, 2988, 2988, 2988, 2988, 2988, 2988,
- 2988, 2988, 2988, 2989, 2989, 0, 0, 2989, 2989, 2989,
-
- 2989, 2989, 2989, 2990, 2990, 0, 2990, 2990, 2990, 2990,
- 2990, 2990, 2990, 2990, 2990, 2990, 2991, 2991, 0, 2991,
- 2991, 2991, 2991, 2991, 2991, 2991, 2991, 2991, 2991, 2992,
- 2992, 2992, 2992, 2992, 2992, 0, 2992, 2992, 2992, 2992,
- 2992, 2992, 2993, 2993, 0, 0, 2993, 2993, 2993, 2993,
- 2993, 2993, 2994, 2994, 0, 2994, 2994, 2994, 2994, 2994,
- 2994, 2994, 2994, 2994, 2994, 2995, 2995, 0, 2995, 2995,
- 2995, 2995, 2995, 2995, 2995, 2995, 2995, 2995, 2996, 2996,
- 0, 0, 2996, 2996, 2996, 2996, 2996, 2996, 2997, 2997,
- 0, 2997, 2997, 2997, 2997, 2997, 2997, 2997, 2997, 2997,
-
- 2997, 2998, 2998, 0, 2998, 2998, 2998, 2998, 2998, 2998,
- 2998, 2998, 2998, 2998, 2999, 2999, 0, 0, 2999, 2999,
- 2999, 2999, 2999, 2999, 3000, 3000, 0, 3000, 3000, 3000,
- 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3001, 3001, 0,
- 3001, 3001, 3001, 3001, 3001, 3001, 3001, 3001, 3001, 3001,
- 3002, 3002, 0, 0, 3002, 3002, 3002, 3002, 3002, 3002,
- 3003, 3003, 0, 3003, 3003, 3003, 3003, 3003, 3003, 3003,
- 3003, 3003, 3003, 3004, 3004, 0, 3004, 3004, 3004, 3004,
- 3004, 3004, 3004, 3004, 3004, 3004, 3005, 3005, 3005, 3005,
- 3005, 3005, 3005, 3005, 3005, 3005, 3005, 3005, 3005, 3006,
-
- 3006, 3006, 3006, 3006, 3006, 0, 3006, 3006, 3006, 3006,
- 3006, 3006, 3007, 3007, 3007, 3007, 3007, 3007, 3007, 3007,
- 3007, 3007, 3009, 3009, 3009, 3009, 3009, 3009, 3009, 3009,
- 3009, 0, 3009, 3009, 3009, 3010, 3010, 0, 3010, 3010,
- 3010, 3010, 3010, 3010, 3010, 3010, 3010, 3010, 3011, 3011,
- 0, 3011, 3011, 3011, 3011, 3011, 3011, 3011, 3011, 3011,
- 3011, 3012, 3012, 3012, 3012, 3012, 3012, 3012, 3012, 3012,
- 3012, 3012, 0, 3012, 3013, 3013, 0, 3013, 3013, 3013,
- 3013, 3013, 3013, 3013, 3013, 3013, 3013, 3014, 3014, 0,
- 3014, 3014, 3014, 3014, 3014, 3014, 3014, 3014, 3014, 3014,
-
- 3015, 3015, 0, 0, 3015, 3015, 3015, 3015, 3015, 3015,
- 3016, 3016, 0, 3016, 3016, 3016, 3016, 3016, 3016, 3016,
- 3016, 3016, 3016, 3017, 3017, 0, 3017, 3017, 3017, 3017,
- 3017, 3017, 3017, 3017, 3017, 3017, 3018, 3018, 0, 0,
- 3018, 3018, 3018, 3018, 3018, 3018, 3019, 3019, 0, 3019,
- 3019, 3019, 3019, 3019, 3019, 3019, 3019, 3019, 3019, 3020,
- 3020, 0, 3020, 3020, 3020, 3020, 3020, 3020, 3020, 3020,
- 3020, 3020, 3021, 3021, 0, 0, 3021, 3021, 3021, 3021,
- 3021, 3021, 3022, 3022, 0, 3022, 3022, 3022, 3022, 3022,
- 3022, 3022, 3022, 3022, 3022, 3023, 3023, 0, 3023, 3023,
-
- 3023, 3023, 3023, 3023, 3023, 3023, 3023, 3023, 3024, 3024,
- 0, 0, 3024, 3024, 3024, 3024, 3024, 3024, 3025, 3025,
- 0, 3025, 3025, 3025, 3025, 3025, 3025, 3025, 3025, 3025,
- 3025, 3026, 3026, 0, 3026, 3026, 3026, 3026, 3026, 3026,
- 3026, 3026, 3026, 3026, 3027, 0, 3027, 3027, 0, 3027,
- 3027, 3027, 3027, 3027, 3027, 3027, 3027, 3028, 0, 3028,
- 3028, 0, 3028, 3028, 3028, 0, 3028, 3028, 3028, 3028,
- 3029, 3029, 0, 0, 3029, 3029, 3029, 3029, 3029, 3029,
- 3030, 3030, 0, 3030, 3030, 3030, 3030, 3030, 3030, 3030,
- 3030, 3030, 3030, 3031, 3031, 0, 3031, 3031, 3031, 3031,
-
- 3031, 3031, 3031, 3031, 3031, 3031, 3032, 0, 3032, 3032,
- 0, 3032, 3032, 3032, 3032, 3032, 3032, 3032, 3032, 3033,
- 3033, 0, 0, 3033, 3033, 3033, 3033, 3033, 3033, 3034,
- 3034, 0, 3034, 3034, 3034, 3034, 3034, 3034, 3034, 3034,
- 3034, 3034, 3035, 3035, 0, 3035, 3035, 3035, 3035, 3035,
- 3035, 3035, 3035, 3035, 3035, 3036, 0, 3036, 3036, 0,
- 3036, 3036, 3036, 3036, 3036, 3036, 3036, 3036, 3037, 3037,
- 0, 0, 3037, 3037, 3037, 3037, 3037, 3037, 3038, 3038,
- 0, 3038, 3038, 3038, 3038, 3038, 3038, 3038, 3038, 3038,
- 3038, 3039, 3039, 0, 3039, 3039, 3039, 3039, 3039, 3039,
-
- 3039, 3039, 3039, 3039, 3040, 3040, 0, 0, 3040, 3040,
- 3040, 3040, 3040, 3040, 3041, 3041, 0, 3041, 3041, 3041,
- 3041, 3041, 3041, 3041, 3041, 3041, 3041, 3042, 3042, 0,
- 3042, 3042, 3042, 3042, 3042, 3042, 3042, 3042, 3042, 3042,
- 3043, 3043, 0, 0, 3043, 3043, 3043, 3043, 3043, 3043,
- 3044, 3044, 0, 3044, 3044, 3044, 3044, 3044, 3044, 3044,
- 3044, 3044, 3044, 3045, 3045, 0, 3045, 3045, 3045, 3045,
- 3045, 3045, 3045, 3045, 3045, 3045, 3046, 3046, 0, 3046,
- 3046, 3046, 3046, 3046, 3046, 3046, 3046, 3046, 3046, 3047,
- 3047, 0, 3047, 3047, 3047, 3047, 3047, 3047, 3047, 3047,
-
- 3047, 3047, 3048, 3048, 0, 3048, 3048, 3048, 3048, 3048,
- 3048, 3048, 3048, 3048, 3048, 3049, 3049, 0, 0, 3049,
- 3049, 3049, 3049, 3049, 3049, 3050, 3050, 0, 3050, 3050,
- 3050, 3050, 3050, 3050, 3050, 3050, 3050, 3050, 3051, 3051,
- 0, 3051, 3051, 3051, 3051, 3051, 3051, 3051, 3051, 3051,
- 3051, 3052, 3052, 0, 0, 3052, 3052, 3052, 3052, 3052,
- 3052, 3053, 3053, 0, 3053, 3053, 3053, 3053, 3053, 3053,
- 3053, 3053, 3053, 3053, 3054, 3054, 0, 3054, 3054, 3054,
- 3054, 3054, 3054, 3054, 3054, 3054, 3054, 3055, 3055, 0,
- 0, 3055, 3055, 3055, 3055, 3055, 3055, 3056, 3056, 0,
-
- 3056, 3056, 3056, 3056, 3056, 3056, 3056, 3056, 3056, 3056,
- 3057, 3057, 0, 3057, 3057, 3057, 3057, 3057, 3057, 3057,
- 3057, 3057, 3057, 3058, 3058, 0, 3058, 3058, 3058, 3058,
- 3058, 3058, 3058, 3058, 3058, 3058, 3059, 3059, 0, 3059,
- 3059, 3059, 3059, 3059, 3059, 3059, 3059, 3059, 3059, 3060,
- 3060, 0, 3060, 3060, 3060, 3060, 3060, 3060, 3060, 3060,
- 3060, 3060, 3061, 3061, 0, 0, 3061, 3061, 3061, 3061,
- 3061, 3061, 3062, 3062, 0, 3062, 3062, 3062, 3062, 3062,
- 3062, 3062, 3062, 3062, 3062, 3063, 3063, 0, 3063, 3063,
- 3063, 3063, 3063, 3063, 3063, 3063, 3063, 3063, 3064, 3064,
-
- 0, 0, 3064, 3064, 3064, 3064, 3064, 3064, 3065, 3065,
- 0, 3065, 3065, 3065, 3065, 3065, 3065, 3065, 3065, 3065,
- 3065, 3066, 3066, 0, 3066, 3066, 3066, 3066, 3066, 3066,
- 3066, 3066, 3066, 3066, 3067, 3067, 0, 0, 3067, 3067,
- 3067, 3067, 3067, 3067, 3068, 3068, 0, 3068, 3068, 3068,
- 3068, 3068, 3068, 3068, 3068, 3068, 3068, 3069, 3069, 0,
- 3069, 3069, 3069, 3069, 3069, 3069, 3069, 3069, 3069, 3069,
- 3070, 3070, 0, 0, 3070, 3070, 3070, 3070, 3070, 3070,
- 3071, 3071, 0, 3071, 3071, 3071, 3071, 3071, 3071, 3071,
- 3071, 3071, 3071, 3072, 3072, 0, 3072, 3072, 3072, 3072,
-
- 3072, 3072, 3072, 3072, 3072, 3072, 3073, 3073, 0, 3073,
- 3073, 3073, 3073, 3073, 3073, 3073, 3073, 3073, 3073, 3074,
- 3074, 3074, 3074, 3074, 0, 3074, 3074, 3074, 3074, 3075,
- 3075, 0, 3075, 3075, 3075, 3075, 3075, 3075, 3075, 3075,
- 3075, 3075, 3076, 3076, 0, 3076, 3076, 3076, 3076, 3076,
- 3076, 3076, 3076, 3076, 3076, 3077, 3077, 0, 3077, 3077,
- 3077, 3077, 3077, 3077, 3077, 3077, 3077, 3077, 3078, 3078,
- 0, 3078, 3078, 3078, 3078, 3078, 3078, 3078, 3078, 3078,
- 3078, 3079, 3079, 0, 3079, 3079, 3079, 3079, 3079, 3079,
- 3079, 3079, 3079, 3079, 3080, 3080, 0, 3080, 3080, 3080,
-
- 3080, 3080, 3080, 3080, 3080, 3080, 3080, 3081, 3081, 0,
- 3081, 3081, 3081, 3081, 3081, 3081, 3081, 3081, 3081, 3081,
- 3082, 3082, 0, 3082, 3082, 3082, 3082, 3082, 3082, 3082,
- 3082, 3082, 3082, 3083, 0, 3083, 3083, 0, 3083, 3083,
- 3083, 3083, 3083, 3083, 3083, 3083, 3084, 3084, 0, 0,
- 3084, 3084, 3084, 3084, 3084, 3084, 3085, 3085, 0, 3085,
- 3085, 3085, 3085, 3085, 3085, 3085, 3085, 3085, 3085, 3086,
- 3086, 0, 3086, 3086, 3086, 3086, 3086, 3086, 3086, 3086,
- 3086, 3086, 3087, 3087, 0, 3087, 3087, 3087, 3087, 3087,
- 3087, 3087, 3087, 3087, 3087, 3088, 3088, 0, 0, 3088,
-
- 3088, 3088, 3088, 3088, 3088, 3089, 3089, 0, 3089, 3089,
- 3089, 3089, 3089, 3089, 3089, 3089, 3089, 3089, 3090, 3090,
- 0, 3090, 3090, 3090, 3090, 3090, 3090, 3090, 3090, 3090,
- 3090, 3091, 3091, 0, 3091, 3091, 3091, 3091, 3091, 3091,
- 3091, 3091, 3091, 3091, 3092, 3092, 3092, 3092, 3092, 3092,
- 3092, 3092, 0, 3092, 3092, 3092, 3092, 3093, 3093, 0,
- 0, 3093, 3093, 3093, 3093, 3093, 3093, 3094, 3094, 3094,
- 3094, 3094, 3094, 3094, 3094, 3094, 3094, 3094, 3094, 3094,
- 3095, 3095, 3095, 3095, 3095, 3095, 3095, 3095, 3095, 3095,
- 3095, 3095, 3095, 3096, 3096, 0, 3096, 3096, 3096, 3096,
-
- 3096, 3096, 3096, 3096, 3096, 3096, 3097, 3097, 0, 3097,
- 3097, 3097, 3097, 3097, 3097, 3097, 3097, 3097, 3097, 3098,
+ 2925, 2925, 2926, 2926, 2926, 2926, 2926, 2926, 2926, 2926,
+
+ 2926, 2926, 2926, 2926, 2926, 2927, 2927, 2927, 2927, 2927,
+ 2927, 2927, 2927, 2927, 2927, 2927, 2927, 2927, 2928, 2928,
+ 2928, 2928, 2928, 2928, 2928, 2928, 2928, 2928, 2928, 2928,
+ 2928, 2929, 2929, 0, 2929, 2929, 2929, 2929, 2929, 2929,
+ 2929, 2929, 2929, 2929, 2930, 2930, 2930, 2930, 2930, 0,
+ 2930, 2930, 2930, 2930, 2931, 2931, 0, 2931, 2931, 2931,
+ 2931, 2931, 2931, 2931, 2931, 2931, 2931, 2932, 2932, 0,
+ 2932, 2932, 2932, 2932, 2932, 2932, 2932, 2932, 2932, 2932,
+ 2933, 2933, 2933, 2933, 2933, 2933, 2933, 2933, 2933, 2933,
+ 2934, 2934, 0, 2934, 2934, 2934, 2934, 2934, 2934, 2934,
+
+ 2934, 2934, 2934, 2935, 2935, 0, 2935, 2935, 2935, 2935,
+ 2935, 2935, 2935, 2935, 2935, 2935, 2936, 2936, 0, 2936,
+ 2936, 2936, 2936, 2936, 2936, 2936, 2936, 2936, 2936, 2937,
+ 0, 2937, 2937, 0, 2937, 2937, 2937, 2937, 2937, 2937,
+ 2937, 2937, 2938, 2938, 0, 0, 2938, 2938, 2938, 2938,
+ 2938, 2938, 2939, 2939, 0, 2939, 2939, 2939, 2939, 2939,
+ 2939, 2939, 2939, 2939, 2939, 2940, 2940, 0, 2940, 2940,
+ 2940, 2940, 2940, 2940, 2940, 2940, 2940, 2940, 2941, 2941,
+ 0, 0, 2941, 2941, 2941, 2941, 2941, 2941, 2942, 2942,
+ 0, 2942, 2942, 2942, 2942, 2942, 2942, 2942, 2942, 2942,
+
+ 2942, 2943, 2943, 0, 2943, 2943, 2943, 2943, 2943, 2943,
+ 2943, 2943, 2943, 2943, 2944, 2944, 2944, 2944, 2944, 2944,
+ 2944, 2944, 0, 2944, 2944, 2944, 2944, 2945, 2945, 0,
+ 0, 2945, 2945, 2945, 2945, 2945, 2945, 2946, 2946, 2946,
+ 2946, 2946, 2946, 2946, 2946, 2946, 2946, 2946, 2946, 2946,
+ 2947, 2947, 2947, 2947, 2947, 2947, 2947, 2947, 2947, 2947,
+ 2947, 2947, 2947, 2948, 2948, 0, 0, 2948, 2948, 2948,
+ 2948, 2948, 2948, 2949, 2949, 0, 2949, 2949, 2949, 2949,
+ 2949, 2949, 2949, 2949, 2949, 2949, 2950, 2950, 0, 2950,
+ 2950, 2950, 2950, 2950, 2950, 2950, 2950, 2950, 2950, 2951,
+
+ 2951, 0, 0, 2951, 2951, 2951, 2951, 2951, 2951, 2952,
+ 2952, 0, 2952, 2952, 2952, 2952, 2952, 2952, 2952, 2952,
+ 2952, 2952, 2953, 2953, 0, 2953, 2953, 2953, 2953, 2953,
+ 2953, 2953, 2953, 2953, 2953, 2954, 2954, 0, 0, 2954,
+ 2954, 2954, 2954, 2954, 2954, 2955, 2955, 0, 2955, 2955,
+ 2955, 2955, 2955, 2955, 2955, 2955, 2955, 2955, 2956, 2956,
+ 0, 2956, 2956, 2956, 2956, 2956, 2956, 2956, 2956, 2956,
+ 2956, 2957, 2957, 0, 0, 2957, 2957, 2957, 2957, 2957,
+ 2957, 2958, 2958, 0, 2958, 2958, 2958, 2958, 2958, 2958,
+ 2958, 2958, 2958, 2958, 2959, 2959, 0, 2959, 2959, 2959,
+
+ 2959, 2959, 2959, 2959, 2959, 2959, 2959, 2960, 2960, 0,
+ 0, 2960, 2960, 2960, 2960, 2960, 2960, 2961, 2961, 0,
+ 2961, 2961, 2961, 2961, 2961, 2961, 2961, 2961, 2961, 2961,
+ 2962, 2962, 0, 2962, 2962, 2962, 2962, 2962, 2962, 2962,
+ 2962, 2962, 2962, 2963, 2963, 0, 0, 2963, 2963, 2963,
+ 2963, 2963, 2963, 2964, 2964, 0, 2964, 2964, 2964, 2964,
+ 2964, 2964, 2964, 2964, 2964, 2964, 2965, 2965, 0, 2965,
+ 2965, 2965, 2965, 2965, 2965, 2965, 2965, 2965, 2965, 2966,
+ 2966, 0, 0, 2966, 2966, 2966, 2966, 2966, 2966, 2967,
+ 2967, 0, 2967, 2967, 2967, 2967, 2967, 2967, 2967, 2967,
+
+ 2967, 2967, 2968, 2968, 0, 2968, 2968, 2968, 2968, 2968,
+ 2968, 2968, 2968, 2968, 2968, 2969, 2969, 0, 0, 2969,
+ 2969, 2969, 2969, 2969, 2969, 2970, 2970, 0, 2970, 2970,
+ 2970, 2970, 2970, 2970, 2970, 2970, 2970, 2970, 2971, 2971,
+ 0, 2971, 2971, 2971, 2971, 2971, 2971, 2971, 2971, 2971,
+ 2971, 2972, 2972, 0, 0, 2972, 2972, 2972, 2972, 2972,
+ 2972, 2973, 2973, 0, 2973, 2973, 2973, 2973, 2973, 2973,
+ 2973, 2973, 2973, 2973, 2974, 2974, 0, 2974, 2974, 2974,
+ 2974, 2974, 2974, 2974, 2974, 2974, 2974, 2975, 2975, 0,
+ 0, 2975, 2975, 2975, 2975, 2975, 2975, 2976, 2976, 0,
+
+ 2976, 2976, 2976, 2976, 2976, 2976, 2976, 2976, 2976, 2976,
+ 2977, 2977, 0, 2977, 2977, 2977, 2977, 2977, 2977, 2977,
+ 2977, 2977, 2977, 2978, 2978, 0, 0, 2978, 2978, 2978,
+ 2978, 2978, 2978, 2979, 2979, 0, 2979, 2979, 2979, 2979,
+ 2979, 2979, 2979, 2979, 2979, 2979, 2980, 2980, 0, 2980,
+ 2980, 2980, 2980, 2980, 2980, 2980, 2980, 2980, 2980, 2981,
+ 2981, 2981, 2981, 0, 2981, 2981, 2981, 2981, 2981, 2981,
+ 2981, 2981, 2982, 2982, 0, 2982, 0, 2982, 2982, 2982,
+ 2982, 2982, 2982, 2982, 2982, 2983, 2983, 0, 0, 2983,
+ 2983, 2983, 2983, 2983, 2983, 2984, 2984, 0, 2984, 2984,
+
+ 2984, 2984, 2984, 2984, 2984, 2984, 2984, 2984, 2985, 2985,
+ 0, 2985, 2985, 2985, 2985, 2985, 2985, 2985, 2985, 2985,
+ 2985, 2986, 2986, 0, 0, 2986, 2986, 2986, 2986, 2986,
+ 2986, 2987, 2987, 0, 2987, 2987, 2987, 2987, 2987, 2987,
+ 2987, 2987, 2987, 2987, 2988, 2988, 0, 2988, 2988, 2988,
+ 2988, 2988, 2988, 2988, 2988, 2988, 2988, 2989, 2989, 0,
+ 0, 2989, 2989, 2989, 2989, 2989, 2989, 2990, 2990, 0,
+ 2990, 2990, 2990, 2990, 2990, 2990, 2990, 2990, 2990, 2990,
+ 2991, 2991, 0, 2991, 2991, 2991, 2991, 2991, 2991, 2991,
+ 2991, 2991, 2991, 2992, 2992, 0, 0, 2992, 2992, 2992,
+
+ 2992, 2992, 2992, 2993, 2993, 0, 2993, 2993, 2993, 2993,
+ 2993, 2993, 2993, 2993, 2993, 2993, 2994, 2994, 0, 2994,
+ 2994, 2994, 2994, 2994, 2994, 2994, 2994, 2994, 2994, 2995,
+ 2995, 2995, 2995, 2995, 2995, 0, 2995, 2995, 2995, 2995,
+ 2995, 2995, 2996, 2996, 0, 0, 2996, 2996, 2996, 2996,
+ 2996, 2996, 2997, 2997, 0, 2997, 2997, 2997, 2997, 2997,
+ 2997, 2997, 2997, 2997, 2997, 2998, 2998, 0, 2998, 2998,
+ 2998, 2998, 2998, 2998, 2998, 2998, 2998, 2998, 2999, 2999,
+ 0, 0, 2999, 2999, 2999, 2999, 2999, 2999, 3000, 3000,
+ 0, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000,
+
+ 3000, 3001, 3001, 0, 3001, 3001, 3001, 3001, 3001, 3001,
+ 3001, 3001, 3001, 3001, 3002, 3002, 0, 0, 3002, 3002,
+ 3002, 3002, 3002, 3002, 3003, 3003, 0, 3003, 3003, 3003,
+ 3003, 3003, 3003, 3003, 3003, 3003, 3003, 3004, 3004, 0,
+ 3004, 3004, 3004, 3004, 3004, 3004, 3004, 3004, 3004, 3004,
+ 3005, 3005, 0, 0, 3005, 3005, 3005, 3005, 3005, 3005,
+ 3006, 3006, 0, 3006, 3006, 3006, 3006, 3006, 3006, 3006,
+ 3006, 3006, 3006, 3007, 3007, 0, 3007, 3007, 3007, 3007,
+ 3007, 3007, 3007, 3007, 3007, 3007, 3008, 3008, 3008, 3008,
+ 3008, 3008, 3008, 3008, 3008, 3008, 3008, 3008, 3008, 3009,
+
+ 3009, 3009, 3009, 3009, 3009, 0, 3009, 3009, 3009, 3009,
+ 3009, 3009, 3010, 3010, 3010, 3010, 3010, 3010, 3010, 3010,
+ 3010, 3010, 3012, 3012, 3012, 3012, 3012, 3012, 3012, 3012,
+ 3012, 0, 3012, 3012, 3012, 3013, 3013, 0, 3013, 3013,
+ 3013, 3013, 3013, 3013, 3013, 3013, 3013, 3013, 3014, 3014,
+ 0, 3014, 3014, 3014, 3014, 3014, 3014, 3014, 3014, 3014,
+ 3014, 3015, 3015, 3015, 3015, 3015, 3015, 3015, 3015, 3015,
+ 3015, 3015, 0, 3015, 3016, 3016, 0, 3016, 3016, 3016,
+ 3016, 3016, 3016, 3016, 3016, 3016, 3016, 3017, 3017, 0,
+ 3017, 3017, 3017, 3017, 3017, 3017, 3017, 3017, 3017, 3017,
+
+ 3018, 3018, 0, 0, 3018, 3018, 3018, 3018, 3018, 3018,
+ 3019, 3019, 0, 3019, 3019, 3019, 3019, 3019, 3019, 3019,
+ 3019, 3019, 3019, 3020, 3020, 0, 3020, 3020, 3020, 3020,
+ 3020, 3020, 3020, 3020, 3020, 3020, 3021, 3021, 0, 0,
+ 3021, 3021, 3021, 3021, 3021, 3021, 3022, 3022, 0, 3022,
+ 3022, 3022, 3022, 3022, 3022, 3022, 3022, 3022, 3022, 3023,
+ 3023, 0, 3023, 3023, 3023, 3023, 3023, 3023, 3023, 3023,
+ 3023, 3023, 3024, 3024, 0, 0, 3024, 3024, 3024, 3024,
+ 3024, 3024, 3025, 3025, 0, 3025, 3025, 3025, 3025, 3025,
+ 3025, 3025, 3025, 3025, 3025, 3026, 3026, 0, 3026, 3026,
+
+ 3026, 3026, 3026, 3026, 3026, 3026, 3026, 3026, 3027, 3027,
+ 0, 0, 3027, 3027, 3027, 3027, 3027, 3027, 3028, 3028,
+ 0, 3028, 3028, 3028, 3028, 3028, 3028, 3028, 3028, 3028,
+ 3028, 3029, 3029, 0, 3029, 3029, 3029, 3029, 3029, 3029,
+ 3029, 3029, 3029, 3029, 3030, 0, 3030, 3030, 0, 3030,
+ 3030, 3030, 3030, 3030, 3030, 3030, 3030, 3031, 0, 3031,
+ 3031, 0, 3031, 3031, 3031, 0, 3031, 3031, 3031, 3031,
+ 3032, 3032, 0, 0, 3032, 3032, 3032, 3032, 3032, 3032,
+ 3033, 3033, 0, 3033, 3033, 3033, 3033, 3033, 3033, 3033,
+ 3033, 3033, 3033, 3034, 3034, 0, 3034, 3034, 3034, 3034,
+
+ 3034, 3034, 3034, 3034, 3034, 3034, 3035, 0, 3035, 3035,
+ 0, 3035, 3035, 3035, 3035, 3035, 3035, 3035, 3035, 3036,
+ 3036, 0, 0, 3036, 3036, 3036, 3036, 3036, 3036, 3037,
+ 3037, 0, 3037, 3037, 3037, 3037, 3037, 3037, 3037, 3037,
+ 3037, 3037, 3038, 3038, 0, 3038, 3038, 3038, 3038, 3038,
+ 3038, 3038, 3038, 3038, 3038, 3039, 0, 3039, 3039, 0,
+ 3039, 3039, 3039, 3039, 3039, 3039, 3039, 3039, 3040, 3040,
+ 0, 0, 3040, 3040, 3040, 3040, 3040, 3040, 3041, 3041,
+ 0, 3041, 3041, 3041, 3041, 3041, 3041, 3041, 3041, 3041,
+ 3041, 3042, 3042, 0, 3042, 3042, 3042, 3042, 3042, 3042,
+
+ 3042, 3042, 3042, 3042, 3043, 3043, 0, 0, 3043, 3043,
+ 3043, 3043, 3043, 3043, 3044, 3044, 0, 3044, 3044, 3044,
+ 3044, 3044, 3044, 3044, 3044, 3044, 3044, 3045, 3045, 0,
+ 3045, 3045, 3045, 3045, 3045, 3045, 3045, 3045, 3045, 3045,
+ 3046, 3046, 0, 0, 3046, 3046, 3046, 3046, 3046, 3046,
+ 3047, 3047, 0, 3047, 3047, 3047, 3047, 3047, 3047, 3047,
+ 3047, 3047, 3047, 3048, 3048, 0, 3048, 3048, 3048, 3048,
+ 3048, 3048, 3048, 3048, 3048, 3048, 3049, 3049, 0, 3049,
+ 3049, 3049, 3049, 3049, 3049, 3049, 3049, 3049, 3049, 3050,
+ 3050, 0, 3050, 3050, 3050, 3050, 3050, 3050, 3050, 3050,
+
+ 3050, 3050, 3051, 3051, 0, 3051, 3051, 3051, 3051, 3051,
+ 3051, 3051, 3051, 3051, 3051, 3052, 3052, 0, 0, 3052,
+ 3052, 3052, 3052, 3052, 3052, 3053, 3053, 0, 3053, 3053,
+ 3053, 3053, 3053, 3053, 3053, 3053, 3053, 3053, 3054, 3054,
+ 0, 3054, 3054, 3054, 3054, 3054, 3054, 3054, 3054, 3054,
+ 3054, 3055, 3055, 0, 0, 3055, 3055, 3055, 3055, 3055,
+ 3055, 3056, 3056, 0, 3056, 3056, 3056, 3056, 3056, 3056,
+ 3056, 3056, 3056, 3056, 3057, 3057, 0, 3057, 3057, 3057,
+ 3057, 3057, 3057, 3057, 3057, 3057, 3057, 3058, 3058, 0,
+ 0, 3058, 3058, 3058, 3058, 3058, 3058, 3059, 3059, 0,
+
+ 3059, 3059, 3059, 3059, 3059, 3059, 3059, 3059, 3059, 3059,
+ 3060, 3060, 0, 3060, 3060, 3060, 3060, 3060, 3060, 3060,
+ 3060, 3060, 3060, 3061, 3061, 0, 3061, 3061, 3061, 3061,
+ 3061, 3061, 3061, 3061, 3061, 3061, 3062, 3062, 0, 3062,
+ 3062, 3062, 3062, 3062, 3062, 3062, 3062, 3062, 3062, 3063,
+ 3063, 0, 3063, 3063, 3063, 3063, 3063, 3063, 3063, 3063,
+ 3063, 3063, 3064, 3064, 0, 0, 3064, 3064, 3064, 3064,
+ 3064, 3064, 3065, 3065, 0, 3065, 3065, 3065, 3065, 3065,
+ 3065, 3065, 3065, 3065, 3065, 3066, 3066, 0, 3066, 3066,
+ 3066, 3066, 3066, 3066, 3066, 3066, 3066, 3066, 3067, 3067,
+
+ 0, 0, 3067, 3067, 3067, 3067, 3067, 3067, 3068, 3068,
+ 0, 3068, 3068, 3068, 3068, 3068, 3068, 3068, 3068, 3068,
+ 3068, 3069, 3069, 0, 3069, 3069, 3069, 3069, 3069, 3069,
+ 3069, 3069, 3069, 3069, 3070, 3070, 0, 0, 3070, 3070,
+ 3070, 3070, 3070, 3070, 3071, 3071, 0, 3071, 3071, 3071,
+ 3071, 3071, 3071, 3071, 3071, 3071, 3071, 3072, 3072, 0,
+ 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072,
+ 3073, 3073, 0, 0, 3073, 3073, 3073, 3073, 3073, 3073,
+ 3074, 3074, 0, 3074, 3074, 3074, 3074, 3074, 3074, 3074,
+ 3074, 3074, 3074, 3075, 3075, 0, 3075, 3075, 3075, 3075,
+
+ 3075, 3075, 3075, 3075, 3075, 3075, 3076, 3076, 0, 3076,
+ 3076, 3076, 3076, 3076, 3076, 3076, 3076, 3076, 3076, 3077,
+ 3077, 3077, 3077, 3077, 0, 3077, 3077, 3077, 3077, 3078,
+ 3078, 0, 3078, 3078, 3078, 3078, 3078, 3078, 3078, 3078,
+ 3078, 3078, 3079, 3079, 0, 3079, 3079, 3079, 3079, 3079,
+ 3079, 3079, 3079, 3079, 3079, 3080, 3080, 0, 3080, 3080,
+ 3080, 3080, 3080, 3080, 3080, 3080, 3080, 3080, 3081, 3081,
+ 0, 3081, 3081, 3081, 3081, 3081, 3081, 3081, 3081, 3081,
+ 3081, 3082, 3082, 0, 3082, 3082, 3082, 3082, 3082, 3082,
+ 3082, 3082, 3082, 3082, 3083, 3083, 0, 3083, 3083, 3083,
+
+ 3083, 3083, 3083, 3083, 3083, 3083, 3083, 3084, 3084, 0,
+ 3084, 3084, 3084, 3084, 3084, 3084, 3084, 3084, 3084, 3084,
+ 3085, 3085, 0, 3085, 3085, 3085, 3085, 3085, 3085, 3085,
+ 3085, 3085, 3085, 3086, 0, 3086, 3086, 0, 3086, 3086,
+ 3086, 3086, 3086, 3086, 3086, 3086, 3087, 3087, 0, 0,
+ 3087, 3087, 3087, 3087, 3087, 3087, 3088, 3088, 0, 3088,
+ 3088, 3088, 3088, 3088, 3088, 3088, 3088, 3088, 3088, 3089,
+ 3089, 0, 3089, 3089, 3089, 3089, 3089, 3089, 3089, 3089,
+ 3089, 3089, 3090, 3090, 0, 3090, 3090, 3090, 3090, 3090,
+ 3090, 3090, 3090, 3090, 3090, 3091, 3091, 0, 0, 3091,
+
+ 3091, 3091, 3091, 3091, 3091, 3092, 3092, 0, 3092, 3092,
+ 3092, 3092, 3092, 3092, 3092, 3092, 3092, 3092, 3093, 3093,
+ 0, 3093, 3093, 3093, 3093, 3093, 3093, 3093, 3093, 3093,
+ 3093, 3094, 3094, 0, 3094, 3094, 3094, 3094, 3094, 3094,
+ 3094, 3094, 3094, 3094, 3095, 3095, 3095, 3095, 3095, 3095,
+ 3095, 3095, 0, 3095, 3095, 3095, 3095, 3096, 3096, 0,
+ 0, 3096, 3096, 3096, 3096, 3096, 3096, 3097, 3097, 3097,
+ 3097, 3097, 3097, 3097, 3097, 3097, 3097, 3097, 3097, 3097,
3098, 3098, 3098, 3098, 3098, 3098, 3098, 3098, 3098, 3098,
- 3098, 3098, 3099, 3099, 0, 3099, 3099, 3099, 3099, 3099,
- 3099, 3099, 3099, 3099, 3099, 3100, 3100, 0, 0, 3100,
- 3100, 3100, 3100, 3100, 3100, 3101, 3101, 0, 3101, 3101,
- 3101, 3101, 3101, 3101, 3101, 3101, 3101, 3101, 3102, 3102,
- 0, 3102, 3102, 3102, 3102, 3102, 3102, 3102, 3102, 3102,
- 3102, 3103, 3103, 0, 3103, 3103, 3103, 3103, 3103, 3103,
- 3103, 3103, 3103, 3103, 3104, 3104, 0, 0, 3104, 3104,
-
- 3104, 3104, 3104, 3104, 3105, 3105, 0, 3105, 3105, 3105,
- 3105, 3105, 3105, 3105, 3105, 3105, 3105, 3106, 3106, 0,
- 3106, 3106, 3106, 3106, 3106, 3106, 3106, 3106, 3106, 3106,
- 3107, 3107, 0, 3107, 3107, 3107, 3107, 3107, 3107, 3107,
- 3107, 3107, 3107, 3108, 3108, 0, 0, 3108, 3108, 3108,
- 3108, 3108, 3108, 3109, 3109, 0, 3109, 3109, 3109, 3109,
- 3109, 3109, 3109, 3109, 3109, 3109, 3110, 3110, 0, 3110,
- 3110, 3110, 3110, 3110, 3110, 3110, 3110, 3110, 3110, 3111,
- 3111, 0, 3111, 3111, 3111, 3111, 3111, 3111, 3111, 3111,
- 3111, 3111, 3112, 3112, 0, 0, 3112, 3112, 3112, 3112,
-
- 3112, 3112, 3113, 3113, 0, 3113, 3113, 3113, 3113, 3113,
- 3113, 3113, 3113, 3113, 3113, 3114, 3114, 0, 3114, 3114,
- 3114, 3114, 3114, 3114, 3114, 3114, 3114, 3114, 3115, 3115,
- 0, 3115, 3115, 3115, 3115, 3115, 3115, 3115, 3115, 3115,
- 3115, 3116, 3116, 0, 0, 3116, 3116, 3116, 3116, 3116,
- 3116, 3117, 3117, 0, 3117, 3117, 3117, 3117, 3117, 3117,
- 3117, 3117, 3117, 3117, 3118, 3118, 0, 3118, 3118, 3118,
- 3118, 3118, 3118, 3118, 3118, 3118, 3118, 3119, 3119, 0,
- 3119, 3119, 3119, 3119, 3119, 3119, 3119, 3119, 3119, 3119,
- 3120, 3120, 0, 0, 3120, 3120, 3120, 3120, 3120, 3120,
-
- 3121, 3121, 0, 3121, 3121, 3121, 3121, 3121, 3121, 3121,
- 3121, 3121, 3121, 3122, 3122, 0, 3122, 3122, 3122, 3122,
- 3122, 3122, 3122, 3122, 3122, 3122, 3123, 3123, 0, 3123,
- 3123, 3123, 3123, 3123, 3123, 3123, 3123, 3123, 3123, 3124,
- 3124, 0, 0, 3124, 3124, 3124, 3124, 3124, 3124, 3125,
- 3125, 0, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125,
- 3125, 3125, 3126, 3126, 0, 3126, 3126, 3126, 3126, 3126,
- 3126, 3126, 3126, 3126, 3126, 3127, 3127, 0, 3127, 3127,
- 3127, 3127, 3127, 3127, 3127, 3127, 3127, 3127, 3128, 3128,
- 0, 0, 3128, 3128, 3128, 3128, 3128, 3128, 3129, 3129,
-
- 0, 3129, 3129, 3129, 3129, 3129, 3129, 3129, 3129, 3129,
- 3129, 3130, 3130, 0, 3130, 3130, 3130, 3130, 3130, 3130,
- 3130, 3130, 3130, 3130, 3131, 3131, 0, 3131, 3131, 3131,
- 3131, 3131, 3131, 3131, 3131, 3131, 3131, 3132, 3132, 0,
- 0, 3132, 3132, 3132, 3132, 3132, 3132, 3133, 3133, 0,
- 3133, 3133, 3133, 3133, 3133, 3133, 3133, 3133, 3133, 3133,
- 3134, 3134, 0, 3134, 3134, 3134, 3134, 3134, 3134, 3134,
- 3134, 3134, 3134, 3135, 3135, 0, 3135, 3135, 3135, 3135,
- 3135, 3135, 3135, 3135, 3135, 3135, 3136, 3136, 0, 0,
- 3136, 3136, 3136, 3136, 3136, 3136, 3137, 3137, 0, 3137,
-
- 3137, 3137, 3137, 3137, 3137, 3137, 3137, 3137, 3137, 3138,
- 3138, 0, 3138, 3138, 3138, 3138, 3138, 3138, 3138, 3138,
- 3138, 3138, 3139, 3139, 0, 3139, 3139, 3139, 3139, 3139,
- 3139, 3139, 3139, 3139, 3139, 3140, 3140, 0, 0, 3140,
- 3140, 3140, 3140, 3140, 3140, 3141, 3141, 0, 3141, 3141,
- 3141, 3141, 3141, 3141, 3141, 3141, 3141, 3141, 3142, 3142,
- 0, 3142, 3142, 3142, 3142, 3142, 3142, 3142, 3142, 3142,
- 3142, 3143, 3143, 0, 3143, 3143, 3143, 3143, 3143, 3143,
- 3143, 3143, 3143, 3143, 3144, 3144, 3144, 3144, 0, 3144,
- 3144, 3144, 3144, 3144, 3144, 3144, 3144, 3145, 3145, 0,
-
- 3145, 0, 3145, 3145, 3145, 3145, 3145, 3145, 3145, 3145,
- 3146, 3146, 0, 0, 3146, 3146, 3146, 3146, 3146, 3146,
- 3147, 3147, 0, 3147, 3147, 3147, 3147, 3147, 3147, 3147,
- 3147, 3147, 3147, 3148, 3148, 0, 3148, 3148, 3148, 3148,
- 3148, 3148, 3148, 3148, 3148, 3148, 3149, 3149, 0, 3149,
- 3149, 3149, 3149, 3149, 3149, 3149, 3149, 3149, 3149, 3150,
- 3150, 0, 0, 3150, 3150, 3150, 3150, 3150, 3150, 3151,
- 3151, 0, 3151, 3151, 3151, 3151, 3151, 3151, 3151, 3151,
- 3151, 3151, 3152, 3152, 0, 3152, 3152, 3152, 3152, 3152,
- 3152, 3152, 3152, 3152, 3152, 3153, 3153, 0, 3153, 3153,
-
- 3153, 3153, 3153, 3153, 3153, 3153, 3153, 3153, 3154, 3154,
- 0, 0, 3154, 3154, 3154, 3154, 3154, 3154, 3155, 3155,
- 0, 3155, 3155, 3155, 3155, 3155, 3155, 3155, 3155, 3155,
- 3155, 3156, 3156, 0, 3156, 3156, 3156, 3156, 3156, 3156,
- 3156, 3156, 3156, 3156, 3157, 3157, 0, 3157, 3157, 3157,
- 3157, 3157, 3157, 3157, 3157, 3157, 3157, 3158, 3158, 0,
- 0, 3158, 3158, 3158, 3158, 3158, 3158, 3159, 3159, 0,
- 3159, 3159, 3159, 3159, 3159, 3159, 3159, 3159, 3159, 3159,
- 3160, 3160, 0, 3160, 3160, 3160, 3160, 3160, 3160, 3160,
- 3160, 3160, 3160, 3161, 3161, 0, 3161, 3161, 3161, 3161,
-
- 3161, 3161, 3161, 3161, 3161, 3161, 3162, 3162, 3162, 3162,
- 3162, 3162, 0, 3162, 3162, 3162, 3162, 3162, 3162, 3163,
- 3163, 0, 0, 3163, 3163, 3163, 3163, 3163, 3163, 3164,
- 3164, 0, 3164, 3164, 3164, 3164, 3164, 3164, 3164, 3164,
- 3164, 3164, 3165, 3165, 0, 3165, 3165, 3165, 3165, 3165,
- 3165, 3165, 3165, 3165, 3165, 3166, 3166, 0, 3166, 3166,
- 3166, 3166, 3166, 3166, 3166, 3166, 3166, 3166, 3167, 3167,
- 0, 0, 3167, 3167, 3167, 3167, 3167, 3167, 3168, 3168,
- 0, 3168, 3168, 3168, 3168, 3168, 3168, 3168, 3168, 3168,
- 3168, 3169, 3169, 0, 3169, 3169, 3169, 3169, 3169, 3169,
-
- 3169, 3169, 3169, 3169, 3170, 3170, 0, 3170, 3170, 3170,
- 3170, 3170, 3170, 3170, 3170, 3170, 3170, 3171, 3171, 0,
- 0, 3171, 3171, 3171, 3171, 3171, 3171, 3172, 3172, 0,
- 3172, 3172, 3172, 3172, 3172, 3172, 3172, 3172, 3172, 3172,
- 3173, 3173, 0, 3173, 3173, 3173, 3173, 3173, 3173, 3173,
- 3173, 3173, 3173, 3174, 3174, 0, 3174, 3174, 3174, 3174,
- 3174, 3174, 3174, 3174, 3174, 3174, 3175, 3175, 0, 0,
- 3175, 3175, 3175, 3175, 3175, 3175, 3176, 3176, 0, 3176,
- 3176, 3176, 3176, 3176, 3176, 3176, 3176, 3176, 3176, 3177,
- 3177, 0, 3177, 3177, 3177, 3177, 3177, 3177, 3177, 3177,
-
- 3177, 3177, 3178, 3178, 0, 3178, 3178, 3178, 3178, 3178,
- 3178, 3178, 3178, 3178, 3178, 3179, 3179, 3179, 3179, 3179,
- 3179, 3179, 3179, 3179, 3179, 3179, 3179, 3179, 3180, 3180,
- 3180, 3180, 3180, 3180, 3180, 3180, 3180, 3180, 3180, 3180,
- 3180, 3181, 3181, 3181, 3181, 3181, 3181, 3181, 3181, 3181,
- 3181, 3181, 3181, 3181, 3182, 3182, 3182, 3182, 3182, 3182,
- 3182, 3182, 3182, 3182, 3182, 3182, 3182, 3183, 3183, 3183,
+ 3098, 3098, 3098, 3099, 3099, 0, 3099, 3099, 3099, 3099,
+
+ 3099, 3099, 3099, 3099, 3099, 3099, 3100, 3100, 0, 3100,
+ 3100, 3100, 3100, 3100, 3100, 3100, 3100, 3100, 3100, 3101,
+ 3101, 3101, 3101, 3101, 3101, 3101, 3101, 3101, 3101, 3101,
+ 3101, 3101, 3102, 3102, 0, 3102, 3102, 3102, 3102, 3102,
+ 3102, 3102, 3102, 3102, 3102, 3103, 3103, 0, 0, 3103,
+ 3103, 3103, 3103, 3103, 3103, 3104, 3104, 0, 3104, 3104,
+ 3104, 3104, 3104, 3104, 3104, 3104, 3104, 3104, 3105, 3105,
+ 0, 3105, 3105, 3105, 3105, 3105, 3105, 3105, 3105, 3105,
+ 3105, 3106, 3106, 0, 3106, 3106, 3106, 3106, 3106, 3106,
+ 3106, 3106, 3106, 3106, 3107, 3107, 0, 0, 3107, 3107,
+
+ 3107, 3107, 3107, 3107, 3108, 3108, 0, 3108, 3108, 3108,
+ 3108, 3108, 3108, 3108, 3108, 3108, 3108, 3109, 3109, 0,
+ 3109, 3109, 3109, 3109, 3109, 3109, 3109, 3109, 3109, 3109,
+ 3110, 3110, 0, 3110, 3110, 3110, 3110, 3110, 3110, 3110,
+ 3110, 3110, 3110, 3111, 3111, 0, 0, 3111, 3111, 3111,
+ 3111, 3111, 3111, 3112, 3112, 0, 3112, 3112, 3112, 3112,
+ 3112, 3112, 3112, 3112, 3112, 3112, 3113, 3113, 0, 3113,
+ 3113, 3113, 3113, 3113, 3113, 3113, 3113, 3113, 3113, 3114,
+ 3114, 0, 3114, 3114, 3114, 3114, 3114, 3114, 3114, 3114,
+ 3114, 3114, 3115, 3115, 0, 0, 3115, 3115, 3115, 3115,
+
+ 3115, 3115, 3116, 3116, 0, 3116, 3116, 3116, 3116, 3116,
+ 3116, 3116, 3116, 3116, 3116, 3117, 3117, 0, 3117, 3117,
+ 3117, 3117, 3117, 3117, 3117, 3117, 3117, 3117, 3118, 3118,
+ 0, 3118, 3118, 3118, 3118, 3118, 3118, 3118, 3118, 3118,
+ 3118, 3119, 3119, 0, 0, 3119, 3119, 3119, 3119, 3119,
+ 3119, 3120, 3120, 0, 3120, 3120, 3120, 3120, 3120, 3120,
+ 3120, 3120, 3120, 3120, 3121, 3121, 0, 3121, 3121, 3121,
+ 3121, 3121, 3121, 3121, 3121, 3121, 3121, 3122, 3122, 0,
+ 3122, 3122, 3122, 3122, 3122, 3122, 3122, 3122, 3122, 3122,
+ 3123, 3123, 0, 0, 3123, 3123, 3123, 3123, 3123, 3123,
+
+ 3124, 3124, 0, 3124, 3124, 3124, 3124, 3124, 3124, 3124,
+ 3124, 3124, 3124, 3125, 3125, 0, 3125, 3125, 3125, 3125,
+ 3125, 3125, 3125, 3125, 3125, 3125, 3126, 3126, 0, 3126,
+ 3126, 3126, 3126, 3126, 3126, 3126, 3126, 3126, 3126, 3127,
+ 3127, 0, 0, 3127, 3127, 3127, 3127, 3127, 3127, 3128,
+ 3128, 0, 3128, 3128, 3128, 3128, 3128, 3128, 3128, 3128,
+ 3128, 3128, 3129, 3129, 0, 3129, 3129, 3129, 3129, 3129,
+ 3129, 3129, 3129, 3129, 3129, 3130, 3130, 0, 3130, 3130,
+ 3130, 3130, 3130, 3130, 3130, 3130, 3130, 3130, 3131, 3131,
+ 0, 0, 3131, 3131, 3131, 3131, 3131, 3131, 3132, 3132,
+
+ 0, 3132, 3132, 3132, 3132, 3132, 3132, 3132, 3132, 3132,
+ 3132, 3133, 3133, 0, 3133, 3133, 3133, 3133, 3133, 3133,
+ 3133, 3133, 3133, 3133, 3134, 3134, 0, 3134, 3134, 3134,
+ 3134, 3134, 3134, 3134, 3134, 3134, 3134, 3135, 3135, 0,
+ 0, 3135, 3135, 3135, 3135, 3135, 3135, 3136, 3136, 0,
+ 3136, 3136, 3136, 3136, 3136, 3136, 3136, 3136, 3136, 3136,
+ 3137, 3137, 0, 3137, 3137, 3137, 3137, 3137, 3137, 3137,
+ 3137, 3137, 3137, 3138, 3138, 0, 3138, 3138, 3138, 3138,
+ 3138, 3138, 3138, 3138, 3138, 3138, 3139, 3139, 0, 0,
+ 3139, 3139, 3139, 3139, 3139, 3139, 3140, 3140, 0, 3140,
+
+ 3140, 3140, 3140, 3140, 3140, 3140, 3140, 3140, 3140, 3141,
+ 3141, 0, 3141, 3141, 3141, 3141, 3141, 3141, 3141, 3141,
+ 3141, 3141, 3142, 3142, 0, 3142, 3142, 3142, 3142, 3142,
+ 3142, 3142, 3142, 3142, 3142, 3143, 3143, 0, 0, 3143,
+ 3143, 3143, 3143, 3143, 3143, 3144, 3144, 0, 3144, 3144,
+ 3144, 3144, 3144, 3144, 3144, 3144, 3144, 3144, 3145, 3145,
+ 0, 3145, 3145, 3145, 3145, 3145, 3145, 3145, 3145, 3145,
+ 3145, 3146, 3146, 0, 3146, 3146, 3146, 3146, 3146, 3146,
+ 3146, 3146, 3146, 3146, 3147, 3147, 3147, 3147, 0, 3147,
+ 3147, 3147, 3147, 3147, 3147, 3147, 3147, 3148, 3148, 0,
+
+ 3148, 0, 3148, 3148, 3148, 3148, 3148, 3148, 3148, 3148,
+ 3149, 3149, 0, 0, 3149, 3149, 3149, 3149, 3149, 3149,
+ 3150, 3150, 0, 3150, 3150, 3150, 3150, 3150, 3150, 3150,
+ 3150, 3150, 3150, 3151, 3151, 0, 3151, 3151, 3151, 3151,
+ 3151, 3151, 3151, 3151, 3151, 3151, 3152, 3152, 0, 3152,
+ 3152, 3152, 3152, 3152, 3152, 3152, 3152, 3152, 3152, 3153,
+ 3153, 0, 0, 3153, 3153, 3153, 3153, 3153, 3153, 3154,
+ 3154, 0, 3154, 3154, 3154, 3154, 3154, 3154, 3154, 3154,
+ 3154, 3154, 3155, 3155, 0, 3155, 3155, 3155, 3155, 3155,
+ 3155, 3155, 3155, 3155, 3155, 3156, 3156, 0, 3156, 3156,
+
+ 3156, 3156, 3156, 3156, 3156, 3156, 3156, 3156, 3157, 3157,
+ 0, 0, 3157, 3157, 3157, 3157, 3157, 3157, 3158, 3158,
+ 0, 3158, 3158, 3158, 3158, 3158, 3158, 3158, 3158, 3158,
+ 3158, 3159, 3159, 0, 3159, 3159, 3159, 3159, 3159, 3159,
+ 3159, 3159, 3159, 3159, 3160, 3160, 0, 3160, 3160, 3160,
+ 3160, 3160, 3160, 3160, 3160, 3160, 3160, 3161, 3161, 0,
+ 0, 3161, 3161, 3161, 3161, 3161, 3161, 3162, 3162, 0,
+ 3162, 3162, 3162, 3162, 3162, 3162, 3162, 3162, 3162, 3162,
+ 3163, 3163, 0, 3163, 3163, 3163, 3163, 3163, 3163, 3163,
+ 3163, 3163, 3163, 3164, 3164, 0, 3164, 3164, 3164, 3164,
+
+ 3164, 3164, 3164, 3164, 3164, 3164, 3165, 3165, 3165, 3165,
+ 3165, 3165, 0, 3165, 3165, 3165, 3165, 3165, 3165, 3166,
+ 3166, 0, 0, 3166, 3166, 3166, 3166, 3166, 3166, 3167,
+ 3167, 0, 3167, 3167, 3167, 3167, 3167, 3167, 3167, 3167,
+ 3167, 3167, 3168, 3168, 0, 3168, 3168, 3168, 3168, 3168,
+ 3168, 3168, 3168, 3168, 3168, 3169, 3169, 0, 3169, 3169,
+ 3169, 3169, 3169, 3169, 3169, 3169, 3169, 3169, 3170, 3170,
+ 0, 0, 3170, 3170, 3170, 3170, 3170, 3170, 3171, 3171,
+ 0, 3171, 3171, 3171, 3171, 3171, 3171, 3171, 3171, 3171,
+ 3171, 3172, 3172, 0, 3172, 3172, 3172, 3172, 3172, 3172,
+
+ 3172, 3172, 3172, 3172, 3173, 3173, 0, 3173, 3173, 3173,
+ 3173, 3173, 3173, 3173, 3173, 3173, 3173, 3174, 3174, 0,
+ 0, 3174, 3174, 3174, 3174, 3174, 3174, 3175, 3175, 0,
+ 3175, 3175, 3175, 3175, 3175, 3175, 3175, 3175, 3175, 3175,
+ 3176, 3176, 0, 3176, 3176, 3176, 3176, 3176, 3176, 3176,
+ 3176, 3176, 3176, 3177, 3177, 0, 3177, 3177, 3177, 3177,
+ 3177, 3177, 3177, 3177, 3177, 3177, 3178, 3178, 0, 0,
+ 3178, 3178, 3178, 3178, 3178, 3178, 3179, 3179, 0, 3179,
+ 3179, 3179, 3179, 3179, 3179, 3179, 3179, 3179, 3179, 3180,
+ 3180, 0, 3180, 3180, 3180, 3180, 3180, 3180, 3180, 3180,
+
+ 3180, 3180, 3181, 3181, 0, 3181, 3181, 3181, 3181, 3181,
+ 3181, 3181, 3181, 3181, 3181, 3182, 3182, 3182, 3182, 3182,
+ 3182, 3182, 3182, 3182, 3182, 3182, 3182, 3182, 3183, 3183,
3183, 3183, 3183, 3183, 3183, 3183, 3183, 3183, 3183, 3183,
- 3184, 3184, 0, 3184, 3184, 3184, 3184, 3184, 3184, 3184,
- 3184, 3184, 3184, 3185, 3185, 0, 3185, 3185, 3185, 3185,
-
- 3185, 3185, 3185, 3185, 3185, 3185, 3186, 3186, 0, 3186,
- 3186, 3186, 3186, 3186, 3186, 3186, 3186, 3186, 3186, 3188,
- 3188, 0, 3188, 3188, 3188, 3188, 3188, 3188, 3188, 3188,
- 3188, 3188, 3189, 3189, 0, 3189, 3189, 3189, 3189, 3189,
- 3189, 3189, 3189, 3189, 3189, 3190, 3190, 3190, 3190, 3190,
- 3190, 3190, 3190, 3190, 3190, 3190, 0, 3190, 3191, 3191,
- 0, 3191, 3191, 3191, 3191, 3191, 3191, 3191, 3191, 3191,
- 3191, 3192, 3192, 0, 3192, 3192, 3192, 3192, 3192, 3192,
- 3192, 3192, 3192, 3192, 3193, 3193, 3193, 3193, 3193, 3193,
- 3193, 3193, 3193, 3193, 3193, 3193, 3193, 3194, 3194, 3194,
-
- 3194, 3194, 3194, 3194, 3194, 3194, 3194, 3194, 3194, 3194,
- 3195, 3195, 3195, 3195, 3195, 3195, 3195, 3195, 3195, 3195,
- 3195, 3195, 3195, 3196, 3196, 3196, 3196, 3196, 3196, 3196,
- 3196, 3196, 3196, 3196, 3196, 3196, 3197, 3197, 3197, 3197,
- 3197, 3197, 3197, 3197, 3197, 3197, 3197, 3197, 3197, 3198,
+ 3183, 3184, 3184, 3184, 3184, 3184, 3184, 3184, 3184, 3184,
+ 3184, 3184, 3184, 3184, 3185, 3185, 3185, 3185, 3185, 3185,
+ 3185, 3185, 3185, 3185, 3185, 3185, 3185, 3186, 3186, 3186,
+ 3186, 3186, 3186, 3186, 3186, 3186, 3186, 3186, 3186, 3186,
+ 3187, 3187, 0, 3187, 3187, 3187, 3187, 3187, 3187, 3187,
+ 3187, 3187, 3187, 3188, 3188, 0, 3188, 3188, 3188, 3188,
+
+ 3188, 3188, 3188, 3188, 3188, 3188, 3189, 3189, 0, 3189,
+ 3189, 3189, 3189, 3189, 3189, 3189, 3189, 3189, 3189, 3191,
+ 3191, 0, 3191, 3191, 3191, 3191, 3191, 3191, 3191, 3191,
+ 3191, 3191, 3192, 3192, 0, 3192, 3192, 3192, 3192, 3192,
+ 3192, 3192, 3192, 3192, 3192, 3193, 3193, 3193, 3193, 3193,
+ 3193, 3193, 3193, 3193, 3193, 3193, 0, 3193, 3194, 3194,
+ 0, 3194, 3194, 3194, 3194, 3194, 3194, 3194, 3194, 3194,
+ 3194, 3195, 3195, 0, 3195, 3195, 3195, 3195, 3195, 3195,
+ 3195, 3195, 3195, 3195, 3196, 3196, 3196, 3196, 3196, 3196,
+ 3196, 3196, 3196, 3196, 3196, 3196, 3196, 3197, 3197, 3197,
+
+ 3197, 3197, 3197, 3197, 3197, 3197, 3197, 3197, 3197, 3197,
3198, 3198, 3198, 3198, 3198, 3198, 3198, 3198, 3198, 3198,
- 3198, 3198, 3199, 3199, 0, 3199, 3199, 3199, 3199, 3199,
- 3199, 3199, 3199, 3199, 3199, 3200, 3200, 0, 0, 3200,
- 3200, 3200, 3200, 3200, 3200, 3201, 3201, 0, 3201, 3201,
- 3201, 3201, 3201, 3201, 3201, 3201, 3201, 3201, 3202, 3202,
-
- 0, 3202, 3202, 3202, 3202, 3202, 3202, 3202, 3202, 3202,
- 3202, 3203, 3203, 0, 3203, 3203, 3203, 3203, 3203, 3203,
- 3203, 3203, 3203, 3203, 3204, 3204, 0, 0, 3204, 3204,
- 3204, 3204, 3204, 3204, 3205, 3205, 0, 3205, 3205, 3205,
- 3205, 3205, 3205, 3205, 3205, 3205, 3205, 3206, 3206, 0,
- 3206, 3206, 3206, 3206, 3206, 3206, 3206, 3206, 3206, 3206,
- 3207, 3207, 0, 3207, 3207, 3207, 3207, 3207, 3207, 3207,
- 3207, 3207, 3207, 3208, 3208, 0, 0, 3208, 3208, 3208,
- 3208, 3208, 3208, 3209, 3209, 0, 3209, 3209, 3209, 3209,
- 3209, 3209, 3209, 3209, 3209, 3209, 3210, 3210, 0, 3210,
-
- 3210, 3210, 3210, 3210, 3210, 3210, 3210, 3210, 3210, 3211,
- 3211, 0, 3211, 3211, 3211, 3211, 3211, 3211, 3211, 3211,
- 3211, 3211, 3212, 3212, 0, 0, 3212, 3212, 3212, 3212,
- 3212, 3212, 3213, 3213, 0, 3213, 3213, 3213, 3213, 3213,
- 3213, 3213, 3213, 3213, 3213, 3214, 3214, 0, 3214, 3214,
- 3214, 3214, 3214, 3214, 3214, 3214, 3214, 3214, 3215, 3215,
- 0, 3215, 3215, 3215, 3215, 3215, 3215, 3215, 3215, 3215,
- 3215, 3216, 0, 3216, 3216, 0, 3216, 3216, 3216, 3216,
- 3216, 3216, 3216, 3216, 3217, 0, 3217, 3217, 0, 3217,
- 3217, 3217, 0, 3217, 3217, 3217, 3217, 3218, 3218, 0,
-
- 0, 3218, 3218, 3218, 3218, 3218, 3218, 3219, 3219, 0,
- 3219, 3219, 3219, 3219, 3219, 3219, 3219, 3219, 3219, 3219,
- 3220, 3220, 0, 3220, 3220, 3220, 3220, 3220, 3220, 3220,
- 3220, 3220, 3220, 3221, 3221, 0, 3221, 3221, 3221, 3221,
- 3221, 3221, 3221, 3221, 3221, 3221, 3222, 0, 3222, 3222,
- 0, 3222, 3222, 3222, 3222, 3222, 3222, 3222, 3222, 3223,
- 3223, 0, 0, 3223, 3223, 3223, 3223, 3223, 3223, 3224,
- 3224, 0, 3224, 3224, 3224, 3224, 3224, 3224, 3224, 3224,
- 3224, 3224, 3225, 3225, 0, 3225, 3225, 3225, 3225, 3225,
- 3225, 3225, 3225, 3225, 3225, 3226, 3226, 0, 3226, 3226,
-
- 3226, 3226, 3226, 3226, 3226, 3226, 3226, 3226, 3227, 0,
- 3227, 3227, 0, 3227, 3227, 3227, 3227, 3227, 3227, 3227,
- 3227, 3228, 3228, 0, 0, 3228, 3228, 3228, 3228, 3228,
- 3228, 3229, 3229, 0, 3229, 3229, 3229, 3229, 3229, 3229,
- 3229, 3229, 3229, 3229, 3230, 3230, 0, 3230, 3230, 3230,
- 3230, 3230, 3230, 3230, 3230, 3230, 3230, 3231, 3231, 0,
- 3231, 3231, 3231, 3231, 3231, 3231, 3231, 3231, 3231, 3231,
- 3232, 3232, 0, 0, 3232, 3232, 3232, 3232, 3232, 3232,
- 3233, 3233, 0, 3233, 3233, 3233, 3233, 3233, 3233, 3233,
- 3233, 3233, 3233, 3234, 3234, 0, 3234, 3234, 3234, 3234,
-
- 3234, 3234, 3234, 3234, 3234, 3234, 3235, 3235, 0, 3235,
- 3235, 3235, 3235, 3235, 3235, 3235, 3235, 3235, 3235, 3236,
- 3236, 0, 0, 3236, 3236, 3236, 3236, 3236, 3236, 3237,
- 3237, 0, 3237, 3237, 3237, 3237, 3237, 3237, 3237, 3237,
- 3237, 3237, 3238, 3238, 0, 3238, 3238, 3238, 3238, 3238,
- 3238, 3238, 3238, 3238, 3238, 3239, 3239, 0, 3239, 3239,
- 3239, 3239, 3239, 3239, 3239, 3239, 3239, 3239, 3240, 3240,
- 0, 3240, 3240, 3240, 3240, 3240, 3240, 3240, 3240, 3240,
- 3240, 3241, 3241, 0, 3241, 3241, 3241, 3241, 3241, 3241,
- 3241, 3241, 3241, 3241, 3242, 3242, 0, 0, 3242, 3242,
-
- 3242, 3242, 3242, 3242, 3243, 3243, 0, 3243, 3243, 3243,
- 3243, 3243, 3243, 3243, 3243, 3243, 3243, 3244, 3244, 0,
- 3244, 3244, 3244, 3244, 3244, 3244, 3244, 3244, 3244, 3244,
- 3245, 3245, 0, 3245, 3245, 3245, 3245, 3245, 3245, 3245,
- 3245, 3245, 3245, 3246, 3246, 0, 0, 3246, 3246, 3246,
- 3246, 3246, 3246, 3247, 3247, 0, 3247, 3247, 3247, 3247,
- 3247, 3247, 3247, 3247, 3247, 3247, 3248, 3248, 0, 3248,
- 3248, 3248, 3248, 3248, 3248, 3248, 3248, 3248, 3248, 3249,
- 3249, 0, 3249, 3249, 3249, 3249, 3249, 3249, 3249, 3249,
- 3249, 3249, 3250, 3250, 0, 0, 3250, 3250, 3250, 3250,
-
- 3250, 3250, 3251, 3251, 0, 3251, 3251, 3251, 3251, 3251,
- 3251, 3251, 3251, 3251, 3251, 3252, 3252, 0, 3252, 3252,
- 3252, 3252, 3252, 3252, 3252, 3252, 3252, 3252, 3253, 3253,
- 0, 3253, 3253, 3253, 3253, 3253, 3253, 3253, 3253, 3253,
- 3253, 3254, 3254, 0, 3254, 3254, 3254, 3254, 3254, 3254,
- 3254, 3254, 3254, 3254, 3255, 3255, 0, 3255, 3255, 3255,
- 3255, 3255, 3255, 3255, 3255, 3255, 3255, 3256, 3256, 0,
- 0, 3256, 3256, 3256, 3256, 3256, 3256, 3257, 3257, 0,
- 3257, 3257, 3257, 3257, 3257, 3257, 3257, 3257, 3257, 3257,
- 3258, 3258, 0, 3258, 3258, 3258, 3258, 3258, 3258, 3258,
-
- 3258, 3258, 3258, 3259, 3259, 0, 3259, 3259, 3259, 3259,
- 3259, 3259, 3259, 3259, 3259, 3259, 3260, 3260, 0, 0,
- 3260, 3260, 3260, 3260, 3260, 3260, 3261, 3261, 0, 3261,
- 3261, 3261, 3261, 3261, 3261, 3261, 3261, 3261, 3261, 3262,
- 3262, 0, 3262, 3262, 3262, 3262, 3262, 3262, 3262, 3262,
- 3262, 3262, 3263, 3263, 0, 3263, 3263, 3263, 3263, 3263,
- 3263, 3263, 3263, 3263, 3263, 3264, 3264, 0, 0, 3264,
- 3264, 3264, 3264, 3264, 3264, 3265, 3265, 0, 3265, 3265,
- 3265, 3265, 3265, 3265, 3265, 3265, 3265, 3265, 3266, 3266,
- 0, 3266, 3266, 3266, 3266, 3266, 3266, 3266, 3266, 3266,
-
- 3266, 3267, 3267, 0, 3267, 3267, 3267, 3267, 3267, 3267,
- 3267, 3267, 3267, 3267, 3268, 3268, 0, 0, 3268, 3268,
- 3268, 3268, 3268, 3268, 3269, 3269, 0, 3269, 3269, 3269,
- 3269, 3269, 3269, 3269, 3269, 3269, 3269, 3270, 3270, 0,
- 3270, 3270, 3270, 3270, 3270, 3270, 3270, 3270, 3270, 3270,
- 3271, 3271, 0, 3271, 3271, 3271, 3271, 3271, 3271, 3271,
- 3271, 3271, 3271, 3272, 3272, 3272, 3272, 3272, 0, 3272,
- 3272, 3272, 3272, 3273, 3273, 3273, 3273, 3273, 3273, 3273,
- 3273, 3273, 3273, 3273, 3273, 3273, 3274, 3274, 0, 3274,
- 3274, 3274, 3274, 3274, 3274, 3274, 3274, 3274, 3274, 3275,
-
- 3275, 0, 3275, 3275, 3275, 3275, 3275, 3275, 3275, 3275,
- 3275, 3275, 3276, 3276, 0, 3276, 3276, 3276, 3276, 3276,
- 3276, 3276, 3276, 3276, 3276, 3277, 3277, 0, 3277, 3277,
- 3277, 3277, 3277, 3277, 3277, 3277, 3277, 3277, 3278, 3278,
- 0, 3278, 3278, 3278, 3278, 3278, 3278, 3278, 3278, 3278,
- 3278, 3279, 0, 3279, 3279, 0, 3279, 3279, 3279, 3279,
- 3279, 3279, 3279, 3279, 3280, 3280, 0, 3280, 3280, 3280,
- 3280, 3280, 3280, 3280, 3280, 3280, 3280, 3281, 3281, 0,
- 3281, 3281, 3281, 3281, 3281, 3281, 3281, 3281, 3281, 3281,
- 3282, 3282, 3282, 3282, 3282, 3282, 3282, 3282, 3282, 3282,
-
- 3282, 3282, 3282, 3283, 3283, 0, 3283, 3283, 3283, 3283,
- 3283, 3283, 3283, 3283, 3283, 3283, 3284, 3284, 0, 3284,
- 3284, 3284, 3284, 3284, 3284, 3284, 3284, 3284, 3284, 3285,
- 3285, 0, 3285, 3285, 3285, 3285, 3285, 3285, 3285, 3285,
- 3285, 3285, 3286, 3286, 0, 3286, 3286, 3286, 3286, 3286,
- 3286, 3286, 3286, 3286, 3286, 3287, 3287, 0, 3287, 3287,
- 3287, 3287, 3287, 3287, 3287, 3287, 3287, 3287, 3288, 3288,
- 0, 3288, 3288, 3288, 3288, 3288, 3288, 3288, 3288, 3288,
- 3288, 3289, 3289, 0, 3289, 3289, 3289, 3289, 3289, 3289,
- 3289, 3289, 3289, 3289, 3290, 3290, 0, 3290, 3290, 3290,
-
- 3290, 3290, 3290, 3290, 3290, 3290, 3290, 3291, 3291, 0,
- 3291, 3291, 3291, 3291, 3291, 3291, 3291, 3291, 3291, 3291,
- 3292, 3292, 0, 3292, 3292, 3292, 3292, 3292, 3292, 3292,
- 3292, 3292, 3292, 3293, 3293, 0, 3293, 3293, 3293, 3293,
- 3293, 3293, 3293, 3293, 3293, 3293, 3294, 3294, 0, 3294,
- 3294, 3294, 3294, 3294, 3294, 3294, 3294, 3294, 3294, 3295,
- 3295, 0, 3295, 3295, 3295, 3295, 3295, 3295, 3295, 3295,
- 3295, 3295, 3296, 3296, 0, 3296, 3296, 3296, 3296, 3296,
- 3296, 3296, 3296, 3296, 3296, 3297, 3297, 0, 3297, 3297,
- 3297, 3297, 3297, 3297, 3297, 3297, 3297, 3297, 3298, 3298,
-
- 0, 3298, 3298, 3298, 3298, 3298, 3298, 3298, 3298, 3298,
- 3298, 3299, 3299, 0, 3299, 3299, 3299, 3299, 3299, 3299,
- 3299, 3299, 3299, 3299, 3300, 3300, 0, 3300, 3300, 3300,
- 3300, 3300, 3300, 3300, 3300, 3300, 3300, 3301, 3301, 0,
- 3301, 3301, 3301, 3301, 3301, 3301, 3301, 3301, 3301, 3301,
- 3302, 3302, 0, 3302, 3302, 3302, 3302, 3302, 3302, 3302,
- 3302, 3302, 3302, 3303, 3303, 3303, 3303, 3303, 3303, 3303,
- 3303, 3303, 3303, 3303, 3303, 3303, 3304, 3304, 3304, 3304,
- 3304, 3304, 3304, 3304, 3304, 3304, 3304, 3304, 3304, 3305,
- 3305, 3305, 3305, 3305, 3305, 3305, 3305, 3305, 3305, 3305,
-
- 3305, 3305, 3306, 3306, 3306, 3306, 3306, 3306, 3306, 3306,
- 3306, 3306, 3306, 3306, 3306, 3307, 3307, 3307, 3307, 3307,
- 3307, 3307, 3307, 3307, 3307, 3307, 3307, 3307, 3308, 3308,
+ 3198, 3198, 3198, 3199, 3199, 3199, 3199, 3199, 3199, 3199,
+ 3199, 3199, 3199, 3199, 3199, 3199, 3200, 3200, 3200, 3200,
+ 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3201,
+ 3201, 3201, 3201, 3201, 3201, 3201, 3201, 3201, 3201, 3201,
+ 3201, 3201, 3202, 3202, 0, 3202, 3202, 3202, 3202, 3202,
+ 3202, 3202, 3202, 3202, 3202, 3203, 3203, 0, 0, 3203,
+ 3203, 3203, 3203, 3203, 3203, 3204, 3204, 0, 3204, 3204,
+ 3204, 3204, 3204, 3204, 3204, 3204, 3204, 3204, 3205, 3205,
+
+ 0, 3205, 3205, 3205, 3205, 3205, 3205, 3205, 3205, 3205,
+ 3205, 3206, 3206, 0, 3206, 3206, 3206, 3206, 3206, 3206,
+ 3206, 3206, 3206, 3206, 3207, 3207, 0, 0, 3207, 3207,
+ 3207, 3207, 3207, 3207, 3208, 3208, 0, 3208, 3208, 3208,
+ 3208, 3208, 3208, 3208, 3208, 3208, 3208, 3209, 3209, 0,
+ 3209, 3209, 3209, 3209, 3209, 3209, 3209, 3209, 3209, 3209,
+ 3210, 3210, 0, 3210, 3210, 3210, 3210, 3210, 3210, 3210,
+ 3210, 3210, 3210, 3211, 3211, 0, 0, 3211, 3211, 3211,
+ 3211, 3211, 3211, 3212, 3212, 0, 3212, 3212, 3212, 3212,
+ 3212, 3212, 3212, 3212, 3212, 3212, 3213, 3213, 0, 3213,
+
+ 3213, 3213, 3213, 3213, 3213, 3213, 3213, 3213, 3213, 3214,
+ 3214, 0, 3214, 3214, 3214, 3214, 3214, 3214, 3214, 3214,
+ 3214, 3214, 3215, 3215, 0, 0, 3215, 3215, 3215, 3215,
+ 3215, 3215, 3216, 3216, 0, 3216, 3216, 3216, 3216, 3216,
+ 3216, 3216, 3216, 3216, 3216, 3217, 3217, 0, 3217, 3217,
+ 3217, 3217, 3217, 3217, 3217, 3217, 3217, 3217, 3218, 3218,
+ 0, 3218, 3218, 3218, 3218, 3218, 3218, 3218, 3218, 3218,
+ 3218, 3219, 0, 3219, 3219, 0, 3219, 3219, 3219, 3219,
+ 3219, 3219, 3219, 3219, 3220, 0, 3220, 3220, 0, 3220,
+ 3220, 3220, 0, 3220, 3220, 3220, 3220, 3221, 3221, 0,
+
+ 0, 3221, 3221, 3221, 3221, 3221, 3221, 3222, 3222, 0,
+ 3222, 3222, 3222, 3222, 3222, 3222, 3222, 3222, 3222, 3222,
+ 3223, 3223, 0, 3223, 3223, 3223, 3223, 3223, 3223, 3223,
+ 3223, 3223, 3223, 3224, 3224, 0, 3224, 3224, 3224, 3224,
+ 3224, 3224, 3224, 3224, 3224, 3224, 3225, 0, 3225, 3225,
+ 0, 3225, 3225, 3225, 3225, 3225, 3225, 3225, 3225, 3226,
+ 3226, 0, 0, 3226, 3226, 3226, 3226, 3226, 3226, 3227,
+ 3227, 0, 3227, 3227, 3227, 3227, 3227, 3227, 3227, 3227,
+ 3227, 3227, 3228, 3228, 0, 3228, 3228, 3228, 3228, 3228,
+ 3228, 3228, 3228, 3228, 3228, 3229, 3229, 0, 3229, 3229,
+
+ 3229, 3229, 3229, 3229, 3229, 3229, 3229, 3229, 3230, 0,
+ 3230, 3230, 0, 3230, 3230, 3230, 3230, 3230, 3230, 3230,
+ 3230, 3231, 3231, 0, 0, 3231, 3231, 3231, 3231, 3231,
+ 3231, 3232, 3232, 0, 3232, 3232, 3232, 3232, 3232, 3232,
+ 3232, 3232, 3232, 3232, 3233, 3233, 0, 3233, 3233, 3233,
+ 3233, 3233, 3233, 3233, 3233, 3233, 3233, 3234, 3234, 0,
+ 3234, 3234, 3234, 3234, 3234, 3234, 3234, 3234, 3234, 3234,
+ 3235, 3235, 0, 0, 3235, 3235, 3235, 3235, 3235, 3235,
+ 3236, 3236, 0, 3236, 3236, 3236, 3236, 3236, 3236, 3236,
+ 3236, 3236, 3236, 3237, 3237, 0, 3237, 3237, 3237, 3237,
+
+ 3237, 3237, 3237, 3237, 3237, 3237, 3238, 3238, 0, 3238,
+ 3238, 3238, 3238, 3238, 3238, 3238, 3238, 3238, 3238, 3239,
+ 3239, 0, 0, 3239, 3239, 3239, 3239, 3239, 3239, 3240,
+ 3240, 0, 3240, 3240, 3240, 3240, 3240, 3240, 3240, 3240,
+ 3240, 3240, 3241, 3241, 0, 3241, 3241, 3241, 3241, 3241,
+ 3241, 3241, 3241, 3241, 3241, 3242, 3242, 0, 3242, 3242,
+ 3242, 3242, 3242, 3242, 3242, 3242, 3242, 3242, 3243, 3243,
+ 0, 3243, 3243, 3243, 3243, 3243, 3243, 3243, 3243, 3243,
+ 3243, 3244, 3244, 0, 3244, 3244, 3244, 3244, 3244, 3244,
+ 3244, 3244, 3244, 3244, 3245, 3245, 0, 0, 3245, 3245,
+
+ 3245, 3245, 3245, 3245, 3246, 3246, 0, 3246, 3246, 3246,
+ 3246, 3246, 3246, 3246, 3246, 3246, 3246, 3247, 3247, 0,
+ 3247, 3247, 3247, 3247, 3247, 3247, 3247, 3247, 3247, 3247,
+ 3248, 3248, 0, 3248, 3248, 3248, 3248, 3248, 3248, 3248,
+ 3248, 3248, 3248, 3249, 3249, 0, 0, 3249, 3249, 3249,
+ 3249, 3249, 3249, 3250, 3250, 0, 3250, 3250, 3250, 3250,
+ 3250, 3250, 3250, 3250, 3250, 3250, 3251, 3251, 0, 3251,
+ 3251, 3251, 3251, 3251, 3251, 3251, 3251, 3251, 3251, 3252,
+ 3252, 0, 3252, 3252, 3252, 3252, 3252, 3252, 3252, 3252,
+ 3252, 3252, 3253, 3253, 0, 0, 3253, 3253, 3253, 3253,
+
+ 3253, 3253, 3254, 3254, 0, 3254, 3254, 3254, 3254, 3254,
+ 3254, 3254, 3254, 3254, 3254, 3255, 3255, 0, 3255, 3255,
+ 3255, 3255, 3255, 3255, 3255, 3255, 3255, 3255, 3256, 3256,
+ 0, 3256, 3256, 3256, 3256, 3256, 3256, 3256, 3256, 3256,
+ 3256, 3257, 3257, 0, 3257, 3257, 3257, 3257, 3257, 3257,
+ 3257, 3257, 3257, 3257, 3258, 3258, 0, 3258, 3258, 3258,
+ 3258, 3258, 3258, 3258, 3258, 3258, 3258, 3259, 3259, 0,
+ 0, 3259, 3259, 3259, 3259, 3259, 3259, 3260, 3260, 0,
+ 3260, 3260, 3260, 3260, 3260, 3260, 3260, 3260, 3260, 3260,
+ 3261, 3261, 0, 3261, 3261, 3261, 3261, 3261, 3261, 3261,
+
+ 3261, 3261, 3261, 3262, 3262, 0, 3262, 3262, 3262, 3262,
+ 3262, 3262, 3262, 3262, 3262, 3262, 3263, 3263, 0, 0,
+ 3263, 3263, 3263, 3263, 3263, 3263, 3264, 3264, 0, 3264,
+ 3264, 3264, 3264, 3264, 3264, 3264, 3264, 3264, 3264, 3265,
+ 3265, 0, 3265, 3265, 3265, 3265, 3265, 3265, 3265, 3265,
+ 3265, 3265, 3266, 3266, 0, 3266, 3266, 3266, 3266, 3266,
+ 3266, 3266, 3266, 3266, 3266, 3267, 3267, 0, 0, 3267,
+ 3267, 3267, 3267, 3267, 3267, 3268, 3268, 0, 3268, 3268,
+ 3268, 3268, 3268, 3268, 3268, 3268, 3268, 3268, 3269, 3269,
+ 0, 3269, 3269, 3269, 3269, 3269, 3269, 3269, 3269, 3269,
+
+ 3269, 3270, 3270, 0, 3270, 3270, 3270, 3270, 3270, 3270,
+ 3270, 3270, 3270, 3270, 3271, 3271, 0, 0, 3271, 3271,
+ 3271, 3271, 3271, 3271, 3272, 3272, 0, 3272, 3272, 3272,
+ 3272, 3272, 3272, 3272, 3272, 3272, 3272, 3273, 3273, 0,
+ 3273, 3273, 3273, 3273, 3273, 3273, 3273, 3273, 3273, 3273,
+ 3274, 3274, 0, 3274, 3274, 3274, 3274, 3274, 3274, 3274,
+ 3274, 3274, 3274, 3275, 3275, 3275, 3275, 3275, 0, 3275,
+ 3275, 3275, 3275, 3276, 3276, 3276, 3276, 3276, 3276, 3276,
+ 3276, 3276, 3276, 3276, 3276, 3276, 3277, 3277, 0, 3277,
+ 3277, 3277, 3277, 3277, 3277, 3277, 3277, 3277, 3277, 3278,
+
+ 3278, 0, 3278, 3278, 3278, 3278, 3278, 3278, 3278, 3278,
+ 3278, 3278, 3279, 3279, 0, 3279, 3279, 3279, 3279, 3279,
+ 3279, 3279, 3279, 3279, 3279, 3280, 3280, 0, 3280, 3280,
+ 3280, 3280, 3280, 3280, 3280, 3280, 3280, 3280, 3281, 3281,
+ 0, 3281, 3281, 3281, 3281, 3281, 3281, 3281, 3281, 3281,
+ 3281, 3282, 0, 3282, 3282, 0, 3282, 3282, 3282, 3282,
+ 3282, 3282, 3282, 3282, 3283, 3283, 0, 3283, 3283, 3283,
+ 3283, 3283, 3283, 3283, 3283, 3283, 3283, 3284, 3284, 0,
+ 3284, 3284, 3284, 3284, 3284, 3284, 3284, 3284, 3284, 3284,
+ 3285, 3285, 3285, 3285, 3285, 3285, 3285, 3285, 3285, 3285,
+
+ 3285, 3285, 3285, 3286, 3286, 0, 3286, 3286, 3286, 3286,
+ 3286, 3286, 3286, 3286, 3286, 3286, 3287, 3287, 0, 3287,
+ 3287, 3287, 3287, 3287, 3287, 3287, 3287, 3287, 3287, 3288,
+ 3288, 0, 3288, 3288, 3288, 3288, 3288, 3288, 3288, 3288,
+ 3288, 3288, 3289, 3289, 0, 3289, 3289, 3289, 3289, 3289,
+ 3289, 3289, 3289, 3289, 3289, 3290, 3290, 0, 3290, 3290,
+ 3290, 3290, 3290, 3290, 3290, 3290, 3290, 3290, 3291, 3291,
+ 0, 3291, 3291, 3291, 3291, 3291, 3291, 3291, 3291, 3291,
+ 3291, 3292, 3292, 0, 3292, 3292, 3292, 3292, 3292, 3292,
+ 3292, 3292, 3292, 3292, 3293, 3293, 0, 3293, 3293, 3293,
+
+ 3293, 3293, 3293, 3293, 3293, 3293, 3293, 3294, 3294, 0,
+ 3294, 3294, 3294, 3294, 3294, 3294, 3294, 3294, 3294, 3294,
+ 3295, 3295, 0, 3295, 3295, 3295, 3295, 3295, 3295, 3295,
+ 3295, 3295, 3295, 3296, 3296, 0, 3296, 3296, 3296, 3296,
+ 3296, 3296, 3296, 3296, 3296, 3296, 3297, 3297, 0, 3297,
+ 3297, 3297, 3297, 3297, 3297, 3297, 3297, 3297, 3297, 3298,
+ 3298, 0, 3298, 3298, 3298, 3298, 3298, 3298, 3298, 3298,
+ 3298, 3298, 3299, 3299, 0, 3299, 3299, 3299, 3299, 3299,
+ 3299, 3299, 3299, 3299, 3299, 3300, 3300, 0, 3300, 3300,
+ 3300, 3300, 3300, 3300, 3300, 3300, 3300, 3300, 3301, 3301,
+
+ 0, 3301, 3301, 3301, 3301, 3301, 3301, 3301, 3301, 3301,
+ 3301, 3302, 3302, 0, 3302, 3302, 3302, 3302, 3302, 3302,
+ 3302, 3302, 3302, 3302, 3303, 3303, 0, 3303, 3303, 3303,
+ 3303, 3303, 3303, 3303, 3303, 3303, 3303, 3304, 3304, 0,
+ 3304, 3304, 3304, 3304, 3304, 3304, 3304, 3304, 3304, 3304,
+ 3305, 3305, 0, 3305, 3305, 3305, 3305, 3305, 3305, 3305,
+ 3305, 3305, 3305, 3306, 3306, 3306, 3306, 3306, 3306, 3306,
+ 3306, 3306, 3306, 3306, 3306, 3306, 3307, 3307, 3307, 3307,
+ 3307, 3307, 3307, 3307, 3307, 3307, 3307, 3307, 3307, 3308,
3308, 3308, 3308, 3308, 3308, 3308, 3308, 3308, 3308, 3308,
- 3308, 3309, 3309, 3309, 3309, 3309, 3309, 3309, 3309, 3309,
- 3309, 3309, 3309, 3309, 3310, 3310, 3310, 3310, 3310, 3310,
- 3310, 3310, 3310, 3310, 3310, 3310, 3310, 3311, 3311, 0,
- 3311, 3311, 3311, 3311, 3311, 3311, 3311, 3311, 3311, 3311,
- 3312, 3312, 3312, 3312, 3312, 3312, 3312, 3312, 3312, 3312,
- 3312, 3312, 3312, 3313, 3313, 3313, 3313, 3313, 3313, 3313,
-
- 3313, 3313, 3313, 3313, 3313, 3313, 3314, 3314, 3314, 3314,
- 3314, 3314, 3314, 3314, 3314, 3314, 3314, 3314, 3314, 3315,
- 3315, 0, 3315, 3315, 3315, 3315, 3315, 3315, 3315, 3315,
- 3315, 3315, 3316, 3316, 0, 3316, 3316, 3316, 3316, 3316,
- 3316, 3316, 3316, 3316, 3316, 3317, 3317, 3317, 3317, 3317,
- 3317, 3317, 3317, 3317, 3317, 3317, 3317, 3317, 3318, 3318,
- 3318, 3318, 3318, 3318, 3318, 3318, 3318, 3318, 3318, 3318,
- 3318, 3319, 3319, 3319, 3319, 3319, 3319, 3319, 3319, 3319,
- 3319, 3319, 3319, 3319, 3320, 3320, 3320, 3320, 3320, 3320,
- 3320, 3320, 3320, 3320, 3320, 3320, 3320, 3321, 3321, 3321,
+ 3308, 3308, 3309, 3309, 3309, 3309, 3309, 3309, 3309, 3309,
+ 3309, 3309, 3309, 3309, 3309, 3310, 3310, 3310, 3310, 3310,
+ 3310, 3310, 3310, 3310, 3310, 3310, 3310, 3310, 3311, 3311,
+ 3311, 3311, 3311, 3311, 3311, 3311, 3311, 3311, 3311, 3311,
+ 3311, 3312, 3312, 3312, 3312, 3312, 3312, 3312, 3312, 3312,
+ 3312, 3312, 3312, 3312, 3313, 3313, 3313, 3313, 3313, 3313,
+ 3313, 3313, 3313, 3313, 3313, 3313, 3313, 3314, 3314, 0,
+ 3314, 3314, 3314, 3314, 3314, 3314, 3314, 3314, 3314, 3314,
+ 3315, 3315, 3315, 3315, 3315, 3315, 3315, 3315, 3315, 3315,
+ 3315, 3315, 3315, 3316, 3316, 3316, 3316, 3316, 3316, 3316,
+
+ 3316, 3316, 3316, 3316, 3316, 3316, 3317, 3317, 3317, 3317,
+ 3317, 3317, 3317, 3317, 3317, 3317, 3317, 3317, 3317, 3318,
+ 3318, 0, 3318, 3318, 3318, 3318, 3318, 3318, 3318, 3318,
+ 3318, 3318, 3319, 3319, 0, 3319, 3319, 3319, 3319, 3319,
+ 3319, 3319, 3319, 3319, 3319, 3320, 3320, 3320, 3320, 3320,
+ 3320, 3320, 3320, 3320, 3320, 3320, 3320, 3320, 3321, 3321,
3321, 3321, 3321, 3321, 3321, 3321, 3321, 3321, 3321, 3321,
- 3322, 3322, 3322, 3322, 3322, 3322, 3322, 3322, 3322, 3322,
- 3322, 3322, 3322, 3323, 3323, 3323, 3323, 3323, 3323, 3323,
- 3323, 3323, 3323, 3323, 3323, 3323, 3324, 3324, 3324, 3324,
- 3324, 3324, 3324, 3324, 3324, 3324, 3324, 3324, 3324, 3325,
- 3325, 0, 3325, 3325, 3325, 3325, 3325, 3325, 3325, 3325,
- 3325, 3325, 3326, 3326, 3326, 3326, 3326, 3326, 3326, 3326,
- 3326, 3326, 3326, 3326, 3326, 3327, 3327, 3327, 3327, 3327,
- 3327, 3327, 3327, 3327, 3327, 3327, 3327, 3327, 3328, 3328,
- 3328, 3328, 3328, 3328, 3328, 3328, 3328, 3328, 3328, 3328,
-
- 3328, 3330, 3330, 0, 3330, 3330, 3330, 3330, 3330, 3330,
- 3330, 3330, 3330, 3330, 3331, 3331, 0, 3331, 3331, 3331,
- 3331, 3331, 3331, 3331, 3331, 3331, 3331, 3332, 3332, 0,
- 3332, 3332, 3332, 3332, 3332, 3332, 3332, 3332, 3332, 3332,
- 3333, 3333, 0, 3333, 3333, 3333, 3333, 3333, 3333, 3333,
- 3333, 3333, 3333, 3334, 3334, 0, 3334, 3334, 3334, 3334,
- 3334, 3334, 3334, 3334, 3334, 3334, 3335, 3335, 0, 3335,
- 3335, 3335, 3335, 3335, 3335, 3335, 3335, 3335, 3335, 3336,
- 3336, 0, 3336, 3336, 3336, 3336, 3336, 3336, 3336, 3336,
- 3336, 3336, 3337, 3337, 0, 3337, 3337, 3337, 3337, 3337,
-
- 3337, 3337, 3337, 3337, 3337, 3338, 3338, 0, 3338, 3338,
- 3338, 3338, 3338, 3338, 3338, 3338, 3338, 3338, 3339, 3339,
- 0, 3339, 3339, 3339, 3339, 3339, 3339, 3339, 3339, 3339,
- 3339, 3340, 3340, 0, 3340, 3340, 3340, 3340, 3340, 3340,
- 3340, 3340, 3340, 3340, 3341, 3341, 0, 3341, 3341, 3341,
- 3341, 3341, 3341, 3341, 3341, 3341, 3341, 3342, 3342, 0,
- 3342, 3342, 3342, 3342, 3342, 3342, 3342, 3342, 3342, 3342,
- 3343, 3343, 0, 3343, 3343, 3343, 3343, 3343, 3343, 3343,
- 3343, 3343, 3343, 3344, 3344, 0, 3344, 3344, 3344, 3344,
- 3344, 3344, 3344, 3344, 3344, 3344, 3345, 3345, 0, 3345,
-
- 3345, 3345, 3345, 3345, 3345, 3345, 3345, 3345, 3345, 3346,
- 3346, 0, 3346, 3346, 3346, 3346, 3346, 3346, 3346, 3346,
- 3346, 3346, 3347, 3347, 0, 3347, 3347, 3347, 3347, 3347,
- 3347, 3347, 3347, 3347, 3347, 3348, 3348, 0, 3348, 3348,
- 3348, 3348, 3348, 3348, 3348, 3348, 3348, 3348, 3349, 3349,
- 3349, 3349, 3349, 0, 3349, 3349, 3349, 3349, 3350, 3350,
- 3350, 3350, 3350, 3350, 3350, 3350, 3350, 3350, 3350, 3350,
- 3350, 3351, 3351, 3351, 3351, 3351, 3351, 3351, 3351, 3351,
- 3351, 3351, 3351, 3351, 3352, 3352, 3352, 3352, 3352, 3352,
- 3352, 3352, 3352, 3352, 3352, 3352, 3352, 3353, 3353, 3353,
-
+ 3321, 3322, 3322, 3322, 3322, 3322, 3322, 3322, 3322, 3322,
+ 3322, 3322, 3322, 3322, 3323, 3323, 3323, 3323, 3323, 3323,
+ 3323, 3323, 3323, 3323, 3323, 3323, 3323, 3324, 3324, 3324,
+
+ 3324, 3324, 3324, 3324, 3324, 3324, 3324, 3324, 3324, 3324,
+ 3325, 3325, 3325, 3325, 3325, 3325, 3325, 3325, 3325, 3325,
+ 3325, 3325, 3325, 3326, 3326, 3326, 3326, 3326, 3326, 3326,
+ 3326, 3326, 3326, 3326, 3326, 3326, 3327, 3327, 3327, 3327,
+ 3327, 3327, 3327, 3327, 3327, 3327, 3327, 3327, 3327, 3328,
+ 3328, 0, 3328, 3328, 3328, 3328, 3328, 3328, 3328, 3328,
+ 3328, 3328, 3329, 3329, 3329, 3329, 3329, 3329, 3329, 3329,
+ 3329, 3329, 3329, 3329, 3329, 3330, 3330, 3330, 3330, 3330,
+ 3330, 3330, 3330, 3330, 3330, 3330, 3330, 3330, 3331, 3331,
+ 3331, 3331, 3331, 3331, 3331, 3331, 3331, 3331, 3331, 3331,
+
+ 3331, 3333, 3333, 0, 3333, 3333, 3333, 3333, 3333, 3333,
+ 3333, 3333, 3333, 3333, 3334, 3334, 0, 3334, 3334, 3334,
+ 3334, 3334, 3334, 3334, 3334, 3334, 3334, 3335, 3335, 0,
+ 3335, 3335, 3335, 3335, 3335, 3335, 3335, 3335, 3335, 3335,
+ 3336, 3336, 0, 3336, 3336, 3336, 3336, 3336, 3336, 3336,
+ 3336, 3336, 3336, 3337, 3337, 0, 3337, 3337, 3337, 3337,
+ 3337, 3337, 3337, 3337, 3337, 3337, 3338, 3338, 0, 3338,
+ 3338, 3338, 3338, 3338, 3338, 3338, 3338, 3338, 3338, 3339,
+ 3339, 0, 3339, 3339, 3339, 3339, 3339, 3339, 3339, 3339,
+ 3339, 3339, 3340, 3340, 0, 3340, 3340, 3340, 3340, 3340,
+
+ 3340, 3340, 3340, 3340, 3340, 3341, 3341, 0, 3341, 3341,
+ 3341, 3341, 3341, 3341, 3341, 3341, 3341, 3341, 3342, 3342,
+ 0, 3342, 3342, 3342, 3342, 3342, 3342, 3342, 3342, 3342,
+ 3342, 3343, 3343, 0, 3343, 3343, 3343, 3343, 3343, 3343,
+ 3343, 3343, 3343, 3343, 3344, 3344, 0, 3344, 3344, 3344,
+ 3344, 3344, 3344, 3344, 3344, 3344, 3344, 3345, 3345, 0,
+ 3345, 3345, 3345, 3345, 3345, 3345, 3345, 3345, 3345, 3345,
+ 3346, 3346, 0, 3346, 3346, 3346, 3346, 3346, 3346, 3346,
+ 3346, 3346, 3346, 3347, 3347, 0, 3347, 3347, 3347, 3347,
+ 3347, 3347, 3347, 3347, 3347, 3347, 3348, 3348, 0, 3348,
+
+ 3348, 3348, 3348, 3348, 3348, 3348, 3348, 3348, 3348, 3349,
+ 3349, 0, 3349, 3349, 3349, 3349, 3349, 3349, 3349, 3349,
+ 3349, 3349, 3350, 3350, 0, 3350, 3350, 3350, 3350, 3350,
+ 3350, 3350, 3350, 3350, 3350, 3351, 3351, 0, 3351, 3351,
+ 3351, 3351, 3351, 3351, 3351, 3351, 3351, 3351, 3352, 3352,
+ 3352, 3352, 3352, 0, 3352, 3352, 3352, 3352, 3353, 3353,
3353, 3353, 3353, 3353, 3353, 3353, 3353, 3353, 3353, 3353,
- 3354, 3354, 0, 3354, 3354, 3354, 3354, 3354, 3354, 3354,
- 3354, 3354, 3354, 3355, 3355, 0, 3355, 3355, 3355, 3355,
- 3355, 3355, 3355, 3355, 3355, 3355, 3356, 3356, 3356, 3356,
- 3356, 3356, 3356, 3356, 3356, 3356, 3356, 3356, 3356, 3357,
- 3357, 3357, 3357, 3357, 3357, 3357, 3357, 3357, 3357, 3357,
- 3357, 3357, 3358, 3358, 3358, 3358, 3358, 3358, 3358, 3358,
- 3358, 3358, 3358, 3358, 3358, 3359, 3359, 0, 3359, 3359,
- 3359, 3359, 3359, 3359, 3359, 3359, 3359, 3359, 3360, 3360,
+ 3353, 3354, 3354, 3354, 3354, 3354, 3354, 3354, 3354, 3354,
+ 3354, 3354, 3354, 3354, 3355, 3355, 3355, 3355, 3355, 3355,
+ 3355, 3355, 3355, 3355, 3355, 3355, 3355, 3356, 3356, 3356,
+
+ 3356, 3356, 3356, 3356, 3356, 3356, 3356, 3356, 3356, 3356,
+ 3357, 3357, 0, 3357, 3357, 3357, 3357, 3357, 3357, 3357,
+ 3357, 3357, 3357, 3358, 3358, 0, 3358, 3358, 3358, 3358,
+ 3358, 3358, 3358, 3358, 3358, 3358, 3359, 3359, 3359, 3359,
+ 3359, 3359, 3359, 3359, 3359, 3359, 3359, 3359, 3359, 3360,
3360, 3360, 3360, 3360, 3360, 3360, 3360, 3360, 3360, 3360,
-
- 3360, 3361, 3361, 3361, 3361, 3361, 3361, 3361, 3361, 3361,
- 3361, 3361, 3361, 3361, 3362, 3362, 3362, 3362, 3362, 3362,
- 3362, 3362, 3362, 3362, 3362, 3362, 3362, 3363, 3363, 3363,
+ 3360, 3360, 3361, 3361, 3361, 3361, 3361, 3361, 3361, 3361,
+ 3361, 3361, 3361, 3361, 3361, 3362, 3362, 0, 3362, 3362,
+ 3362, 3362, 3362, 3362, 3362, 3362, 3362, 3362, 3363, 3363,
3363, 3363, 3363, 3363, 3363, 3363, 3363, 3363, 3363, 3363,
- 3364, 3364, 3364, 3364, 3364, 3364, 3364, 3364, 3364, 3364,
- 3364, 3364, 3364, 3365, 3365, 3365, 3365, 3365, 3365, 3365,
- 3365, 3365, 3365, 3365, 3365, 3365, 3366, 3366, 3366, 3366,
- 3366, 3366, 3366, 3366, 3366, 3366, 3366, 3366, 3366, 3367,
- 3367, 3367, 3367, 3367, 3367, 3367, 3367, 3367, 3367, 3367,
- 3367, 3367, 3368, 3368, 3368, 3368, 3368, 3368, 3368, 3368,
- 3368, 3368, 3368, 3368, 3368, 3369, 3369, 3369, 3369, 3369,
- 3369, 3369, 3369, 3369, 3369, 3369, 3369, 3369, 3370, 3370,
+ 3363, 3364, 3364, 3364, 3364, 3364, 3364, 3364, 3364, 3364,
+ 3364, 3364, 3364, 3364, 3365, 3365, 3365, 3365, 3365, 3365,
+ 3365, 3365, 3365, 3365, 3365, 3365, 3365, 3366, 3366, 3366,
+ 3366, 3366, 3366, 3366, 3366, 3366, 3366, 3366, 3366, 3366,
+ 3367, 3367, 3367, 3367, 3367, 3367, 3367, 3367, 3367, 3367,
+ 3367, 3367, 3367, 3368, 3368, 3368, 3368, 3368, 3368, 3368,
+ 3368, 3368, 3368, 3368, 3368, 3368, 3369, 3369, 3369, 3369,
+ 3369, 3369, 3369, 3369, 3369, 3369, 3369, 3369, 3369, 3370,
3370, 3370, 3370, 3370, 3370, 3370, 3370, 3370, 3370, 3370,
- 3370, 3371, 3371, 3371, 3371, 3371, 3371, 3371, 3371, 3371,
- 3371, 3371, 3371, 3371, 3372, 3372, 3372, 3372, 3372, 3372,
- 3372, 3372, 3372, 3372, 3372, 3372, 3372, 3373, 3373, 3373,
+ 3370, 3370, 3371, 3371, 3371, 3371, 3371, 3371, 3371, 3371,
+
+ 3371, 3371, 3371, 3371, 3371, 3372, 3372, 3372, 3372, 3372,
+ 3372, 3372, 3372, 3372, 3372, 3372, 3372, 3372, 3373, 3373,
3373, 3373, 3373, 3373, 3373, 3373, 3373, 3373, 3373, 3373,
- 3374, 3374, 3374, 3374, 3374, 3374, 3374, 3374, 3374, 3374,
- 3374, 3374, 3374, 3375, 3375, 3375, 3375, 3375, 3375, 3375,
- 3375, 3375, 3375, 3375, 3375, 3375, 3376, 3376, 3376, 3376,
-
- 3376, 3376, 3376, 3376, 3376, 3376, 3376, 3376, 3376, 3377,
- 3377, 0, 3377, 3377, 3377, 3377, 3377, 3377, 3377, 3377,
- 3377, 3377, 3378, 3378, 3378, 3378, 3378, 3378, 3378, 3378,
- 3378, 3378, 3378, 3378, 3378, 3379, 3379, 3379, 3379, 3379,
- 3379, 3379, 3379, 3379, 3379, 3379, 3379, 3379, 3380, 3380,
- 3380, 3380, 3380, 3380, 3380, 3380, 3380, 3380, 3380, 3380,
- 3380, 3381, 3381, 3381, 3381, 3381, 3381, 3381, 3381, 3381,
- 3381, 3381, 3381, 3381, 3382, 3382, 3382, 3382, 3382, 3382,
- 3382, 3382, 3382, 3382, 3382, 3382, 3382, 3383, 3383, 3383,
+ 3373, 3374, 3374, 3374, 3374, 3374, 3374, 3374, 3374, 3374,
+ 3374, 3374, 3374, 3374, 3375, 3375, 3375, 3375, 3375, 3375,
+ 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3376, 3376, 3376,
+ 3376, 3376, 3376, 3376, 3376, 3376, 3376, 3376, 3376, 3376,
+ 3377, 3377, 3377, 3377, 3377, 3377, 3377, 3377, 3377, 3377,
+ 3377, 3377, 3377, 3378, 3378, 3378, 3378, 3378, 3378, 3378,
+ 3378, 3378, 3378, 3378, 3378, 3378, 3379, 3379, 3379, 3379,
+
+ 3379, 3379, 3379, 3379, 3379, 3379, 3379, 3379, 3379, 3380,
+ 3380, 0, 3380, 3380, 3380, 3380, 3380, 3380, 3380, 3380,
+ 3380, 3380, 3381, 3381, 3381, 3381, 3381, 3381, 3381, 3381,
+ 3381, 3381, 3381, 3381, 3381, 3382, 3382, 3382, 3382, 3382,
+ 3382, 3382, 3382, 3382, 3382, 3382, 3382, 3382, 3383, 3383,
3383, 3383, 3383, 3383, 3383, 3383, 3383, 3383, 3383, 3383,
+ 3383, 3384, 3384, 3384, 3384, 3384, 3384, 3384, 3384, 3384,
+ 3384, 3384, 3384, 3384, 3385, 3385, 3385, 3385, 3385, 3385,
+ 3385, 3385, 3385, 3385, 3385, 3385, 3385, 3386, 3386, 3386,
+ 3386, 3386, 3386, 3386, 3386, 3386, 3386, 3386, 3386, 3386,
- 3384, 3384, 3384, 3384, 3384, 3384, 3384, 3384, 3384, 3384,
- 3384, 3384, 3384, 3385, 3385, 3385, 3385, 3385, 3385, 3385,
- 3385, 3385, 3385, 3385, 3385, 3385, 3386, 3386, 3386, 3386,
- 3386, 3386, 3386, 3386, 3386, 3386, 3386, 3386, 3386, 3387,
3387, 3387, 3387, 3387, 3387, 3387, 3387, 3387, 3387, 3387,
- 3387, 3387, 3388, 3388, 3388, 3388, 3388, 3388, 3388, 3388,
- 3388, 3388, 3388, 3388, 3388, 3389, 3389, 3389, 3389, 3389,
- 3389, 3389, 3389, 3389, 3389, 3389, 3389, 3389, 3390, 3390,
+ 3387, 3387, 3387, 3388, 3388, 3388, 3388, 3388, 3388, 3388,
+ 3388, 3388, 3388, 3388, 3388, 3388, 3389, 3389, 3389, 3389,
+ 3389, 3389, 3389, 3389, 3389, 3389, 3389, 3389, 3389, 3390,
3390, 3390, 3390, 3390, 3390, 3390, 3390, 3390, 3390, 3390,
- 3390, 3391, 3391, 3391, 3391, 3391, 3391, 3391, 3391, 3391,
-
- 3391, 3391, 3391, 3391, 3392, 3392, 3392, 3392, 3392, 3392,
- 3392, 3392, 3392, 3392, 3392, 3392, 3392, 3393, 3393, 3393,
+ 3390, 3390, 3391, 3391, 3391, 3391, 3391, 3391, 3391, 3391,
+ 3391, 3391, 3391, 3391, 3391, 3392, 3392, 3392, 3392, 3392,
+ 3392, 3392, 3392, 3392, 3392, 3392, 3392, 3392, 3393, 3393,
3393, 3393, 3393, 3393, 3393, 3393, 3393, 3393, 3393, 3393,
- 3394, 3394, 3394, 3394, 3394, 3394, 3394, 3394, 3394, 3394,
- 3394, 3394, 3394, 3395, 3395, 3395, 3395, 3395, 0, 3395,
- 3395, 3395, 3395, 3396, 3396, 0, 3396, 3396, 3396, 3396,
- 3396, 3396, 3396, 3396, 3396, 3396, 3397, 3397, 0, 3397,
- 3397, 3397, 3397, 3397, 3397, 3397, 3397, 3397, 3397, 3398,
- 3398, 3398, 3398, 3398, 3398, 3398, 3398, 3398, 3398, 3398,
- 3398, 3398, 3399, 3399, 3399, 3399, 3399, 3399, 3399, 3399,
-
- 3399, 3399, 3399, 3399, 3399, 3400, 3400, 3400, 3400, 3400,
- 3400, 3400, 3400, 3400, 3400, 3400, 3400, 3400, 3401, 3401,
- 3401, 3401, 3401, 3401, 3401, 3401, 0, 3401, 3401, 3401,
- 3401, 3402, 3402, 3402, 3402, 3402, 3402, 3402, 3402, 3402,
- 3402, 3402, 3402, 3402, 3403, 3403, 3403, 3403, 3403, 3403,
- 3403, 3403, 3403, 3403, 3403, 3403, 3403, 3404, 3404, 3404,
- 3404, 3404, 3404, 3404, 3404, 0, 3404, 3404, 3404, 3404,
- 3405, 3405, 3405, 3405, 3405, 3405, 3405, 3405, 3405, 3405,
- 3405, 3405, 3405, 3406, 3406, 3406, 3406, 3406, 3406, 3406,
- 3406, 3406, 3406, 3406, 3406, 3406, 3407, 3407, 3407, 3407,
-
- 3407, 3407, 3407, 3407, 3407, 3407, 3407, 3407, 3407, 3408,
+ 3393, 3394, 3394, 3394, 3394, 3394, 3394, 3394, 3394, 3394,
+
+ 3394, 3394, 3394, 3394, 3395, 3395, 3395, 3395, 3395, 3395,
+ 3395, 3395, 3395, 3395, 3395, 3395, 3395, 3396, 3396, 3396,
+ 3396, 3396, 3396, 3396, 3396, 3396, 3396, 3396, 3396, 3396,
+ 3397, 3397, 3397, 3397, 3397, 3397, 3397, 3397, 3397, 3397,
+ 3397, 3397, 3397, 3398, 3398, 3398, 3398, 3398, 0, 3398,
+ 3398, 3398, 3398, 3399, 3399, 0, 3399, 3399, 3399, 3399,
+ 3399, 3399, 3399, 3399, 3399, 3399, 3400, 3400, 0, 3400,
+ 3400, 3400, 3400, 3400, 3400, 3400, 3400, 3400, 3400, 3401,
+ 3401, 3401, 3401, 3401, 3401, 3401, 3401, 3401, 3401, 3401,
+ 3401, 3401, 3402, 3402, 3402, 3402, 3402, 3402, 3402, 3402,
+
+ 3402, 3402, 3402, 3402, 3402, 3403, 3403, 3403, 3403, 3403,
+ 3403, 3403, 3403, 3403, 3403, 3403, 3403, 3403, 3404, 3404,
+ 3404, 3404, 3404, 3404, 3404, 3404, 0, 3404, 3404, 3404,
+ 3404, 3405, 3405, 3405, 3405, 3405, 3405, 3405, 3405, 3405,
+ 3405, 3405, 3405, 3405, 3406, 3406, 3406, 3406, 3406, 3406,
+ 3406, 3406, 3406, 3406, 3406, 3406, 3406, 3407, 3407, 3407,
+ 3407, 3407, 3407, 3407, 3407, 0, 3407, 3407, 3407, 3407,
3408, 3408, 3408, 3408, 3408, 3408, 3408, 3408, 3408, 3408,
- 3408, 3408, 3409, 3409, 3409, 3409, 3409, 3409, 3409, 3409,
- 3409, 3409, 3409, 3409, 3409, 3410, 3410, 3410, 3410, 3410,
- 3410, 3410, 3410, 3410, 3410, 3410, 3410, 3410, 3411, 3411,
+ 3408, 3408, 3408, 3409, 3409, 3409, 3409, 3409, 3409, 3409,
+ 3409, 3409, 3409, 3409, 3409, 3409, 3410, 3410, 3410, 3410,
+
+ 3410, 3410, 3410, 3410, 3410, 3410, 3410, 3410, 3410, 3411,
3411, 3411, 3411, 3411, 3411, 3411, 3411, 3411, 3411, 3411,
- 3411, 3412, 3412, 3412, 3412, 3412, 3412, 3412, 3412, 3412,
- 3412, 3412, 3412, 3412, 3413, 3413, 3413, 3413, 3413, 3413,
- 3413, 3413, 3413, 3413, 3413, 3413, 3413, 3414, 3414, 3414,
+ 3411, 3411, 3412, 3412, 3412, 3412, 3412, 3412, 3412, 3412,
+ 3412, 3412, 3412, 3412, 3412, 3413, 3413, 3413, 3413, 3413,
+ 3413, 3413, 3413, 3413, 3413, 3413, 3413, 3413, 3414, 3414,
3414, 3414, 3414, 3414, 3414, 3414, 3414, 3414, 3414, 3414,
+ 3414, 3415, 3415, 3415, 3415, 3415, 3415, 3415, 3415, 3415,
+ 3415, 3415, 3415, 3415, 3416, 3416, 3416, 3416, 3416, 3416,
+ 3416, 3416, 3416, 3416, 3416, 3416, 3416, 3417, 3417, 3417,
+ 3417, 3417, 3417, 3417, 3417, 3417, 3417, 3417, 3417, 3417,
- 3415, 3415, 3415, 3415, 3415, 3415, 3415, 3415, 3415, 3415,
- 3415, 3415, 3415, 3416, 3416, 3416, 3416, 3416, 3416, 3416,
- 3416, 3416, 3416, 3416, 3416, 3416, 3417, 3417, 3417, 3417,
- 3417, 3417, 3417, 3417, 3417, 3417, 3417, 3417, 3417, 3418,
3418, 3418, 3418, 3418, 3418, 3418, 3418, 3418, 3418, 3418,
- 3418, 3418, 3419, 3419, 3419, 3419, 3419, 3419, 3419, 3419,
- 3419, 3419, 3419, 3419, 3419, 3420, 3420, 3420, 3420, 3420,
- 3420, 3420, 3420, 3420, 3420, 3420, 3420, 3420, 3421, 3421,
+ 3418, 3418, 3418, 3419, 3419, 3419, 3419, 3419, 3419, 3419,
+ 3419, 3419, 3419, 3419, 3419, 3419, 3420, 3420, 3420, 3420,
+ 3420, 3420, 3420, 3420, 3420, 3420, 3420, 3420, 3420, 3421,
3421, 3421, 3421, 3421, 3421, 3421, 3421, 3421, 3421, 3421,
- 3421, 3422, 3422, 3422, 3422, 3422, 3422, 3422, 3422, 3422,
-
- 3422, 3422, 3422, 3422, 3423, 3423, 3423, 3423, 3423, 3423,
- 3423, 3423, 3423, 3423, 3423, 3423, 3423, 3424, 3424, 3424,
+ 3421, 3421, 3422, 3422, 3422, 3422, 3422, 3422, 3422, 3422,
+ 3422, 3422, 3422, 3422, 3422, 3423, 3423, 3423, 3423, 3423,
+ 3423, 3423, 3423, 3423, 3423, 3423, 3423, 3423, 3424, 3424,
3424, 3424, 3424, 3424, 3424, 3424, 3424, 3424, 3424, 3424,
- 3425, 3425, 3425, 3425, 3425, 3425, 3425, 3425, 3425, 3425,
- 3425, 3425, 3425, 3426, 3426, 3426, 3426, 3426, 3426, 3426,
- 3426, 3426, 3426, 3426, 3426, 3426, 3427, 3427, 3427, 3427,
- 3427, 3427, 3427, 3427, 3427, 3427, 3427, 3427, 3427, 3428,
- 3428, 3428, 3428, 3428, 3428, 3428, 3428, 3428, 3428, 3428,
- 3428, 3428, 3429, 3429, 3429, 3429, 3429, 3429, 3429, 3429,
- 3429, 3429, 3429, 3429, 3429, 3430, 3430, 3430, 3430, 3430,
+ 3424, 3425, 3425, 3425, 3425, 3425, 3425, 3425, 3425, 3425,
- 3430, 3430, 3430, 3430, 3430, 3430, 3430, 3430, 3431, 3431,
+ 3425, 3425, 3425, 3425, 3426, 3426, 3426, 3426, 3426, 3426,
+ 3426, 3426, 3426, 3426, 3426, 3426, 3426, 3427, 3427, 3427,
+ 3427, 3427, 3427, 3427, 3427, 3427, 3427, 3427, 3427, 3427,
+ 3428, 3428, 3428, 3428, 3428, 3428, 3428, 3428, 3428, 3428,
+ 3428, 3428, 3428, 3429, 3429, 3429, 3429, 3429, 3429, 3429,
+ 3429, 3429, 3429, 3429, 3429, 3429, 3430, 3430, 3430, 3430,
+ 3430, 3430, 3430, 3430, 3430, 3430, 3430, 3430, 3430, 3431,
3431, 3431, 3431, 3431, 3431, 3431, 3431, 3431, 3431, 3431,
- 3431, 3432, 3432, 3432, 3432, 3432, 3432, 3432, 3432, 3432,
- 3432, 3432, 3432, 3432, 3433, 3433, 3433, 3433, 3433, 3433,
- 3433, 3433, 3433, 3433, 3433, 3433, 3433, 3434, 3434, 3434,
- 3434, 3434, 3434, 3434, 3434, 3434, 3434, 3434, 3434, 3434,
- 3435, 3435, 3435, 3435, 3435, 3435, 3435, 3435, 3435, 3435,
- 3435, 3435, 3435, 3436, 3436, 3436, 3436, 3436, 3436, 3436,
- 3436, 3436, 3436, 3436, 3436, 3436, 3437, 3437, 3437, 3437,
- 3437, 3437, 3437, 3437, 3437, 3437, 3437, 3437, 3437, 3438,
+ 3431, 3431, 3432, 3432, 3432, 3432, 3432, 3432, 3432, 3432,
+ 3432, 3432, 3432, 3432, 3432, 3433, 3433, 3433, 3433, 3433,
+ 3433, 3433, 3433, 3433, 3433, 3433, 3433, 3433, 3434, 3434,
+ 3434, 3434, 3434, 3434, 3434, 3434, 3434, 3434, 3434, 3434,
+ 3434, 3435, 3435, 3435, 3435, 3435, 3435, 3435, 3435, 3435,
+ 3435, 3435, 3435, 3435, 3436, 3436, 3436, 3436, 3436, 3436,
+ 3436, 3436, 3436, 3436, 3436, 3436, 3436, 3437, 3437, 3437,
+ 3437, 3437, 3437, 3437, 3437, 3437, 3437, 3437, 3437, 3437,
3438, 3438, 3438, 3438, 3438, 3438, 3438, 3438, 3438, 3438,
- 3438, 3438, 3439, 3439, 3439, 3439, 3439, 3439, 3439, 3439,
- 3439, 3439, 3439, 3439, 3439, 3440, 3440, 3440, 3440, 3440,
- 3440, 3440, 3440, 3440, 3440, 3440, 3440, 3440, 3441, 3441,
+ 3438, 3438, 3438, 3439, 3439, 3439, 3439, 3439, 3439, 3439,
+ 3439, 3439, 3439, 3439, 3439, 3439, 3440, 3440, 3440, 3440,
+ 3440, 3440, 3440, 3440, 3440, 3440, 3440, 3440, 3440, 3441,
+
3441, 3441, 3441, 3441, 3441, 3441, 3441, 3441, 3441, 3441,
- 3441, 3442, 3442, 3442, 3442, 3442, 3442, 3442, 3442, 3442,
- 3442, 3442, 3442, 3443, 3443, 3443, 3443, 3443, 0, 3443,
- 3443, 3443, 3443, 3444, 3444, 3444, 3444, 3444, 3444, 3444,
- 3444, 0, 3444, 3444, 3444, 3444, 3445, 3445, 3445, 3445,
- 3445, 3445, 3445, 3445, 0, 3445, 3445, 3445, 3445, 3446,
-
- 3446, 0, 3446, 3446, 3446, 3446, 3446, 3446, 3446, 3446,
- 3446, 3446, 3447, 3447, 0, 3447, 3447, 3447, 3447, 3447,
- 3447, 3447, 3447, 3447, 3447, 3448, 3448, 3448, 3448, 3448,
- 3448, 3448, 3448, 3448, 3448, 3448, 3448, 3448, 3449, 3449,
- 3449, 3449, 3449, 3449, 3449, 3449, 0, 3449, 3449, 3449,
- 3449, 3450, 3450, 0, 3450, 3450, 3450, 3450, 3450, 3450,
- 3450, 3450, 3450, 3450, 3451, 3451, 3451, 3451, 3451, 3451,
- 3451, 3451, 3451, 3451, 3451, 3451, 3451, 3452, 3452, 3452,
- 3452, 3452, 3452, 3452, 3452, 0, 3452, 3452, 3452, 3452,
- 3453, 3453, 3453, 3453, 3453, 3453, 3453, 3453, 3453, 3453,
-
- 3453, 3453, 3453, 3454, 3454, 3454, 3454, 3454, 3454, 3454,
- 3454, 3454, 3454, 3454, 3454, 3454, 3455, 3455, 3455, 3455,
- 3455, 3455, 3455, 3455, 0, 3455, 3455, 3455, 3455, 3456,
+ 3441, 3441, 3442, 3442, 3442, 3442, 3442, 3442, 3442, 3442,
+ 3442, 3442, 3442, 3442, 3442, 3443, 3443, 3443, 3443, 3443,
+ 3443, 3443, 3443, 3443, 3443, 3443, 3443, 3443, 3444, 3444,
+ 3444, 3444, 3444, 3444, 3444, 3444, 3444, 3444, 3444, 3444,
+ 3444, 3445, 3445, 3445, 3445, 3445, 3445, 3445, 3445, 3445,
+ 3445, 3445, 3445, 3446, 3446, 3446, 3446, 3446, 0, 3446,
+ 3446, 3446, 3446, 3447, 3447, 3447, 3447, 3447, 3447, 3447,
+ 3447, 0, 3447, 3447, 3447, 3447, 3448, 3448, 3448, 3448,
+ 3448, 3448, 3448, 3448, 0, 3448, 3448, 3448, 3448, 3449,
+
+ 3449, 0, 3449, 3449, 3449, 3449, 3449, 3449, 3449, 3449,
+ 3449, 3449, 3450, 3450, 0, 3450, 3450, 3450, 3450, 3450,
+ 3450, 3450, 3450, 3450, 3450, 3451, 3451, 3451, 3451, 3451,
+ 3451, 3451, 3451, 3451, 3451, 3451, 3451, 3451, 3452, 3452,
+ 3452, 3452, 3452, 3452, 3452, 3452, 0, 3452, 3452, 3452,
+ 3452, 3453, 3453, 0, 3453, 3453, 3453, 3453, 3453, 3453,
+ 3453, 3453, 3453, 3453, 3454, 3454, 3454, 3454, 3454, 3454,
+ 3454, 3454, 3454, 3454, 3454, 3454, 3454, 3455, 3455, 3455,
+ 3455, 3455, 3455, 3455, 3455, 0, 3455, 3455, 3455, 3455,
3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456,
- 3456, 3456, 3457, 3457, 3457, 3457, 3457, 3457, 3457, 3457,
- 3457, 3457, 3457, 3457, 3457, 3458, 3458, 3458, 3458, 3458,
- 3458, 3458, 3458, 3458, 3458, 3458, 3458, 3458, 3459, 3459,
- 3459, 3459, 3459, 3459, 3459, 3459, 3459, 3459, 3459, 3459,
- 3459, 3460, 3460, 3460, 3460, 3460, 3460, 3460, 3460, 3460,
- 3460, 3460, 3460, 3460, 3461, 3461, 3461, 3461, 3461, 3461,
- 3461, 3461, 3461, 3461, 3461, 3461, 3461, 3462, 3462, 3462,
+ 3456, 3456, 3456, 3457, 3457, 3457, 3457, 3457, 3457, 3457,
+ 3457, 3457, 3457, 3457, 3457, 3457, 3458, 3458, 3458, 3458,
+ 3458, 3458, 3458, 3458, 0, 3458, 3458, 3458, 3458, 3459,
+ 3459, 3459, 3459, 3459, 3459, 3459, 3459, 3459, 3459, 3459,
+ 3459, 3459, 3460, 3460, 3460, 3460, 3460, 3460, 3460, 3460,
+ 3460, 3460, 3460, 3460, 3460, 3461, 3461, 3461, 3461, 3461,
+ 3461, 3461, 3461, 3461, 3461, 3461, 3461, 3461, 3462, 3462,
3462, 3462, 3462, 3462, 3462, 3462, 3462, 3462, 3462, 3462,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
- 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912, 2912,
-
- 2912, 2912, 2912, 2912, 2912
+ 3462, 3463, 3463, 3463, 3463, 3463, 3463, 3463, 3463, 3463,
+ 3463, 3463, 3463, 3463, 3464, 3464, 3464, 3464, 3464, 3464,
+
+ 3464, 3464, 3464, 3464, 3464, 3464, 3464, 3465, 3465, 3465,
+ 3465, 3465, 3465, 3465, 3465, 3465, 3465, 3465, 3465, 3465,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+ 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915, 2915,
+
+ 2915, 2915, 2915, 2915, 2915
} ;
/* Table of booleans, true if rule could match eol. */
@@ -7733,18 +7733,18 @@ static int yy_more_len = 0;
#define YY_RESTORE_YY_MORE_OFFSET
char *interfacetext;
#line 1 "interface.l"
-/* Foma: a finite-state toolkit and library. */
-/* Copyright © 2008-2015 Mans Hulden */
-/* This file is part of foma. */
-/* Foma is free software: you can redistribute it and/or modify */
-/* it under the terms of the GNU General Public License version 2 as */
-/* published by the Free Software Foundation. */
-/* Foma is distributed in the hope that it will be useful, */
-/* but WITHOUT ANY WARRANTY; without even the implied warranty of */
-/* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the */
-/* GNU General Public License for more details. */
-/* You should have received a copy of the GNU General Public License */
-/* along with foma. If not, see <http://www.gnu.org/licenses/>. */
+/* Foma: a finite-state toolkit and library. */
+/* Copyright © 2008-2015 Mans Hulden */
+/* This file is part of foma. */
+/* Licensed under the Apache License, Version 2.0 (the "License"); */
+/* you may not use this file except in compliance with the License. */
+/* You may obtain a copy of the License at */
+/* http://www.apache.org/licenses/LICENSE-2.0 */
+/* Unless required by applicable law or agreed to in writing, software */
+/* distributed under the License is distributed on an "AS IS" BASIS, */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */
+/* See the License for the specific language governing permissions and */
+/* limitations under the License. */
#define YY_NO_INPUT 1
#line 26 "interface.l"
@@ -7821,7 +7821,7 @@ char *rep_func_arg(char *s) {
return(argstr);
}
}
- return(strdup(s));
+ return(xxstrdup(s));
}
void my_interfaceparse(char *my_string) {
@@ -8155,7 +8155,7 @@ yy_match:
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 2913 )
+ if ( yy_current_state >= 2916 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
@@ -8920,7 +8920,7 @@ YY_DO_BEFORE_ACTION; /* set up interfacetext again */
YY_RULE_SETUP
#line 266 "interface.l"
{
- tempstr = trim(strdup(interfacetext));
+ tempstr = trim(xxstrdup(interfacetext));
BEGIN(APPLY_FILE_EATUP);
}
YY_BREAK
@@ -9283,7 +9283,7 @@ case 171:
YY_RULE_SETUP
#line 461 "interface.l"
{
- tempstr = strdup(trim(interfacetext));
+ tempstr = xxstrdup(trim(interfacetext));
BEGIN(PRINT_NET_NAME_FILE2);
}
YY_BREAK
@@ -9946,7 +9946,7 @@ static int yy_get_next_buffer (void)
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 2913 )
+ if ( yy_current_state >= 2916 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
@@ -9969,11 +9969,11 @@ static int yy_get_next_buffer (void)
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 2913 )
+ if ( yy_current_state >= 2916 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
- yy_is_jam = (yy_current_state == 2912);
+ yy_is_jam = (yy_current_state == 2915);
if ( ! yy_is_jam )
*(yy_state_ptr)++ = yy_current_state;
diff --git a/back-ends/foma/lex.lexc.c b/back-ends/foma/lex.lexc.c
index 359e11a..7961c69 100644
--- a/back-ends/foma/lex.lexc.c
+++ b/back-ends/foma/lex.lexc.c
@@ -1317,18 +1317,18 @@ static int yy_more_len = 0;
#define YY_RESTORE_YY_MORE_OFFSET
char *lexctext;
#line 1 "lexc.l"
-/* Foma: a finite-state toolkit and library. */
-/* Copyright © 2008-2015 Mans Hulden */
-/* This file is part of foma. */
-/* Foma is free software: you can redistribute it and/or modify */
-/* it under the terms of the GNU General Public License version 2 as */
-/* published by the Free Software Foundation. */
-/* Foma is distributed in the hope that it will be useful, */
-/* but WITHOUT ANY WARRANTY; without even the implied warranty of */
-/* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the */
-/* GNU General Public License for more details. */
-/* You should have received a copy of the GNU General Public License */
-/* along with foma. If not, see <http://www.gnu.org/licenses/>. */
+/* Foma: a finite-state toolkit and library. */
+/* Copyright © 2008-2015 Mans Hulden */
+/* This file is part of foma. */
+/* Licensed under the Apache License, Version 2.0 (the "License"); */
+/* you may not use this file except in compliance with the License. */
+/* You may obtain a copy of the License at */
+/* http://www.apache.org/licenses/LICENSE-2.0 */
+/* Unless required by applicable law or agreed to in writing, software */
+/* distributed under the License is distributed on an "AS IS" BASIS, */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */
+/* See the License for the specific language governing permissions and */
+/* limitations under the License. */
#define YY_NO_INPUT 1
#line 25 "lexc.l"
@@ -1390,7 +1390,7 @@ void lexc_trim(char *s) {
/* Nonreserved = anything except ; < > ! or space */
-#line 1394 "lex.lexc.c"
+#line 1390 "lex.lexc.c"
#define INITIAL 0
#define MCS 1
@@ -1587,11 +1587,11 @@ YY_DECL
register char *yy_cp, *yy_bp;
register int yy_act;
-#line 98 "lexc.l"
+#line 94 "lexc.l"
/* Files begin with one of these three identifiers */
-#line 1595 "lex.lexc.c"
+#line 1591 "lex.lexc.c"
if ( !(yy_init) )
{
@@ -1711,14 +1711,14 @@ do_action: /* This label is used only to access EOF actions. */
{ /* beginning of action switch */
case 1:
YY_RULE_SETUP
-#line 101 "lexc.l"
+#line 97 "lexc.l"
{
BEGIN(MCS);
}
YY_BREAK
case 2:
YY_RULE_SETUP
-#line 105 "lexc.l"
+#line 101 "lexc.l"
{
BEGIN(DEF);
}
@@ -1728,20 +1728,20 @@ YY_RULE_SETUP
case 3:
/* rule 3 can match eol */
YY_RULE_SETUP
-#line 111 "lexc.l"
+#line 107 "lexc.l"
{
yymore();
}
YY_BREAK
case 4:
YY_RULE_SETUP
-#line 115 "lexc.l"
+#line 111 "lexc.l"
{ }
YY_BREAK
case 5:
/* rule 5 can match eol */
YY_RULE_SETUP
-#line 116 "lexc.l"
+#line 112 "lexc.l"
{ lexclineno++; lexccolumn = 1;}
YY_BREAK
/* Multichar definitions */
@@ -1749,7 +1749,7 @@ YY_RULE_SETUP
case 6:
/* rule 6 can match eol */
YY_RULE_SETUP
-#line 120 "lexc.l"
+#line 116 "lexc.l"
{
lexc_add_mc(lexctext);
}
@@ -1757,13 +1757,13 @@ YY_RULE_SETUP
case 7:
/* rule 7 can match eol */
YY_RULE_SETUP
-#line 124 "lexc.l"
+#line 120 "lexc.l"
{
lexc_trim(lexctext+8);
if (lexentries != -1) {
printf("%i, ",lexentries);
}
- printf("%s...",lexctext+8);
+ printf("%s...",lexctext+8);
fflush(stdout);
lexentries = 0;
lexc_set_current_lexicon(lexctext+8, SOURCE_LEXICON);
@@ -1773,7 +1773,7 @@ YY_RULE_SETUP
/* Grab info string */
case 8:
YY_RULE_SETUP
-#line 137 "lexc.l"
+#line 133 "lexc.l"
{
BEGIN(LEXENTRIES);
}
@@ -1782,7 +1782,7 @@ YY_RULE_SETUP
case 9:
/* rule 9 can match eol */
YY_RULE_SETUP
-#line 141 "lexc.l"
+#line 137 "lexc.l"
{
lexc_trim(lexctext);
lexc_set_current_lexicon(lexctext, TARGET_LEXICON);
@@ -1800,7 +1800,7 @@ YY_RULE_SETUP
case 10:
/* rule 10 can match eol */
YY_RULE_SETUP
-#line 156 "lexc.l"
+#line 152 "lexc.l"
{
lexc_set_current_word(lexctext);
}
@@ -1808,7 +1808,7 @@ YY_RULE_SETUP
case 11:
/* rule 11 can match eol */
YY_RULE_SETUP
-#line 161 "lexc.l"
+#line 157 "lexc.l"
{
//printf("[%s]\n", lexctext);
lexc_trim(lexctext);
@@ -1825,7 +1825,7 @@ YY_RULE_SETUP
/* A REGEX entry begins and ends with a < , > */
case 12:
YY_RULE_SETUP
-#line 175 "lexc.l"
+#line 171 "lexc.l"
{
BEGIN(REGEX);
}
@@ -1833,7 +1833,7 @@ YY_RULE_SETUP
/* \076 = > */
case 13:
YY_RULE_SETUP
-#line 179 "lexc.l"
+#line 175 "lexc.l"
{
*(lexctext+lexcleng-1) = ';';
if (my_yyparse(lexctext, lexclineno, g_defines, NULL) == 0) {
@@ -1844,7 +1844,7 @@ YY_RULE_SETUP
YY_BREAK
case 14:
YY_RULE_SETUP
-#line 187 "lexc.l"
+#line 183 "lexc.l"
{
BEGIN(REGEXB);
yymore();
@@ -1853,14 +1853,14 @@ YY_RULE_SETUP
case 15:
/* rule 15 can match eol */
YY_RULE_SETUP
-#line 191 "lexc.l"
+#line 187 "lexc.l"
{
yymore();
}
YY_BREAK
case 16:
YY_RULE_SETUP
-#line 194 "lexc.l"
+#line 190 "lexc.l"
{
BEGIN(REGEX);
yymore();
@@ -1868,7 +1868,7 @@ YY_RULE_SETUP
YY_BREAK
case 17:
YY_RULE_SETUP
-#line 198 "lexc.l"
+#line 194 "lexc.l"
{
BEGIN(REGEXQ);
yymore();
@@ -1877,14 +1877,14 @@ YY_RULE_SETUP
case 18:
/* rule 18 can match eol */
YY_RULE_SETUP
-#line 202 "lexc.l"
+#line 198 "lexc.l"
{
yymore();
}
YY_BREAK
case 19:
YY_RULE_SETUP
-#line 205 "lexc.l"
+#line 201 "lexc.l"
{
BEGIN(REGEX);
yymore();
@@ -1893,7 +1893,7 @@ YY_RULE_SETUP
case 20:
/* rule 20 can match eol */
YY_RULE_SETUP
-#line 209 "lexc.l"
+#line 205 "lexc.l"
{
lexc_trim(lexctext);
tempstr = xxstrdup(lexctext);
@@ -1903,7 +1903,7 @@ YY_RULE_SETUP
/* \073 = ; */
case 21:
YY_RULE_SETUP
-#line 215 "lexc.l"
+#line 211 "lexc.l"
{
if (my_yyparse(lexctext, lexclineno, g_defines, NULL) == 0) {
add_defined(g_defines, fsm_topsort(fsm_minimize(current_parse)),tempstr);
@@ -1915,14 +1915,14 @@ YY_RULE_SETUP
case 22:
/* rule 22 can match eol */
YY_RULE_SETUP
-#line 222 "lexc.l"
+#line 218 "lexc.l"
{
yymore();
}
YY_BREAK
case 23:
YY_RULE_SETUP
-#line 225 "lexc.l"
+#line 221 "lexc.l"
{
BEGIN(DEFREGEXB);
yymore();
@@ -1931,14 +1931,14 @@ YY_RULE_SETUP
case 24:
/* rule 24 can match eol */
YY_RULE_SETUP
-#line 229 "lexc.l"
+#line 225 "lexc.l"
{
yymore();
}
YY_BREAK
case 25:
YY_RULE_SETUP
-#line 232 "lexc.l"
+#line 228 "lexc.l"
{
BEGIN(DEFREGEX);
yymore();
@@ -1946,7 +1946,7 @@ YY_RULE_SETUP
YY_BREAK
case 26:
YY_RULE_SETUP
-#line 236 "lexc.l"
+#line 232 "lexc.l"
{
BEGIN(DEFREGEXQ);
yymore();
@@ -1955,14 +1955,14 @@ YY_RULE_SETUP
case 27:
/* rule 27 can match eol */
YY_RULE_SETUP
-#line 240 "lexc.l"
+#line 236 "lexc.l"
{
yymore();
}
YY_BREAK
case 28:
YY_RULE_SETUP
-#line 243 "lexc.l"
+#line 239 "lexc.l"
{
BEGIN(DEFREGEX);
yymore();
@@ -1971,20 +1971,20 @@ YY_RULE_SETUP
case 29:
/* rule 29 can match eol */
YY_RULE_SETUP
-#line 247 "lexc.l"
+#line 243 "lexc.l"
{ /* printf ("Comment: [%s]\n",lexctext); */ }
YY_BREAK
case 30:
YY_RULE_SETUP
-#line 249 "lexc.l"
+#line 245 "lexc.l"
{ printf("\n***Syntax error on line %i column %i at '%s'\n",lexclineno,lexccolumn,lexctext); return 1;}
YY_BREAK
case 31:
YY_RULE_SETUP
-#line 250 "lexc.l"
+#line 246 "lexc.l"
ECHO;
YY_BREAK
-#line 1988 "lex.lexc.c"
+#line 1984 "lex.lexc.c"
case YY_STATE_EOF(INITIAL):
case YY_STATE_EOF(MCS):
case YY_STATE_EOF(LEXICON):
@@ -2931,4 +2931,4 @@ void lexcfree (void * ptr )
#define YYTABLES_NAME "yytables"
-#line 250 "lexc.l"
+#line 246 "lexc.l"
diff --git a/back-ends/foma/lex.yy.c b/back-ends/foma/lex.yy.c
index de29b9c..b1825f2 100644
--- a/back-ends/foma/lex.yy.c
+++ b/back-ends/foma/lex.yy.c
@@ -365,8 +365,8 @@ static void yy_fatal_error (yyconst char msg[] ,yyscan_t yyscanner );
*yy_cp = '\0'; \
yyg->yy_c_buf_p = yy_cp;
-#define YY_NUM_RULES 128
-#define YY_END_OF_BUFFER 129
+#define YY_NUM_RULES 129
+#define YY_END_OF_BUFFER 130
/* This struct is not used in this scanner,
but its presence is necessary. */
struct yy_trans_info
@@ -374,26 +374,26 @@ struct yy_trans_info
flex_int32_t yy_verify;
flex_int32_t yy_nxt;
};
-static yyconst flex_int16_t yy_accept[573] =
+static yyconst flex_int16_t yy_accept[574] =
{ 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 129, 127, 10, 125, 126, 127,
- 127, 107, 127, 123, 46, 117, 118, 109, 108, 96,
- 104, 127, 99, 94, 121, 63, 46, 64, 127, 112,
- 103, 113, 127, 48, 89, 127, 122, 57, 127, 127,
- 127, 127, 127, 127, 127, 127, 127, 127, 128, 128,
- 128, 128, 128, 18, 128, 128, 128, 128, 128, 128,
- 128, 128, 15, 10, 125, 126, 0, 124, 0, 19,
- 0, 0, 0, 106, 105, 46, 0, 0, 0, 0,
+ 0, 0, 0, 0, 130, 128, 10, 126, 127, 128,
+ 128, 108, 128, 124, 46, 118, 119, 110, 109, 97,
+ 105, 128, 100, 95, 122, 63, 46, 64, 128, 113,
+ 104, 114, 128, 48, 89, 128, 123, 57, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128, 129, 129,
+ 129, 129, 129, 18, 129, 129, 129, 129, 129, 129,
+ 129, 129, 15, 10, 126, 127, 0, 125, 0, 19,
+ 0, 0, 0, 107, 106, 46, 0, 0, 0, 0,
46, 45, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 97, 66, 0,
- 0, 0, 49, 50, 0, 0, 53, 51, 110, 0,
- 0, 111, 0, 91, 0, 120, 119, 68, 62, 82,
+ 0, 0, 0, 0, 0, 0, 0, 98, 66, 0,
+ 0, 0, 49, 50, 0, 0, 53, 51, 111, 0,
+ 0, 112, 0, 91, 0, 121, 120, 68, 62, 82,
88, 78, 0, 0, 86, 74, 0, 0, 0, 0,
93, 92, 6, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 90, 57, 95, 21, 0, 0, 0, 0, 0,
+ 0, 90, 57, 96, 21, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 17, 0,
0, 0, 0, 0, 0, 13, 0, 0, 0, 0,
0, 0, 14, 0, 0, 0, 0, 0, 16, 0,
@@ -401,44 +401,44 @@ static yyconst flex_int16_t yy_accept[573] =
0, 46, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 76, 47, 65,
- 98, 59, 60, 58, 61, 100, 102, 70, 80, 0,
- 72, 84, 0, 0, 0, 52, 101, 9, 8, 0,
+ 99, 59, 60, 58, 61, 101, 103, 70, 80, 0,
+ 72, 84, 0, 0, 0, 52, 102, 9, 8, 0,
22, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 1, 0, 1, 0, 0,
0, 0, 1, 0, 0, 0, 0, 1, 0, 0,
- 0, 0, 54, 55, 20, 56, 123, 122, 116, 114,
- 115, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 67,
- 0, 69, 0, 0, 83, 79, 0, 87, 0, 75,
- 0, 0, 2, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 34, 0, 0, 0, 0, 0, 27, 0,
- 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,
- 1, 0, 0, 0, 0, 1, 0, 0, 0, 0,
- 1, 0, 0, 0, 0, 0, 1, 0, 0, 0,
+ 0, 94, 0, 54, 55, 20, 56, 124, 123, 117,
+ 115, 116, 0, 0, 0, 0, 0, 0, 0, 0,
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 67, 0, 69, 0, 0, 83, 79, 0, 87, 0,
+ 75, 0, 0, 2, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 34, 0, 0, 0, 0, 0, 27,
+ 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
0, 1, 0, 0, 0, 0, 1, 0, 0, 0,
- 0, 0, 77, 71, 81, 73, 85, 0, 0, 0,
+ 0, 1, 0, 0, 0, 0, 0, 1, 0, 0,
+ 0, 0, 1, 0, 0, 0, 0, 1, 0, 0,
+ 0, 0, 0, 77, 71, 81, 73, 85, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,
- 0, 0, 0, 0, 1, 0, 0, 0, 0, 1,
- 0, 0, 0, 0, 1, 0, 0, 0, 0, 1,
- 0, 0, 0, 0, 1, 0, 0, 0, 11, 0,
- 0, 12, 0, 0, 3, 0, 0, 7, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 4,
- 0, 0, 0, 0, 0, 0, 0, 43, 0, 0,
- 0, 0, 0, 0, 0, 0, 26, 0, 0, 5,
-
- 0, 0, 0, 0, 0, 33, 0, 44, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 38, 0,
- 39, 0, 32, 41, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 30, 0, 0, 0, 40, 0,
- 35, 42, 0, 0, 0, 0, 0, 0, 28, 0,
- 31, 0, 0, 0, 24, 0, 0, 0, 0, 0,
- 0, 0, 36, 0, 25, 0, 29, 0, 23, 0,
- 37, 0
+ 1, 0, 0, 0, 0, 1, 0, 0, 0, 0,
+ 1, 0, 0, 0, 0, 1, 0, 0, 0, 0,
+ 1, 0, 0, 0, 0, 1, 0, 0, 0, 11,
+ 0, 0, 12, 0, 0, 3, 0, 0, 7, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 4, 0, 0, 0, 0, 0, 0, 0, 43, 0,
+ 0, 0, 0, 0, 0, 0, 0, 26, 0, 0,
+
+ 5, 0, 0, 0, 0, 0, 33, 0, 44, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 38,
+ 0, 39, 0, 32, 41, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 30, 0, 0, 0, 40,
+ 0, 35, 42, 0, 0, 0, 0, 0, 0, 28,
+ 0, 31, 0, 0, 0, 24, 0, 0, 0, 0,
+ 0, 0, 0, 36, 0, 25, 0, 29, 0, 23,
+ 0, 37, 0
} ;
static yyconst flex_int32_t yy_ec[256] =
@@ -489,26 +489,26 @@ static yyconst flex_int32_t yy_meta[106] =
5, 5, 5, 5, 11
} ;
-static yyconst flex_int16_t yy_base[713] =
+static yyconst flex_int16_t yy_base[714] =
{ 0,
- 0, 0, 0, 0, 99, 198, 1516, 1515, 290, 381,
- 472, 563, 1507, 1504, 1517, 4440, 105, 1513, 1511, 1511,
- 111, 90, 25, 4440, 121, 124, 4440, 4440, 4440, 1496,
- 1484, 660, 95, 4440, 89, 115, 134, 1481, 298, 117,
- 120, 4440, 285, 206, 4440, 232, 1380, 4440, 0, 138,
+ 0, 0, 0, 0, 99, 198, 1517, 1516, 290, 381,
+ 472, 563, 1508, 1507, 1518, 4440, 105, 1514, 1512, 1512,
+ 111, 90, 25, 4440, 121, 124, 4440, 4440, 4440, 1497,
+ 1485, 660, 95, 4440, 89, 115, 134, 1482, 298, 117,
+ 120, 4440, 285, 206, 4440, 232, 1448, 4440, 0, 138,
160, 0, 158, 0, 0, 653, 0, 0, 4440, 302,
755, 0, 0, 4440, 323, 858, 0, 0, 334, 950,
- 0, 0, 4440, 106, 1419, 1417, 1407, 4440, 312, 4440,
+ 0, 0, 4440, 106, 1440, 1418, 1418, 4440, 312, 4440,
1042, 0, 0, 4440, 4440, 407, 0, 0, 0, 392,
429, 4440, 0, 153, 162, 0, 161, 0, 0, 1080,
- 0, 0, 1381, 119, 1378, 120, 1332, 4440, 1376, 1386,
- 1385, 1384, 4440, 4440, 1382, 1381, 4440, 4440, 4440, 1380,
- 1379, 4440, 1378, 1376, 1375, 4440, 4440, 84, 4440, 4440,
- 4440, 4440, 0, 1366, 1375, 4440, 1348, 1324, 1290, 0,
- 4440, 1314, 136, 237, 299, 317, 1335, 300, 1298, 1292,
- 1296, 1288, 273, 1303, 1267, 1260, 1256, 514, 695, 705,
- 716, 4440, 4440, 4440, 4440, 0, 0, 224, 444, 1303,
+ 0, 0, 1382, 119, 1379, 120, 1333, 4440, 1377, 1387,
+ 1386, 1385, 4440, 4440, 1384, 1382, 4440, 4440, 4440, 1381,
+ 1380, 4440, 1379, 1377, 1376, 4440, 4440, 84, 4440, 4440,
+ 4440, 4440, 0, 1367, 1376, 4440, 1349, 1335, 1321, 0,
+ 4440, 1315, 136, 237, 299, 317, 1336, 300, 1299, 1293,
+ 1297, 1289, 273, 1304, 1291, 1261, 1267, 514, 695, 705,
+ 716, 1244, 4440, 4440, 4440, 0, 0, 224, 444, 1303,
437, 0, 514, 498, 0, 0, 0, 755, 4440, 1182,
0, 0, 0, 0, 524, 4440, 1285, 0, 0, 0,
0, 574, 4440, 1377, 0, 0, 0, 0, 4440, 0,
@@ -521,138 +521,138 @@ static yyconst flex_int16_t yy_base[713] =
4440, 1253, 1246, 1253, 1241, 1279, 1251, 639, 1241, 1272,
1225, 1228, 1225, 1240, 1240, 4440, 799, 809, 890, 911,
952, 993, 1276, 1134, 1213, 1225, 1236, 0, 1287, 1338,
- 1506, 1182, 4440, 4440, 4440, 4440, 4440, 4440, 4440, 4440,
- 4440, 258, 0, 0, 0, 0, 0, 0, 587, 626,
-
- 1245, 685, 0, 992, 784, 0, 0, 0, 0, 4440,
- 1233, 4440, 1232, 1231, 4440, 4440, 1230, 4440, 1229, 4440,
- 820, 861, 4440, 0, 1185, 1234, 682, 760, 655, 1195,
- 555, 1171, 4440, 1158, 1156, 1170, 1161, 1154, 4440, 1166,
- 1161, 1161, 1158, 1156, 1004, 1122, 1516, 1526, 1536, 1578,
- 1588, 1638, 1648, 1658, 1668, 1710, 1720, 1770, 1780, 1790,
- 1202, 1800, 1842, 1852, 1201, 1902, 1912, 1922, 1932, 1974,
- 1984, 1200, 2034, 2044, 2054, 2064, 1199, 2106, 2116, 2166,
- 1109, 0, 4440, 4440, 4440, 4440, 4440, 1316, 2268, 0,
- 0, 1328, 2360, 0, 0, 1193, 1192, 0, 763, 982,
-
- 1150, 1144, 1143, 1147, 1146, 1140, 1151, 1146, 1132, 1137,
- 1143, 1106, 1090, 1089, 1086, 1099, 1101, 1099, 1446, 1456,
- 2178, 2188, 2229, 2300, 2310, 2320, 2362, 2391, 2453, 2464,
- 2474, 2494, 2515, 2536, 2577, 2596, 2606, 2618, 2647, 2659,
- 2709, 2721, 2731, 2750, 2772, 2791, 2834, 2853, 4440, 0,
- 0, 4440, 0, 0, 4440, 0, 1132, 4440, 1087, 1046,
- 1036, 1035, 1034, 1034, 1028, 383, 1038, 1034, 1024, 1024,
- 1028, 1013, 1030, 1056, 1025, 1012, 0, 0, 1056, 4440,
- 1021, 1007, 1013, 954, 961, 987, 942, 4440, 985, 946,
- 927, 926, 941, 922, 929, 932, 4440, 934, 937, 4440,
-
- 927, 961, 926, 959, 922, 4440, 957, 4440, 956, 921,
- 920, 919, 833, 840, 836, 830, 820, 823, 4440, 820,
- 4440, 851, 4440, 4440, 769, 753, 743, 775, 713, 753,
- 748, 703, 665, 676, 4440, 663, 649, 645, 4440, 297,
- 4440, 4440, 689, 649, 644, 650, 676, 555, 4440, 553,
- 4440, 551, 544, 536, 4440, 478, 466, 507, 462, 487,
- 390, 343, 4440, 290, 4440, 301, 4440, 210, 4440, 241,
- 4440, 4440, 2957, 2968, 2979, 2990, 3001, 3012, 3020, 3024,
- 3030, 3041, 3052, 3063, 3074, 3085, 3096, 3104, 3115, 3126,
- 3137, 3148, 3159, 3170, 3181, 3192, 3203, 3211, 3222, 240,
-
- 3233, 3244, 3255, 3266, 3270, 3276, 3287, 3298, 3309, 3320,
- 3331, 3342, 3353, 3364, 3375, 3386, 3397, 3408, 3416, 3427,
- 3438, 3449, 3460, 3471, 3482, 3493, 3504, 3515, 3526, 3537,
- 3548, 111, 3559, 3570, 3581, 3592, 3603, 3614, 3625, 3636,
- 3647, 3658, 3669, 3680, 3691, 3702, 3713, 3724, 3735, 3746,
- 3757, 3768, 3779, 3790, 3801, 3812, 3823, 3834, 3845, 3856,
- 3867, 3878, 3889, 3900, 3911, 3922, 3933, 3944, 3955, 3966,
- 3977, 3988, 3999, 4010, 4021, 4032, 4043, 4054, 4065, 4076,
- 4087, 4098, 4109, 4120, 4131, 4142, 4153, 4164, 4175, 4186,
- 4197, 4208, 4219, 4230, 4241, 4252, 4263, 4274, 4285, 4296,
-
- 4307, 4318, 4329, 4340, 4351, 4362, 4373, 4384, 4395, 4406,
- 4417, 4428
+ 1506, 4440, 1182, 4440, 4440, 4440, 4440, 4440, 4440, 4440,
+ 4440, 4440, 258, 0, 0, 0, 0, 0, 0, 587,
+
+ 626, 1245, 685, 0, 992, 784, 0, 0, 0, 0,
+ 4440, 1233, 4440, 1232, 1231, 4440, 4440, 1230, 4440, 1229,
+ 4440, 820, 861, 4440, 0, 1185, 1234, 682, 760, 655,
+ 1195, 555, 1171, 4440, 1158, 1156, 1170, 1161, 1154, 4440,
+ 1166, 1161, 1161, 1158, 1156, 1004, 1122, 1516, 1526, 1536,
+ 1578, 1588, 1638, 1648, 1658, 1668, 1710, 1720, 1770, 1780,
+ 1790, 1202, 1800, 1842, 1852, 1201, 1902, 1912, 1922, 1932,
+ 1974, 1984, 1200, 2034, 2044, 2054, 2064, 1199, 2106, 2116,
+ 2166, 1109, 0, 4440, 4440, 4440, 4440, 4440, 1316, 2268,
+ 0, 0, 1328, 2360, 0, 0, 1193, 1192, 0, 763,
+
+ 982, 1150, 1144, 1143, 1147, 1146, 1140, 1151, 1146, 1132,
+ 1137, 1143, 1106, 1090, 1089, 1086, 1099, 1101, 1099, 1446,
+ 1456, 2178, 2188, 2229, 2300, 2310, 2320, 2362, 2391, 2453,
+ 2464, 2474, 2494, 2515, 2536, 2577, 2596, 2606, 2618, 2647,
+ 2659, 2709, 2721, 2731, 2750, 2772, 2791, 2834, 2853, 4440,
+ 0, 0, 4440, 0, 0, 4440, 0, 1132, 4440, 1087,
+ 1046, 1036, 1035, 1034, 1034, 1028, 383, 1038, 1034, 1024,
+ 1024, 1028, 1013, 1030, 1056, 1025, 1012, 0, 0, 1056,
+ 4440, 1021, 1007, 1013, 954, 961, 987, 942, 4440, 985,
+ 946, 927, 926, 941, 922, 929, 932, 4440, 934, 937,
+
+ 4440, 927, 961, 926, 959, 922, 4440, 957, 4440, 956,
+ 921, 920, 919, 833, 840, 836, 830, 820, 823, 4440,
+ 820, 4440, 851, 4440, 4440, 769, 753, 743, 775, 713,
+ 753, 748, 703, 665, 676, 4440, 663, 649, 645, 4440,
+ 297, 4440, 4440, 689, 649, 644, 650, 676, 555, 4440,
+ 553, 4440, 551, 544, 536, 4440, 478, 466, 507, 462,
+ 487, 390, 343, 4440, 290, 4440, 301, 4440, 210, 4440,
+ 241, 4440, 4440, 2957, 2968, 2979, 2990, 3001, 3012, 3020,
+ 3024, 3030, 3041, 3052, 3063, 3074, 3085, 3096, 3104, 3115,
+ 3126, 3137, 3148, 3159, 3170, 3181, 3192, 3203, 3211, 3222,
+
+ 240, 3233, 3244, 3255, 3266, 3270, 3276, 3287, 3298, 3309,
+ 3320, 3331, 3342, 3353, 3364, 3375, 3386, 3397, 3408, 3416,
+ 3427, 3438, 3449, 3460, 3471, 3482, 3493, 3504, 3515, 3526,
+ 3537, 3548, 111, 3559, 3570, 3581, 3592, 3603, 3614, 3625,
+ 3636, 3647, 3658, 3669, 3680, 3691, 3702, 3713, 3724, 3735,
+ 3746, 3757, 3768, 3779, 3790, 3801, 3812, 3823, 3834, 3845,
+ 3856, 3867, 3878, 3889, 3900, 3911, 3922, 3933, 3944, 3955,
+ 3966, 3977, 3988, 3999, 4010, 4021, 4032, 4043, 4054, 4065,
+ 4076, 4087, 4098, 4109, 4120, 4131, 4142, 4153, 4164, 4175,
+ 4186, 4197, 4208, 4219, 4230, 4241, 4252, 4263, 4274, 4285,
+
+ 4296, 4307, 4318, 4329, 4340, 4351, 4362, 4373, 4384, 4395,
+ 4406, 4417, 4428
} ;
-static yyconst flex_int16_t yy_def[713] =
+static yyconst flex_int16_t yy_def[714] =
{ 0,
- 572, 1, 573, 573, 574, 574, 573, 573, 575, 575,
- 576, 576, 573, 573, 572, 572, 572, 572, 572, 577,
- 578, 572, 579, 572, 580, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 580, 572, 572, 572,
- 572, 572, 572, 572, 572, 581, 572, 572, 582, 582,
- 582, 582, 582, 582, 583, 584, 585, 586, 572, 587,
- 587, 588, 589, 572, 590, 590, 591, 592, 593, 593,
- 594, 595, 572, 572, 572, 572, 596, 572, 597, 572,
- 597, 598, 599, 572, 572, 600, 601, 602, 603, 604,
- 605, 572, 606, 606, 606, 606, 606, 606, 607, 608,
-
- 609, 610, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 611, 572, 572, 572, 572, 572, 572, 612,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 613, 614, 615,
- 616, 572, 572, 572, 572, 606, 606, 606, 606, 606,
- 606, 606, 606, 606, 606, 606, 617, 618, 572, 618,
- 619, 620, 180, 621, 622, 572, 622, 623, 624, 187,
- 625, 626, 572, 626, 627, 628, 194, 629, 572, 630,
-
- 631, 632, 633, 633, 633, 633, 633, 633, 634, 635,
- 636, 637, 638, 639, 640, 640, 640, 640, 640, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 641,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 642, 642, 643, 644,
- 645, 643, 642, 646, 647, 648, 649, 650, 651, 652,
- 653, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 640, 654, 655, 656, 657, 658, 658, 658, 658,
-
- 658, 658, 658, 658, 658, 658, 658, 659, 660, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 661, 662, 572, 663, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 664, 664, 665, 666, 667, 665,
- 665, 668, 669, 670, 671, 671, 672, 673, 674, 665,
- 664, 668, 669, 670, 664, 665, 665, 668, 669, 670,
- 668, 665, 675, 676, 677, 672, 671, 678, 679, 680,
- 572, 681, 572, 572, 572, 572, 572, 682, 682, 683,
- 684, 685, 685, 686, 687, 688, 572, 689, 572, 572,
-
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 690, 690,
- 691, 692, 693, 691, 691, 694, 695, 696, 697, 697,
- 698, 699, 700, 691, 691, 694, 695, 696, 694, 694,
- 701, 702, 703, 698, 698, 704, 705, 706, 572, 707,
- 708, 572, 709, 710, 572, 711, 712, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 707, 709, 711, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
-
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 0, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
-
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
-
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572
+ 573, 1, 574, 574, 575, 575, 574, 574, 576, 576,
+ 577, 577, 574, 574, 573, 573, 573, 573, 573, 578,
+ 579, 573, 580, 573, 581, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 581, 573, 573, 573,
+ 573, 573, 573, 573, 573, 582, 573, 573, 583, 583,
+ 583, 583, 583, 583, 584, 585, 586, 587, 573, 588,
+ 588, 589, 590, 573, 591, 591, 592, 593, 594, 594,
+ 595, 596, 573, 573, 573, 573, 597, 573, 598, 573,
+ 598, 599, 600, 573, 573, 601, 602, 603, 604, 605,
+ 606, 573, 607, 607, 607, 607, 607, 607, 608, 609,
+
+ 610, 611, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 612, 573, 573, 573, 573, 573, 573, 613,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 614, 615, 616,
+ 617, 573, 573, 573, 573, 607, 607, 607, 607, 607,
+ 607, 607, 607, 607, 607, 607, 618, 619, 573, 619,
+ 620, 621, 180, 622, 623, 573, 623, 624, 625, 187,
+ 626, 627, 573, 627, 628, 629, 194, 630, 573, 631,
+
+ 632, 633, 634, 634, 634, 634, 634, 634, 635, 636,
+ 637, 638, 639, 640, 641, 641, 641, 641, 641, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 642,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 643, 643, 644, 645,
+ 646, 644, 643, 647, 648, 649, 650, 651, 652, 653,
+ 654, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 641, 655, 656, 657, 658, 659, 659, 659,
+
+ 659, 659, 659, 659, 659, 659, 659, 659, 660, 661,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 662, 663, 573, 664, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 665, 665, 666, 667, 668,
+ 666, 666, 669, 670, 671, 672, 672, 673, 674, 675,
+ 666, 665, 669, 670, 671, 665, 666, 666, 669, 670,
+ 671, 669, 666, 676, 677, 678, 673, 672, 679, 680,
+ 681, 573, 682, 573, 573, 573, 573, 573, 683, 683,
+ 684, 685, 686, 686, 687, 688, 689, 573, 690, 573,
+
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 691,
+ 691, 692, 693, 694, 692, 692, 695, 696, 697, 698,
+ 698, 699, 700, 701, 692, 692, 695, 696, 697, 695,
+ 695, 702, 703, 704, 699, 699, 705, 706, 707, 573,
+ 708, 709, 573, 710, 711, 573, 712, 713, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 708, 710, 712,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 0, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573
} ;
static yyconst flex_int16_t yy_nxt[4546] =
@@ -669,11 +669,11 @@ static yyconst flex_int16_t yy_nxt[4546] =
16, 16, 16, 16, 49, 50, 51, 52, 53, 54,
55, 56, 57, 58, 16, 59, 74, 74, 84, 74,
- 74, 238, 318, 239, 124, 202, 319, 80, 85, 87,
+ 74, 238, 319, 239, 124, 202, 320, 80, 85, 87,
87, 87, 87, 87, 87, 88, 88, 88, 89, 125,
- 90, 126, 128, 92, 310, 140, 221, 224, 127, 141,
+ 90, 126, 128, 92, 311, 140, 221, 224, 127, 141,
91, 103, 129, 90, 130, 225, 92, 226, 222, 104,
- 311, 105, 80, 106, 142, 91, 143, 143, 143, 202,
+ 312, 105, 80, 106, 142, 91, 143, 143, 143, 202,
91, 131, 91, 91, 91, 59, 59, 59, 59, 59,
59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
@@ -683,47 +683,47 @@ static yyconst flex_int16_t yy_nxt[4546] =
81, 82, 82, 82, 83, 93, 94, 95, 96, 97,
98, 99, 100, 101, 102, 107, 91, 163, 93, 94,
95, 96, 97, 98, 99, 100, 101, 102, 147, 164,
- 165, 572, 572, 572, 202, 148, 572, 149, 80, 150,
- 151, 572, 152, 571, 153, 154, 155, 248, 248, 248,
- 572, 156, 570, 157, 59, 59, 59, 59, 59, 59,
+ 165, 573, 573, 573, 202, 148, 573, 149, 80, 150,
+ 151, 573, 152, 572, 153, 154, 155, 248, 248, 248,
+ 573, 156, 571, 157, 59, 59, 59, 59, 59, 59,
59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
59, 59, 61, 61, 61, 61, 61, 61, 62, 62,
- 62, 63, 59, 59, 133, 143, 143, 143, 179, 549,
- 144, 321, 145, 569, 322, 134, 259, 282, 199, 249,
+ 62, 63, 59, 59, 133, 143, 143, 143, 179, 550,
+ 144, 322, 145, 570, 323, 134, 259, 283, 199, 249,
249, 249, 260, 135, 261, 136, 159, 159, 159, 159,
159, 159, 160, 160, 160, 161, 186, 250, 250, 250,
- 550, 568, 252, 248, 248, 248, 146, 193, 253, 254,
- 202, 91, 137, 138, 139, 567, 59, 59, 59, 59,
+ 551, 569, 252, 248, 248, 248, 146, 193, 253, 254,
+ 202, 91, 137, 138, 139, 568, 59, 59, 59, 59,
59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
59, 59, 59, 59, 66, 66, 66, 66, 66, 66,
- 67, 67, 67, 68, 59, 488, 180, 180, 180, 180,
+ 67, 67, 67, 68, 59, 489, 180, 180, 180, 180,
180, 180, 181, 181, 181, 182, 81, 81, 81, 81,
81, 81, 82, 82, 82, 83, 90, 187, 187, 187,
- 187, 187, 187, 188, 188, 188, 189, 572, 194, 194,
+ 187, 187, 187, 188, 188, 188, 189, 573, 194, 194,
194, 194, 194, 194, 195, 195, 195, 196, 90, 91,
- 489, 92, 249, 249, 249, 566, 91, 59, 59, 59,
+ 490, 92, 249, 249, 249, 567, 91, 59, 59, 59,
59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
59, 59, 59, 59, 59, 66, 66, 66, 66, 66,
66, 67, 67, 67, 68, 59, 87, 87, 87, 87,
- 87, 87, 88, 88, 88, 89, 572, 572, 202, 565,
+ 87, 87, 88, 88, 88, 89, 573, 573, 202, 566,
91, 203, 204, 205, 206, 207, 208, 209, 210, 211,
- 212, 113, 114, 564, 283, 284, 91, 572, 572, 563,
- 312, 562, 91, 93, 94, 95, 96, 97, 98, 99,
- 100, 101, 102, 91, 313, 561, 314, 186, 59, 59,
+ 212, 113, 114, 565, 284, 285, 91, 573, 573, 564,
+ 313, 563, 91, 93, 94, 95, 96, 97, 98, 99,
+ 100, 101, 102, 91, 314, 562, 315, 186, 59, 59,
59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
59, 59, 59, 59, 59, 59, 70, 70, 70, 70,
- 70, 70, 71, 71, 71, 72, 59, 266, 572, 289,
- 572, 572, 572, 572, 560, 285, 572, 193, 286, 202,
- 290, 291, 559, 572, 234, 572, 572, 406, 129, 287,
+ 70, 70, 71, 71, 71, 72, 59, 266, 573, 290,
+ 573, 573, 573, 573, 561, 286, 573, 193, 287, 202,
+ 291, 292, 560, 573, 234, 573, 573, 407, 129, 288,
- 288, 287, 288, 558, 572, 557, 556, 407, 159, 159,
+ 289, 288, 289, 559, 573, 558, 557, 408, 159, 159,
159, 159, 159, 159, 160, 160, 160, 161, 187, 187,
187, 187, 187, 187, 188, 188, 188, 189, 202, 59,
59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
@@ -731,257 +731,257 @@ static yyconst flex_int16_t yy_nxt[4546] =
59, 59, 59, 59, 59, 59, 59, 70, 70, 70,
70, 70, 70, 71, 71, 71, 72, 110, 194, 194,
194, 194, 194, 194, 195, 195, 195, 196, 111, 112,
- 572, 113, 114, 335, 90, 336, 327, 202, 555, 554,
- 115, 116, 553, 572, 572, 117, 337, 158, 328, 401,
+ 573, 113, 114, 336, 90, 337, 328, 202, 556, 555,
+ 115, 116, 554, 573, 573, 117, 338, 158, 329, 402,
- 552, 551, 548, 402, 118, 403, 119, 158, 114, 547,
- 404, 120, 121, 546, 122, 545, 544, 113, 158, 123,
+ 553, 552, 549, 403, 118, 404, 119, 158, 114, 548,
+ 405, 120, 121, 547, 122, 546, 545, 113, 158, 123,
168, 169, 170, 170, 170, 171, 172, 173, 174, 175,
175, 175, 175, 175, 175, 175, 175, 175, 175, 175,
175, 175, 175, 175, 175, 175, 175, 175, 175, 175,
- 175, 175, 175, 175, 175, 175, 175, 572, 268, 543,
- 542, 179, 572, 572, 399, 541, 540, 399, 273, 203,
+ 175, 175, 175, 175, 175, 175, 175, 573, 268, 544,
+ 543, 179, 573, 573, 400, 542, 541, 400, 273, 203,
204, 205, 206, 207, 208, 209, 210, 211, 212, 278,
- 400, 400, 400, 400, 400, 400, 202, 539, 538, 269,
+ 401, 401, 401, 401, 401, 401, 202, 540, 539, 269,
269, 269, 269, 269, 269, 270, 270, 270, 271, 274,
- 274, 274, 274, 274, 274, 275, 275, 275, 276, 537,
+ 274, 274, 274, 274, 274, 275, 275, 275, 276, 538,
279, 279, 279, 279, 279, 279, 280, 280, 280, 281,
- 536, 178, 178, 178, 178, 178, 178, 178, 178, 178,
+ 537, 178, 178, 178, 178, 178, 178, 178, 178, 178,
178, 178, 178, 178, 178, 178, 178, 178, 178, 178,
178, 178, 178, 178, 178, 178, 178, 178, 178, 180,
180, 180, 180, 180, 180, 181, 181, 181, 182, 178,
- 572, 572, 266, 535, 572, 572, 572, 572, 572, 572,
- 534, 533, 266, 327, 532, 572, 572, 572, 531, 572,
- 572, 572, 572, 572, 530, 328, 529, 528, 91, 250,
+ 573, 573, 266, 536, 573, 573, 573, 573, 573, 573,
+ 535, 534, 266, 328, 533, 573, 573, 573, 532, 573,
+ 573, 573, 573, 573, 531, 329, 530, 529, 91, 250,
250, 250, 158, 159, 159, 159, 159, 159, 159, 160,
160, 160, 161, 159, 159, 159, 159, 159, 159, 160,
- 160, 160, 161, 158, 389, 389, 389, 389, 389, 389,
- 390, 390, 390, 391, 185, 185, 185, 185, 185, 185,
+ 160, 160, 161, 158, 390, 390, 390, 390, 390, 390,
+ 391, 391, 391, 392, 185, 185, 185, 185, 185, 185,
185, 185, 185, 185, 185, 185, 185, 185, 185, 185,
185, 185, 185, 185, 185, 185, 185, 185, 185, 185,
- 185, 185, 572, 268, 158, 393, 393, 393, 393, 393,
- 393, 394, 394, 394, 395, 527, 526, 525, 524, 523,
- 522, 521, 520, 519, 273, 518, 517, 516, 515, 514,
- 513, 512, 511, 510, 269, 269, 269, 269, 269, 269,
- 270, 270, 270, 271, 202, 158, 509, 508, 507, 506,
+ 185, 185, 573, 268, 158, 394, 394, 394, 394, 394,
+ 394, 395, 395, 395, 396, 528, 527, 526, 525, 524,
+ 523, 522, 521, 520, 273, 519, 518, 517, 516, 515,
+ 514, 513, 512, 511, 269, 269, 269, 269, 269, 269,
+ 270, 270, 270, 271, 202, 158, 510, 509, 508, 507,
- 505, 504, 400, 400, 400, 274, 274, 274, 274, 274,
+ 506, 505, 401, 401, 401, 274, 274, 274, 274, 274,
274, 275, 275, 275, 276, 278, 192, 192, 192, 192,
192, 192, 192, 192, 192, 192, 192, 192, 192, 192,
192, 192, 192, 192, 192, 192, 192, 192, 192, 192,
- 192, 192, 192, 192, 572, 458, 279, 279, 279, 279,
- 279, 279, 280, 280, 280, 281, 268, 503, 572, 502,
- 501, 572, 500, 572, 499, 498, 572, 266, 497, 496,
- 495, 494, 572, 493, 492, 491, 572, 572, 572, 572,
- 572, 490, 487, 486, 485, 484, 483, 269, 269, 269,
- 269, 269, 269, 270, 270, 270, 271, 482, 159, 159,
+ 192, 192, 192, 192, 573, 459, 279, 279, 279, 279,
+ 279, 279, 280, 280, 280, 281, 268, 504, 573, 503,
+ 502, 573, 501, 573, 500, 499, 573, 266, 498, 497,
+ 496, 495, 573, 494, 493, 492, 573, 573, 573, 573,
+ 573, 491, 488, 487, 486, 485, 484, 269, 269, 269,
+ 269, 269, 269, 270, 270, 270, 271, 483, 159, 159,
159, 159, 159, 159, 160, 160, 160, 161, 79, 79,
79, 79, 79, 79, 79, 79, 79, 79, 79, 79,
79, 79, 79, 79, 79, 79, 79, 79, 79, 79,
- 79, 79, 79, 79, 79, 79, 158, 481, 480, 476,
- 475, 474, 473, 472, 471, 470, 79, 215, 216, 170,
+ 79, 79, 79, 79, 79, 79, 158, 482, 481, 477,
+ 476, 475, 474, 473, 472, 471, 79, 215, 216, 170,
170, 170, 217, 172, 218, 219, 175, 175, 175, 175,
175, 175, 175, 175, 175, 175, 175, 175, 175, 175,
175, 175, 175, 175, 175, 175, 175, 175, 175, 175,
- 175, 175, 175, 175, 572, 266, 469, 468, 467, 466,
- 465, 464, 463, 462, 461, 460, 459, 346, 456, 455,
-
- 119, 572, 572, 572, 572, 418, 417, 416, 415, 414,
- 413, 412, 411, 410, 409, 158, 159, 159, 159, 159,
- 159, 159, 160, 160, 160, 161, 408, 158, 347, 347,
- 347, 347, 347, 347, 348, 348, 348, 349, 158, 405,
- 398, 397, 387, 386, 385, 384, 383, 202, 178, 178,
+ 175, 175, 175, 175, 573, 266, 470, 469, 468, 467,
+ 466, 465, 464, 463, 462, 461, 460, 347, 457, 456,
+
+ 119, 573, 573, 573, 573, 419, 418, 417, 416, 415,
+ 414, 413, 412, 411, 410, 158, 159, 159, 159, 159,
+ 159, 159, 160, 160, 160, 161, 409, 158, 348, 348,
+ 348, 348, 348, 348, 349, 349, 349, 350, 158, 406,
+ 399, 398, 388, 387, 386, 385, 384, 202, 178, 178,
178, 178, 178, 178, 178, 178, 178, 178, 178, 178,
178, 178, 178, 178, 178, 178, 178, 178, 178, 178,
- 178, 178, 178, 178, 178, 178, 351, 381, 572, 344,
- 343, 342, 341, 340, 339, 338, 178, 572, 356, 158,
- 334, 333, 332, 331, 330, 329, 326, 325, 324, 361,
-
- 323, 320, 317, 316, 315, 91, 265, 352, 352, 352,
- 352, 352, 352, 353, 353, 353, 354, 264, 263, 357,
- 357, 357, 357, 357, 357, 358, 358, 358, 359, 449,
- 362, 362, 362, 362, 362, 362, 363, 363, 363, 364,
- 158, 452, 262, 258, 257, 256, 255, 251, 247, 245,
- 367, 185, 185, 185, 185, 185, 185, 185, 185, 185,
+ 178, 178, 178, 178, 178, 178, 352, 382, 573, 345,
+ 344, 343, 342, 341, 340, 339, 178, 573, 357, 158,
+ 335, 334, 333, 332, 331, 330, 327, 326, 325, 362,
+
+ 324, 321, 318, 317, 316, 91, 282, 353, 353, 353,
+ 353, 353, 353, 354, 354, 354, 355, 265, 264, 358,
+ 358, 358, 358, 358, 358, 359, 359, 359, 360, 450,
+ 363, 363, 363, 363, 363, 363, 364, 364, 364, 365,
+ 158, 453, 263, 262, 258, 257, 256, 255, 251, 247,
+ 368, 185, 185, 185, 185, 185, 185, 185, 185, 185,
185, 185, 185, 185, 185, 185, 185, 185, 185, 185,
- 185, 185, 185, 185, 185, 185, 185, 185, 185, 572,
- 244, 368, 368, 368, 368, 368, 368, 369, 369, 369,
- 370, 243, 242, 241, 237, 236, 164, 235, 234, 233,
-
- 232, 372, 231, 230, 229, 228, 227, 223, 220, 78,
- 389, 389, 389, 389, 389, 389, 390, 390, 390, 391,
- 76, 75, 393, 393, 393, 393, 393, 393, 394, 394,
- 394, 395, 373, 373, 373, 373, 373, 373, 374, 374,
- 374, 375, 162, 192, 192, 192, 192, 192, 192, 192,
+ 185, 185, 185, 185, 185, 185, 185, 185, 185, 573,
+ 245, 369, 369, 369, 369, 369, 369, 370, 370, 370,
+ 371, 244, 243, 242, 241, 237, 236, 164, 235, 234,
+
+ 233, 373, 232, 231, 230, 229, 228, 227, 223, 220,
+ 390, 390, 390, 390, 390, 390, 391, 391, 391, 392,
+ 78, 76, 394, 394, 394, 394, 394, 394, 395, 395,
+ 395, 396, 374, 374, 374, 374, 374, 374, 375, 375,
+ 375, 376, 75, 192, 192, 192, 192, 192, 192, 192,
192, 192, 192, 192, 192, 192, 192, 192, 192, 192,
192, 192, 192, 192, 192, 192, 192, 192, 192, 192,
- 192, 299, 300, 301, 301, 301, 302, 303, 304, 305,
- 306, 306, 306, 306, 306, 306, 306, 306, 306, 306,
- 306, 306, 306, 306, 306, 306, 306, 306, 306, 306,
-
- 306, 306, 306, 306, 306, 306, 306, 306, 158, 266,
- 132, 109, 108, 78, 76, 75, 572, 73, 158, 266,
- 73, 64, 64, 572, 572, 572, 572, 572, 158, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 158, 572,
+ 192, 300, 301, 302, 302, 302, 303, 304, 305, 306,
+ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307,
+ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307,
+
+ 307, 307, 307, 307, 307, 307, 307, 307, 158, 266,
+ 162, 132, 109, 108, 78, 76, 75, 573, 158, 266,
+ 73, 73, 64, 64, 573, 573, 573, 573, 158, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 158, 573,
159, 159, 159, 159, 159, 159, 160, 160, 160, 161,
159, 159, 159, 159, 159, 159, 160, 160, 160, 161,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 377,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 268,
- 158, 572, 572, 572, 572, 572, 572, 572, 572, 273,
- 158, 572, 572, 572, 572, 572, 572, 572, 572, 278,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 378,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 268,
+ 158, 573, 573, 573, 573, 573, 573, 573, 573, 273,
+ 158, 573, 573, 573, 573, 573, 573, 573, 573, 278,
- 378, 378, 378, 378, 378, 378, 379, 379, 379, 380,
+ 379, 379, 379, 379, 379, 379, 380, 380, 380, 381,
269, 269, 269, 269, 269, 269, 270, 270, 270, 271,
274, 274, 274, 274, 274, 274, 275, 275, 275, 276,
279, 279, 279, 279, 279, 279, 280, 280, 280, 281,
- 158, 268, 572, 572, 572, 572, 572, 572, 572, 572,
- 158, 268, 572, 572, 572, 572, 572, 572, 572, 572,
- 158, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 158, 572, 269, 269, 269, 269, 269, 269, 270, 270,
+ 158, 268, 573, 573, 573, 573, 573, 573, 573, 573,
+ 158, 268, 573, 573, 573, 573, 573, 573, 573, 573,
+ 158, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 158, 573, 269, 269, 269, 269, 269, 269, 270, 270,
270, 271, 269, 269, 269, 269, 269, 269, 270, 270,
- 270, 271, 572, 572, 572, 572, 572, 572, 572, 572,
-
- 572, 346, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 351, 158, 572, 572, 572, 572, 572, 572, 572,
- 572, 356, 158, 572, 572, 572, 572, 572, 572, 572,
- 572, 361, 347, 347, 347, 347, 347, 347, 348, 348,
- 348, 349, 352, 352, 352, 352, 352, 352, 353, 353,
- 353, 354, 357, 357, 357, 357, 357, 357, 358, 358,
- 358, 359, 362, 362, 362, 362, 362, 362, 363, 363,
- 363, 364, 158, 361, 572, 572, 572, 572, 572, 572,
- 572, 572, 158, 367, 572, 572, 572, 572, 572, 572,
- 572, 572, 158, 572, 572, 572, 572, 572, 572, 572,
-
- 572, 572, 158, 572, 362, 362, 362, 362, 362, 362,
- 363, 363, 363, 364, 368, 368, 368, 368, 368, 368,
- 369, 369, 369, 370, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 372, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 377, 158, 572, 572, 572, 572, 572,
- 572, 572, 572, 268, 158, 572, 572, 572, 572, 572,
- 572, 572, 572, 346, 373, 373, 373, 373, 373, 373,
- 374, 374, 374, 375, 378, 378, 378, 378, 378, 378,
- 379, 379, 379, 380, 269, 269, 269, 269, 269, 269,
- 270, 270, 270, 271, 347, 347, 347, 347, 347, 347,
-
- 348, 348, 348, 349, 158, 351, 572, 572, 572, 572,
- 572, 572, 572, 572, 158, 356, 572, 572, 572, 572,
- 572, 572, 572, 572, 158, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 158, 572, 352, 352, 352, 352,
- 352, 352, 353, 353, 353, 354, 357, 357, 357, 357,
- 357, 357, 358, 358, 358, 359, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 268, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 268, 158, 572, 572, 572,
- 572, 572, 572, 572, 572, 346, 158, 572, 572, 572,
- 572, 572, 572, 572, 572, 351, 269, 269, 269, 269,
+ 270, 271, 573, 573, 573, 573, 573, 573, 573, 573,
+
+ 573, 347, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 352, 158, 573, 573, 573, 573, 573, 573, 573,
+ 573, 357, 158, 573, 573, 573, 573, 573, 573, 573,
+ 573, 362, 348, 348, 348, 348, 348, 348, 349, 349,
+ 349, 350, 353, 353, 353, 353, 353, 353, 354, 354,
+ 354, 355, 358, 358, 358, 358, 358, 358, 359, 359,
+ 359, 360, 363, 363, 363, 363, 363, 363, 364, 364,
+ 364, 365, 158, 362, 573, 573, 573, 573, 573, 573,
+ 573, 573, 158, 368, 573, 573, 573, 573, 573, 573,
+ 573, 573, 158, 573, 573, 573, 573, 573, 573, 573,
+
+ 573, 573, 158, 573, 363, 363, 363, 363, 363, 363,
+ 364, 364, 364, 365, 369, 369, 369, 369, 369, 369,
+ 370, 370, 370, 371, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 373, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 378, 158, 573, 573, 573, 573, 573,
+ 573, 573, 573, 268, 158, 573, 573, 573, 573, 573,
+ 573, 573, 573, 347, 374, 374, 374, 374, 374, 374,
+ 375, 375, 375, 376, 379, 379, 379, 379, 379, 379,
+ 380, 380, 380, 381, 269, 269, 269, 269, 269, 269,
+ 270, 270, 270, 271, 348, 348, 348, 348, 348, 348,
+
+ 349, 349, 349, 350, 158, 352, 573, 573, 573, 573,
+ 573, 573, 573, 573, 158, 357, 573, 573, 573, 573,
+ 573, 573, 573, 573, 158, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 158, 573, 353, 353, 353, 353,
+ 353, 353, 354, 354, 354, 355, 358, 358, 358, 358,
+ 358, 358, 359, 359, 359, 360, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 268, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 268, 158, 573, 573, 573,
+ 573, 573, 573, 573, 573, 347, 158, 573, 573, 573,
+ 573, 573, 573, 573, 573, 352, 269, 269, 269, 269,
269, 269, 270, 270, 270, 271, 269, 269, 269, 269,
- 269, 269, 270, 270, 270, 271, 347, 347, 347, 347,
- 347, 347, 348, 348, 348, 349, 352, 352, 352, 352,
- 352, 352, 353, 353, 353, 354, 158, 356, 572, 572,
- 572, 572, 572, 572, 572, 572, 158, 346, 572, 572,
- 572, 572, 572, 572, 572, 572, 158, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 158, 572, 357, 357,
- 357, 357, 357, 357, 358, 358, 358, 359, 347, 347,
- 347, 347, 347, 347, 348, 348, 348, 349, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 420, 572, 572,
-
- 572, 572, 572, 572, 572, 572, 572, 425, 158, 572,
- 572, 572, 572, 572, 572, 572, 572, 430, 158, 572,
- 572, 572, 572, 572, 572, 572, 572, 367, 421, 421,
- 421, 421, 421, 421, 422, 422, 422, 423, 426, 426,
- 426, 426, 426, 426, 427, 427, 427, 428, 431, 431,
- 431, 431, 431, 431, 432, 432, 432, 433, 368, 368,
- 368, 368, 368, 368, 369, 369, 369, 370, 158, 435,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 440,
- 158, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 158, 572, 572, 572, 572, 572, 572, 572, 572, 572,
-
- 436, 436, 436, 436, 436, 436, 437, 437, 437, 438,
- 441, 441, 441, 441, 441, 441, 442, 442, 442, 443,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 445,
- 572, 158, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 268, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 273, 572, 572, 572, 572, 572, 572, 572, 572,
- 446, 446, 446, 446, 446, 446, 447, 447, 447, 448,
- 572, 572, 269, 269, 269, 269, 269, 269, 270, 270,
+ 269, 269, 270, 270, 270, 271, 348, 348, 348, 348,
+ 348, 348, 349, 349, 349, 350, 353, 353, 353, 353,
+ 353, 353, 354, 354, 354, 355, 158, 357, 573, 573,
+ 573, 573, 573, 573, 573, 573, 158, 347, 573, 573,
+ 573, 573, 573, 573, 573, 573, 158, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 158, 573, 358, 358,
+ 358, 358, 358, 358, 359, 359, 359, 360, 348, 348,
+ 348, 348, 348, 348, 349, 349, 349, 350, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 421, 573, 573,
+
+ 573, 573, 573, 573, 573, 573, 573, 426, 158, 573,
+ 573, 573, 573, 573, 573, 573, 573, 431, 158, 573,
+ 573, 573, 573, 573, 573, 573, 573, 368, 422, 422,
+ 422, 422, 422, 422, 423, 423, 423, 424, 427, 427,
+ 427, 427, 427, 427, 428, 428, 428, 429, 432, 432,
+ 432, 432, 432, 432, 433, 433, 433, 434, 369, 369,
+ 369, 369, 369, 369, 370, 370, 370, 371, 158, 436,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 441,
+ 158, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 158, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+
+ 437, 437, 437, 437, 437, 437, 438, 438, 438, 439,
+ 442, 442, 442, 442, 442, 442, 443, 443, 443, 444,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 446,
+ 573, 158, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 268, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 273, 573, 573, 573, 573, 573, 573, 573, 573,
+ 447, 447, 447, 447, 447, 447, 448, 448, 448, 449,
+ 573, 573, 269, 269, 269, 269, 269, 269, 270, 270,
270, 271, 274, 274, 274, 274, 274, 274, 275, 275,
- 275, 276, 278, 572, 572, 572, 572, 572, 572, 572,
-
- 572, 572, 158, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 158, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 158, 279, 279, 279, 279, 279, 279, 280,
- 280, 280, 281, 572, 388, 388, 388, 388, 388, 388,
- 388, 388, 388, 388, 388, 388, 388, 388, 388, 388,
- 388, 388, 388, 388, 388, 388, 388, 388, 388, 388,
- 388, 388, 572, 268, 158, 572, 572, 572, 572, 572,
- 572, 572, 572, 268, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 346, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 158, 269, 269, 269, 269, 269, 269,
+ 275, 276, 278, 573, 573, 573, 573, 573, 573, 573,
+
+ 573, 573, 158, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 158, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 158, 279, 279, 279, 279, 279, 279, 280,
+ 280, 280, 281, 573, 389, 389, 389, 389, 389, 389,
+ 389, 389, 389, 389, 389, 389, 389, 389, 389, 389,
+ 389, 389, 389, 389, 389, 389, 389, 389, 389, 389,
+ 389, 389, 573, 268, 158, 573, 573, 573, 573, 573,
+ 573, 573, 573, 268, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 347, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 158, 269, 269, 269, 269, 269, 269,
270, 270, 270, 271, 269, 269, 269, 269, 269, 269,
- 270, 270, 270, 271, 347, 347, 347, 347, 347, 347,
- 348, 348, 348, 349, 572, 351, 392, 392, 392, 392,
- 392, 392, 392, 392, 392, 392, 392, 392, 392, 392,
- 392, 392, 392, 392, 392, 392, 392, 392, 392, 392,
- 392, 392, 392, 392, 356, 158, 352, 352, 352, 352,
- 352, 352, 353, 353, 353, 354, 158, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 158, 572, 572, 572,
- 572, 572, 572, 572, 572, 357, 357, 357, 357, 357,
- 357, 358, 358, 358, 359, 572, 158, 572, 572, 572,
-
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 361, 158, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 361, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 367, 158, 572,
- 572, 572, 572, 572, 572, 572, 572, 362, 362, 362,
- 362, 362, 362, 363, 363, 363, 364, 372, 362, 362,
- 362, 362, 362, 362, 363, 363, 363, 364, 368, 368,
- 368, 368, 368, 368, 369, 369, 369, 370, 377, 158,
- 572, 572, 572, 572, 572, 572, 572, 572, 373, 373,
- 373, 373, 373, 373, 374, 374, 374, 375, 158, 268,
-
- 572, 572, 572, 572, 572, 572, 572, 572, 158, 378,
- 378, 378, 378, 378, 378, 379, 379, 379, 380, 572,
- 158, 572, 572, 572, 572, 572, 572, 572, 572, 572,
+ 270, 270, 270, 271, 348, 348, 348, 348, 348, 348,
+ 349, 349, 349, 350, 573, 352, 393, 393, 393, 393,
+ 393, 393, 393, 393, 393, 393, 393, 393, 393, 393,
+ 393, 393, 393, 393, 393, 393, 393, 393, 393, 393,
+ 393, 393, 393, 393, 357, 158, 353, 353, 353, 353,
+ 353, 353, 354, 354, 354, 355, 158, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 158, 573, 573, 573,
+ 573, 573, 573, 573, 573, 358, 358, 358, 358, 358,
+ 358, 359, 359, 359, 360, 573, 158, 573, 573, 573,
+
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 362, 158, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 362, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 368, 158, 573,
+ 573, 573, 573, 573, 573, 573, 573, 363, 363, 363,
+ 363, 363, 363, 364, 364, 364, 365, 373, 363, 363,
+ 363, 363, 363, 363, 364, 364, 364, 365, 369, 369,
+ 369, 369, 369, 369, 370, 370, 370, 371, 378, 158,
+ 573, 573, 573, 573, 573, 573, 573, 573, 374, 374,
+ 374, 374, 374, 374, 375, 375, 375, 376, 158, 268,
+
+ 573, 573, 573, 573, 573, 573, 573, 573, 158, 379,
+ 379, 379, 379, 379, 379, 380, 380, 380, 381, 573,
+ 158, 573, 573, 573, 573, 573, 573, 573, 573, 573,
269, 269, 269, 269, 269, 269, 270, 270, 270, 271,
- 268, 572, 572, 572, 572, 572, 572, 572, 572, 158,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 346,
- 572, 158, 572, 572, 572, 572, 572, 572, 572, 351,
- 572, 269, 269, 269, 269, 269, 269, 270, 270, 270,
- 271, 356, 572, 572, 572, 572, 572, 572, 572, 572,
- 347, 347, 347, 347, 347, 347, 348, 348, 348, 349,
-
- 352, 352, 352, 352, 352, 352, 353, 353, 353, 354,
- 346, 158, 357, 357, 357, 357, 357, 357, 358, 358,
- 358, 359, 346, 158, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 158, 572, 572, 572, 572, 572, 572,
- 572, 347, 347, 347, 347, 347, 347, 348, 348, 348,
- 349, 572, 158, 347, 347, 347, 347, 347, 347, 348,
- 348, 348, 349, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 420, 572, 158, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 425, 572, 572, 572, 572, 572,
- 572, 572, 572, 158, 430, 572, 572, 572, 572, 572,
-
- 572, 572, 572, 421, 421, 421, 421, 421, 421, 422,
- 422, 422, 423, 367, 572, 426, 426, 426, 426, 426,
- 426, 427, 427, 427, 428, 431, 431, 431, 431, 431,
- 431, 432, 432, 432, 433, 367, 158, 572, 572, 572,
- 572, 572, 572, 572, 368, 368, 368, 368, 368, 368,
- 369, 369, 369, 370, 435, 158, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 368, 368, 368, 368,
- 368, 368, 369, 369, 369, 370, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 436, 436, 436, 436, 436,
- 436, 437, 437, 437, 438, 572, 572, 440, 572, 572,
-
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 445, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 441, 441,
- 441, 441, 441, 441, 442, 442, 442, 443, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 446, 446, 446,
- 446, 446, 446, 447, 447, 447, 448, 59, 59, 59,
+ 268, 573, 573, 573, 573, 573, 573, 573, 573, 158,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 347,
+ 573, 158, 573, 573, 573, 573, 573, 573, 573, 352,
+ 573, 269, 269, 269, 269, 269, 269, 270, 270, 270,
+ 271, 357, 573, 573, 573, 573, 573, 573, 573, 573,
+ 348, 348, 348, 348, 348, 348, 349, 349, 349, 350,
+
+ 353, 353, 353, 353, 353, 353, 354, 354, 354, 355,
+ 347, 158, 358, 358, 358, 358, 358, 358, 359, 359,
+ 359, 360, 347, 158, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 158, 573, 573, 573, 573, 573, 573,
+ 573, 348, 348, 348, 348, 348, 348, 349, 349, 349,
+ 350, 573, 158, 348, 348, 348, 348, 348, 348, 349,
+ 349, 349, 350, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 421, 573, 158, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 426, 573, 573, 573, 573, 573,
+ 573, 573, 573, 158, 431, 573, 573, 573, 573, 573,
+
+ 573, 573, 573, 422, 422, 422, 422, 422, 422, 423,
+ 423, 423, 424, 368, 573, 427, 427, 427, 427, 427,
+ 427, 428, 428, 428, 429, 432, 432, 432, 432, 432,
+ 432, 433, 433, 433, 434, 368, 158, 573, 573, 573,
+ 573, 573, 573, 573, 369, 369, 369, 369, 369, 369,
+ 370, 370, 370, 371, 436, 158, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 369, 369, 369, 369,
+ 369, 369, 370, 370, 370, 371, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 437, 437, 437, 437, 437,
+ 437, 438, 438, 438, 439, 573, 573, 441, 573, 573,
+
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 446, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 442, 442,
+ 442, 442, 442, 442, 443, 443, 443, 444, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 447, 447, 447,
+ 447, 447, 447, 448, 448, 448, 449, 59, 59, 59,
59, 59, 59, 59, 59, 59, 59, 59, 60, 60,
60, 60, 60, 60, 60, 60, 60, 60, 60, 65,
65, 65, 65, 65, 65, 65, 65, 65, 65, 65,
@@ -990,173 +990,173 @@ static yyconst flex_int16_t yy_nxt[4546] =
69, 77, 77, 77, 77, 77, 77, 77, 77, 77,
77, 77, 79, 79, 79, 79, 79, 79, 79, 79,
86, 86, 86, 86, 86, 86, 86, 86, 91, 91,
- 158, 158, 158, 158, 158, 158, 158, 572, 158, 158,
- 158, 91, 572, 91, 91, 91, 91, 91, 91, 91,
- 91, 91, 166, 572, 166, 166, 166, 166, 166, 166,
+ 158, 158, 158, 158, 158, 158, 158, 573, 158, 158,
+ 158, 91, 573, 91, 91, 91, 91, 91, 91, 91,
+ 91, 91, 166, 573, 166, 166, 166, 166, 166, 166,
166, 166, 166, 167, 167, 167, 167, 167, 167, 167,
- 167, 167, 167, 167, 176, 572, 176, 176, 176, 176,
- 176, 176, 176, 176, 176, 177, 572, 177, 177, 177,
+ 167, 167, 167, 167, 176, 573, 176, 176, 176, 176,
+ 176, 176, 176, 176, 176, 177, 573, 177, 177, 177,
177, 177, 177, 177, 177, 177, 178, 178, 178, 178,
- 178, 178, 178, 178, 183, 572, 183, 183, 183, 183,
- 183, 183, 183, 183, 183, 184, 572, 184, 184, 184,
+ 178, 178, 178, 178, 183, 573, 183, 183, 183, 183,
+ 183, 183, 183, 183, 183, 184, 573, 184, 184, 184,
184, 184, 184, 184, 184, 184, 185, 185, 185, 185,
- 185, 185, 185, 185, 572, 572, 185, 190, 572, 190,
- 190, 190, 190, 190, 190, 190, 190, 190, 191, 572,
+ 185, 185, 185, 185, 573, 573, 185, 190, 573, 190,
+ 190, 190, 190, 190, 190, 190, 190, 190, 191, 573,
191, 191, 191, 191, 191, 191, 191, 191, 191, 192,
- 192, 192, 192, 192, 192, 192, 192, 572, 572, 192,
- 197, 572, 197, 197, 197, 197, 197, 197, 197, 197,
- 197, 198, 572, 198, 198, 198, 198, 198, 198, 198,
+ 192, 192, 192, 192, 192, 192, 192, 573, 573, 192,
+ 197, 573, 197, 197, 197, 197, 197, 197, 197, 197,
+ 197, 198, 573, 198, 198, 198, 198, 198, 198, 198,
198, 198, 77, 77, 77, 77, 77, 77, 77, 77,
77, 77, 77, 79, 79, 79, 79, 79, 79, 79,
- 79, 200, 572, 200, 200, 200, 200, 200, 200, 200,
- 200, 200, 201, 572, 201, 201, 201, 201, 201, 201,
- 201, 201, 201, 86, 572, 86, 86, 86, 86, 86,
- 86, 86, 86, 86, 213, 572, 213, 213, 213, 213,
- 213, 213, 213, 213, 213, 214, 572, 214, 214, 214,
+ 79, 200, 573, 200, 200, 200, 200, 200, 200, 200,
+ 200, 200, 201, 573, 201, 201, 201, 201, 201, 201,
+ 201, 201, 201, 86, 573, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 213, 573, 213, 213, 213, 213,
+ 213, 213, 213, 213, 213, 214, 573, 214, 214, 214,
214, 214, 214, 214, 214, 214, 86, 86, 86, 86,
- 86, 86, 86, 86, 91, 91, 91, 572, 91, 91,
- 91, 91, 91, 91, 91, 91, 91, 166, 572, 166,
+ 86, 86, 86, 86, 91, 91, 91, 573, 91, 91,
+ 91, 91, 91, 91, 91, 91, 91, 166, 573, 166,
166, 166, 166, 166, 166, 166, 166, 166, 167, 167,
167, 167, 167, 167, 167, 167, 167, 167, 167, 176,
- 572, 176, 176, 176, 176, 176, 176, 176, 176, 176,
- 177, 572, 177, 177, 177, 177, 177, 177, 177, 177,
- 177, 240, 240, 572, 240, 240, 240, 240, 240, 240,
- 240, 240, 246, 246, 246, 572, 246, 246, 246, 246,
+ 573, 176, 176, 176, 176, 176, 176, 176, 176, 176,
+ 177, 573, 177, 177, 177, 177, 177, 177, 177, 177,
+ 177, 240, 240, 573, 240, 240, 240, 240, 240, 240,
+ 240, 240, 246, 246, 246, 573, 246, 246, 246, 246,
246, 246, 246, 158, 158, 158, 158, 158, 158, 158,
158, 158, 158, 158, 267, 267, 267, 267, 267, 267,
267, 267, 267, 267, 267, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 277, 277, 277, 277,
- 277, 277, 277, 277, 277, 277, 277, 292, 572, 292,
-
- 292, 292, 292, 292, 292, 292, 292, 292, 178, 178,
- 178, 178, 178, 178, 178, 178, 183, 572, 183, 183,
- 183, 183, 183, 183, 183, 183, 183, 184, 572, 184,
- 184, 184, 184, 184, 184, 184, 184, 184, 293, 572,
- 293, 293, 293, 293, 293, 293, 293, 293, 293, 185,
- 185, 185, 185, 185, 185, 185, 185, 572, 572, 185,
- 190, 572, 190, 190, 190, 190, 190, 190, 190, 190,
- 190, 191, 572, 191, 191, 191, 191, 191, 191, 191,
- 191, 191, 294, 572, 294, 294, 294, 294, 294, 294,
- 294, 294, 294, 192, 192, 192, 192, 192, 192, 192,
-
- 192, 572, 572, 192, 197, 572, 197, 197, 197, 197,
- 197, 197, 197, 197, 197, 198, 572, 198, 198, 198,
- 198, 198, 198, 198, 198, 198, 295, 572, 295, 295,
- 295, 295, 295, 295, 295, 295, 295, 79, 572, 79,
- 79, 79, 79, 79, 79, 79, 79, 79, 296, 572,
- 296, 296, 296, 296, 296, 296, 296, 296, 296, 202,
- 572, 202, 202, 202, 202, 202, 202, 202, 202, 202,
- 297, 572, 297, 297, 297, 297, 297, 297, 297, 297,
- 297, 298, 298, 298, 298, 298, 298, 298, 298, 298,
- 298, 298, 307, 572, 307, 307, 307, 307, 307, 307,
-
- 307, 307, 307, 308, 572, 308, 308, 308, 308, 308,
- 308, 308, 308, 308, 86, 572, 86, 86, 86, 86,
- 86, 86, 86, 86, 86, 309, 572, 309, 309, 309,
- 309, 309, 309, 309, 309, 309, 91, 91, 91, 91,
- 91, 91, 91, 91, 91, 572, 91, 240, 240, 240,
+ 277, 277, 277, 277, 277, 277, 277, 293, 573, 293,
+
+ 293, 293, 293, 293, 293, 293, 293, 293, 178, 178,
+ 178, 178, 178, 178, 178, 178, 183, 573, 183, 183,
+ 183, 183, 183, 183, 183, 183, 183, 184, 573, 184,
+ 184, 184, 184, 184, 184, 184, 184, 184, 294, 573,
+ 294, 294, 294, 294, 294, 294, 294, 294, 294, 185,
+ 185, 185, 185, 185, 185, 185, 185, 573, 573, 185,
+ 190, 573, 190, 190, 190, 190, 190, 190, 190, 190,
+ 190, 191, 573, 191, 191, 191, 191, 191, 191, 191,
+ 191, 191, 295, 573, 295, 295, 295, 295, 295, 295,
+ 295, 295, 295, 192, 192, 192, 192, 192, 192, 192,
+
+ 192, 573, 573, 192, 197, 573, 197, 197, 197, 197,
+ 197, 197, 197, 197, 197, 198, 573, 198, 198, 198,
+ 198, 198, 198, 198, 198, 198, 296, 573, 296, 296,
+ 296, 296, 296, 296, 296, 296, 296, 79, 573, 79,
+ 79, 79, 79, 79, 79, 79, 79, 79, 297, 573,
+ 297, 297, 297, 297, 297, 297, 297, 297, 297, 202,
+ 573, 202, 202, 202, 202, 202, 202, 202, 202, 202,
+ 298, 573, 298, 298, 298, 298, 298, 298, 298, 298,
+ 298, 299, 299, 299, 299, 299, 299, 299, 299, 299,
+ 299, 299, 308, 573, 308, 308, 308, 308, 308, 308,
+
+ 308, 308, 308, 309, 573, 309, 309, 309, 309, 309,
+ 309, 309, 309, 309, 86, 573, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 310, 573, 310, 310, 310,
+ 310, 310, 310, 310, 310, 310, 91, 91, 91, 91,
+ 91, 91, 91, 91, 91, 573, 91, 240, 240, 240,
240, 240, 240, 240, 240, 240, 240, 240, 158, 158,
158, 158, 158, 158, 158, 158, 158, 158, 158, 267,
267, 267, 267, 267, 267, 267, 267, 267, 267, 267,
272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
272, 277, 277, 277, 277, 277, 277, 277, 277, 277,
- 277, 277, 345, 345, 345, 345, 345, 345, 345, 345,
- 345, 345, 345, 350, 350, 350, 350, 350, 350, 350,
- 350, 350, 350, 350, 355, 355, 355, 355, 355, 355,
- 355, 355, 355, 355, 355, 360, 360, 360, 360, 360,
- 360, 360, 360, 360, 360, 360, 365, 572, 365, 365,
- 365, 365, 365, 365, 365, 365, 365, 366, 366, 366,
- 366, 366, 366, 366, 366, 366, 366, 366, 371, 371,
- 371, 371, 371, 371, 371, 371, 371, 371, 371, 376,
- 376, 376, 376, 376, 376, 376, 376, 376, 376, 376,
- 178, 572, 178, 178, 178, 178, 178, 178, 178, 178,
-
- 178, 185, 572, 185, 185, 185, 185, 185, 185, 185,
- 185, 185, 192, 572, 192, 192, 192, 192, 192, 192,
- 192, 192, 192, 79, 572, 79, 79, 79, 79, 79,
- 79, 79, 79, 79, 202, 572, 202, 202, 202, 202,
- 202, 202, 202, 202, 202, 382, 572, 382, 382, 382,
- 382, 382, 382, 382, 382, 382, 86, 572, 86, 86,
- 86, 86, 86, 86, 86, 86, 86, 388, 388, 388,
- 388, 388, 388, 572, 388, 572, 572, 388, 392, 392,
- 392, 392, 392, 392, 572, 392, 572, 572, 392, 396,
- 396, 572, 396, 396, 396, 396, 396, 396, 396, 396,
+ 277, 277, 346, 346, 346, 346, 346, 346, 346, 346,
+ 346, 346, 346, 351, 351, 351, 351, 351, 351, 351,
+ 351, 351, 351, 351, 356, 356, 356, 356, 356, 356,
+ 356, 356, 356, 356, 356, 361, 361, 361, 361, 361,
+ 361, 361, 361, 361, 361, 361, 366, 573, 366, 366,
+ 366, 366, 366, 366, 366, 366, 366, 367, 367, 367,
+ 367, 367, 367, 367, 367, 367, 367, 367, 372, 372,
+ 372, 372, 372, 372, 372, 372, 372, 372, 372, 377,
+ 377, 377, 377, 377, 377, 377, 377, 377, 377, 377,
+ 178, 573, 178, 178, 178, 178, 178, 178, 178, 178,
+
+ 178, 185, 573, 185, 185, 185, 185, 185, 185, 185,
+ 185, 185, 192, 573, 192, 192, 192, 192, 192, 192,
+ 192, 192, 192, 79, 573, 79, 79, 79, 79, 79,
+ 79, 79, 79, 79, 202, 573, 202, 202, 202, 202,
+ 202, 202, 202, 202, 202, 383, 573, 383, 383, 383,
+ 383, 383, 383, 383, 383, 383, 86, 573, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 389, 389, 389,
+ 389, 389, 389, 573, 389, 573, 573, 389, 393, 393,
+ 393, 393, 393, 393, 573, 393, 573, 573, 393, 397,
+ 397, 573, 397, 397, 397, 397, 397, 397, 397, 397,
158, 158, 158, 158, 158, 158, 158, 158, 158, 158,
158, 267, 267, 267, 267, 267, 267, 267, 267, 267,
267, 267, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 277, 277, 277, 277, 277, 277, 277,
- 277, 277, 277, 277, 345, 345, 345, 345, 345, 345,
- 345, 345, 345, 345, 345, 350, 350, 350, 350, 350,
- 350, 350, 350, 350, 350, 350, 355, 355, 355, 355,
- 355, 355, 355, 355, 355, 355, 355, 360, 360, 360,
- 360, 360, 360, 360, 360, 360, 360, 360, 366, 366,
- 366, 366, 366, 366, 366, 366, 366, 366, 366, 371,
-
- 371, 371, 371, 371, 371, 371, 371, 371, 371, 371,
- 376, 376, 376, 376, 376, 376, 376, 376, 376, 376,
- 376, 419, 419, 419, 419, 419, 419, 419, 419, 419,
- 419, 419, 424, 424, 424, 424, 424, 424, 424, 424,
- 424, 424, 424, 429, 429, 429, 429, 429, 429, 429,
- 429, 429, 429, 429, 434, 434, 434, 434, 434, 434,
- 434, 434, 434, 434, 434, 439, 439, 439, 439, 439,
- 439, 439, 439, 439, 439, 439, 444, 444, 444, 444,
- 444, 444, 444, 444, 444, 444, 444, 202, 572, 202,
- 202, 202, 202, 202, 202, 202, 202, 202, 388, 388,
-
- 388, 388, 388, 388, 388, 388, 572, 572, 388, 450,
- 572, 450, 450, 450, 450, 450, 450, 450, 450, 450,
- 451, 572, 451, 451, 451, 451, 451, 451, 451, 451,
- 451, 392, 392, 392, 392, 392, 392, 392, 392, 572,
- 572, 392, 453, 572, 453, 453, 453, 453, 453, 453,
- 453, 453, 453, 454, 572, 454, 454, 454, 454, 454,
- 454, 454, 454, 454, 396, 396, 396, 396, 396, 396,
- 396, 396, 396, 396, 396, 457, 457, 572, 457, 457,
- 457, 457, 457, 457, 457, 457, 158, 158, 158, 158,
+ 277, 277, 277, 277, 346, 346, 346, 346, 346, 346,
+ 346, 346, 346, 346, 346, 351, 351, 351, 351, 351,
+ 351, 351, 351, 351, 351, 351, 356, 356, 356, 356,
+ 356, 356, 356, 356, 356, 356, 356, 361, 361, 361,
+ 361, 361, 361, 361, 361, 361, 361, 361, 367, 367,
+ 367, 367, 367, 367, 367, 367, 367, 367, 367, 372,
+
+ 372, 372, 372, 372, 372, 372, 372, 372, 372, 372,
+ 377, 377, 377, 377, 377, 377, 377, 377, 377, 377,
+ 377, 420, 420, 420, 420, 420, 420, 420, 420, 420,
+ 420, 420, 425, 425, 425, 425, 425, 425, 425, 425,
+ 425, 425, 425, 430, 430, 430, 430, 430, 430, 430,
+ 430, 430, 430, 430, 435, 435, 435, 435, 435, 435,
+ 435, 435, 435, 435, 435, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 445, 445, 445, 445,
+ 445, 445, 445, 445, 445, 445, 445, 202, 573, 202,
+ 202, 202, 202, 202, 202, 202, 202, 202, 389, 389,
+
+ 389, 389, 389, 389, 389, 389, 573, 573, 389, 451,
+ 573, 451, 451, 451, 451, 451, 451, 451, 451, 451,
+ 452, 573, 452, 452, 452, 452, 452, 452, 452, 452,
+ 452, 393, 393, 393, 393, 393, 393, 393, 393, 573,
+ 573, 393, 454, 573, 454, 454, 454, 454, 454, 454,
+ 454, 454, 454, 455, 573, 455, 455, 455, 455, 455,
+ 455, 455, 455, 455, 397, 397, 397, 397, 397, 397,
+ 397, 397, 397, 397, 397, 458, 458, 573, 458, 458,
+ 458, 458, 458, 458, 458, 458, 158, 158, 158, 158,
158, 158, 158, 158, 158, 158, 158, 267, 267, 267,
267, 267, 267, 267, 267, 267, 267, 267, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272, 277,
277, 277, 277, 277, 277, 277, 277, 277, 277, 277,
- 345, 345, 345, 345, 345, 345, 345, 345, 345, 345,
- 345, 350, 350, 350, 350, 350, 350, 350, 350, 350,
- 350, 350, 355, 355, 355, 355, 355, 355, 355, 355,
- 355, 355, 355, 360, 360, 360, 360, 360, 360, 360,
- 360, 360, 360, 360, 366, 366, 366, 366, 366, 366,
- 366, 366, 366, 366, 366, 371, 371, 371, 371, 371,
- 371, 371, 371, 371, 371, 371, 376, 376, 376, 376,
-
- 376, 376, 376, 376, 376, 376, 376, 419, 419, 419,
- 419, 419, 419, 419, 419, 419, 419, 419, 424, 424,
- 424, 424, 424, 424, 424, 424, 424, 424, 424, 429,
- 429, 429, 429, 429, 429, 429, 429, 429, 429, 429,
- 434, 434, 434, 434, 434, 434, 434, 434, 434, 434,
- 434, 439, 439, 439, 439, 439, 439, 439, 439, 439,
- 439, 439, 444, 444, 444, 444, 444, 444, 444, 444,
- 444, 444, 444, 388, 572, 388, 388, 388, 388, 388,
- 388, 388, 388, 388, 477, 572, 477, 477, 477, 477,
- 477, 477, 477, 477, 477, 392, 572, 392, 392, 392,
-
- 392, 392, 392, 392, 392, 392, 478, 572, 478, 478,
- 478, 478, 478, 478, 478, 478, 478, 479, 479, 572,
- 479, 479, 479, 479, 479, 479, 479, 479, 457, 457,
- 457, 457, 457, 457, 457, 457, 457, 457, 457, 15,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
-
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572
+ 346, 346, 346, 346, 346, 346, 346, 346, 346, 346,
+ 346, 351, 351, 351, 351, 351, 351, 351, 351, 351,
+ 351, 351, 356, 356, 356, 356, 356, 356, 356, 356,
+ 356, 356, 356, 361, 361, 361, 361, 361, 361, 361,
+ 361, 361, 361, 361, 367, 367, 367, 367, 367, 367,
+ 367, 367, 367, 367, 367, 372, 372, 372, 372, 372,
+ 372, 372, 372, 372, 372, 372, 377, 377, 377, 377,
+
+ 377, 377, 377, 377, 377, 377, 377, 420, 420, 420,
+ 420, 420, 420, 420, 420, 420, 420, 420, 425, 425,
+ 425, 425, 425, 425, 425, 425, 425, 425, 425, 430,
+ 430, 430, 430, 430, 430, 430, 430, 430, 430, 430,
+ 435, 435, 435, 435, 435, 435, 435, 435, 435, 435,
+ 435, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 445, 445, 445, 445, 445, 445, 445, 445,
+ 445, 445, 445, 389, 573, 389, 389, 389, 389, 389,
+ 389, 389, 389, 389, 478, 573, 478, 478, 478, 478,
+ 478, 478, 478, 478, 478, 393, 573, 393, 393, 393,
+
+ 393, 393, 393, 393, 393, 393, 479, 573, 479, 479,
+ 479, 479, 479, 479, 479, 479, 479, 480, 480, 573,
+ 480, 480, 480, 480, 480, 480, 480, 480, 458, 458,
+ 458, 458, 458, 458, 458, 458, 458, 458, 458, 15,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573
} ;
static yyconst flex_int16_t yy_chk[4546] =
@@ -1173,7 +1173,7 @@ static yyconst flex_int16_t yy_chk[4546] =
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 5, 17, 74, 22, 17,
- 74, 128, 225, 128, 33, 632, 225, 21, 22, 23,
+ 74, 128, 225, 128, 33, 633, 225, 21, 22, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 33,
25, 35, 36, 25, 220, 40, 104, 106, 35, 41,
50, 26, 36, 37, 36, 106, 37, 106, 104, 26,
@@ -1187,484 +1187,484 @@ static yyconst flex_int16_t yy_chk[4546] =
21, 21, 21, 21, 21, 25, 25, 25, 25, 25,
25, 25, 25, 25, 25, 26, 168, 50, 37, 37,
37, 37, 37, 37, 37, 37, 37, 37, 44, 51,
- 53, 95, 94, 97, 600, 44, 204, 44, 53, 44,
- 44, 97, 44, 570, 44, 44, 44, 144, 144, 144,
- 292, 44, 568, 44, 6, 6, 6, 6, 6, 6,
+ 53, 95, 94, 97, 601, 44, 204, 44, 53, 44,
+ 44, 97, 44, 571, 44, 44, 44, 144, 144, 144,
+ 293, 44, 569, 44, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
- 6, 6, 6, 9, 39, 43, 43, 43, 60, 540,
- 43, 227, 43, 566, 227, 39, 153, 168, 79, 145,
+ 6, 6, 6, 9, 39, 43, 43, 43, 60, 541,
+ 43, 227, 43, 567, 227, 39, 153, 168, 79, 145,
145, 145, 153, 39, 153, 39, 46, 46, 46, 46,
46, 46, 46, 46, 46, 46, 65, 146, 146, 146,
- 540, 564, 148, 248, 248, 248, 43, 69, 148, 148,
- 205, 292, 39, 39, 39, 562, 9, 9, 9, 9,
+ 541, 565, 148, 248, 248, 248, 43, 69, 148, 148,
+ 205, 293, 39, 39, 39, 563, 9, 9, 9, 9,
9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 10, 466, 60, 60, 60, 60,
+ 9, 9, 9, 9, 10, 467, 60, 60, 60, 60,
60, 60, 60, 60, 60, 60, 79, 79, 79, 79,
79, 79, 79, 79, 79, 79, 86, 65, 65, 65,
65, 65, 65, 65, 65, 65, 65, 205, 69, 69,
69, 69, 69, 69, 69, 69, 69, 69, 91, 171,
- 466, 91, 249, 249, 249, 561, 169, 10, 10, 10,
+ 467, 91, 249, 249, 249, 562, 169, 10, 10, 10,
10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
10, 10, 10, 10, 10, 11, 90, 90, 90, 90,
- 90, 90, 90, 90, 90, 90, 216, 216, 207, 560,
+ 90, 90, 90, 90, 90, 90, 216, 216, 207, 561,
174, 86, 86, 86, 86, 86, 86, 86, 86, 86,
- 86, 169, 169, 559, 171, 171, 173, 217, 217, 558,
- 221, 557, 216, 91, 91, 91, 91, 91, 91, 91,
- 91, 91, 91, 217, 221, 556, 221, 185, 11, 11,
+ 86, 169, 169, 560, 171, 171, 173, 217, 217, 559,
+ 221, 558, 216, 91, 91, 91, 91, 91, 91, 91,
+ 91, 91, 91, 217, 221, 557, 221, 185, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 12, 158, 207, 174,
- 173, 174, 174, 173, 554, 173, 207, 192, 173, 299,
- 174, 174, 553, 219, 173, 219, 219, 331, 173, 173,
+ 173, 174, 174, 173, 555, 173, 207, 192, 173, 300,
+ 174, 174, 554, 219, 173, 219, 219, 332, 173, 173,
- 173, 173, 173, 552, 219, 550, 548, 331, 158, 158,
+ 173, 173, 173, 553, 219, 551, 549, 332, 158, 158,
158, 158, 158, 158, 158, 158, 158, 158, 185, 185,
- 185, 185, 185, 185, 185, 185, 185, 185, 300, 12,
+ 185, 185, 185, 185, 185, 185, 185, 185, 301, 12,
12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
12, 12, 12, 12, 12, 12, 12, 32, 192, 192,
192, 192, 192, 192, 192, 192, 192, 192, 32, 32,
- 299, 32, 32, 258, 202, 258, 327, 302, 547, 546,
- 32, 32, 545, 300, 300, 32, 258, 159, 327, 329,
+ 300, 32, 32, 258, 202, 258, 328, 303, 548, 547,
+ 32, 32, 546, 301, 301, 32, 258, 159, 328, 330,
- 544, 543, 538, 329, 32, 329, 32, 160, 32, 537,
- 329, 32, 32, 536, 32, 534, 533, 32, 161, 32,
+ 545, 544, 539, 330, 32, 330, 32, 160, 32, 538,
+ 330, 32, 32, 537, 32, 535, 534, 32, 161, 32,
56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 61, 159, 532,
- 531, 178, 302, 302, 328, 530, 529, 399, 160, 202,
+ 56, 56, 56, 56, 56, 56, 56, 61, 159, 533,
+ 532, 178, 303, 303, 329, 531, 530, 400, 160, 202,
202, 202, 202, 202, 202, 202, 202, 202, 202, 161,
- 328, 328, 328, 399, 399, 399, 305, 528, 527, 159,
+ 329, 329, 329, 400, 400, 400, 306, 529, 528, 159,
159, 159, 159, 159, 159, 159, 159, 159, 159, 160,
- 160, 160, 160, 160, 160, 160, 160, 160, 160, 526,
+ 160, 160, 160, 160, 160, 160, 160, 160, 160, 527,
161, 161, 161, 161, 161, 161, 161, 161, 161, 161,
- 525, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 526, 61, 61, 61, 61, 61, 61, 61, 61, 61,
61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
61, 61, 61, 61, 61, 61, 61, 61, 61, 178,
178, 178, 178, 178, 178, 178, 178, 178, 178, 61,
- 66, 218, 267, 522, 218, 305, 218, 305, 305, 218,
- 520, 518, 268, 250, 517, 218, 305, 305, 516, 218,
- 218, 218, 218, 218, 515, 250, 514, 513, 218, 250,
+ 66, 218, 267, 523, 218, 306, 218, 306, 306, 218,
+ 521, 519, 268, 250, 518, 218, 306, 306, 517, 218,
+ 218, 218, 218, 218, 516, 250, 515, 514, 218, 250,
250, 250, 269, 267, 267, 267, 267, 267, 267, 267,
267, 267, 267, 268, 268, 268, 268, 268, 268, 268,
- 268, 268, 268, 270, 321, 321, 321, 321, 321, 321,
- 321, 321, 321, 321, 66, 66, 66, 66, 66, 66,
+ 268, 268, 268, 270, 322, 322, 322, 322, 322, 322,
+ 322, 322, 322, 322, 66, 66, 66, 66, 66, 66,
66, 66, 66, 66, 66, 66, 66, 66, 66, 66,
66, 66, 66, 66, 66, 66, 66, 66, 66, 66,
- 66, 66, 70, 269, 271, 322, 322, 322, 322, 322,
- 322, 322, 322, 322, 322, 512, 511, 510, 509, 507,
- 505, 504, 503, 502, 270, 501, 499, 498, 496, 495,
- 494, 493, 492, 491, 269, 269, 269, 269, 269, 269,
- 269, 269, 269, 269, 304, 272, 490, 489, 487, 486,
+ 66, 66, 70, 269, 271, 323, 323, 323, 323, 323,
+ 323, 323, 323, 323, 323, 513, 512, 511, 510, 508,
+ 506, 505, 504, 503, 270, 502, 500, 499, 497, 496,
+ 495, 494, 493, 492, 269, 269, 269, 269, 269, 269,
+ 269, 269, 269, 269, 305, 272, 491, 490, 488, 487,
- 485, 484, 400, 400, 400, 270, 270, 270, 270, 270,
+ 486, 485, 401, 401, 401, 270, 270, 270, 270, 270,
270, 270, 270, 270, 270, 271, 70, 70, 70, 70,
70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
- 70, 70, 70, 70, 81, 400, 271, 271, 271, 271,
- 271, 271, 271, 271, 271, 271, 272, 483, 304, 482,
- 481, 304, 479, 304, 476, 475, 304, 345, 474, 473,
- 472, 471, 304, 470, 469, 468, 304, 304, 304, 304,
- 304, 467, 465, 464, 463, 462, 461, 272, 272, 272,
- 272, 272, 272, 272, 272, 272, 272, 460, 345, 345,
-
- 345, 345, 345, 345, 345, 345, 345, 345, 81, 81,
+ 70, 70, 70, 70, 81, 401, 271, 271, 271, 271,
+ 271, 271, 271, 271, 271, 271, 272, 484, 305, 483,
+ 482, 305, 480, 305, 477, 476, 305, 346, 475, 474,
+ 473, 472, 305, 471, 470, 469, 305, 305, 305, 305,
+ 305, 468, 466, 465, 464, 463, 462, 272, 272, 272,
+ 272, 272, 272, 272, 272, 272, 272, 461, 346, 346,
+
+ 346, 346, 346, 346, 346, 346, 346, 346, 81, 81,
81, 81, 81, 81, 81, 81, 81, 81, 81, 81,
81, 81, 81, 81, 81, 81, 81, 81, 81, 81,
- 81, 81, 81, 81, 81, 81, 274, 459, 457, 418,
- 417, 416, 415, 414, 413, 412, 81, 100, 100, 100,
+ 81, 81, 81, 81, 81, 81, 274, 460, 458, 419,
+ 418, 417, 416, 415, 414, 413, 81, 100, 100, 100,
100, 100, 100, 100, 100, 100, 100, 100, 100, 100,
100, 100, 100, 100, 100, 100, 100, 100, 100, 100,
100, 100, 100, 100, 100, 100, 100, 100, 100, 100,
- 100, 100, 100, 100, 180, 346, 411, 410, 409, 408,
- 407, 406, 405, 404, 403, 402, 401, 274, 397, 396,
-
- 381, 377, 372, 365, 361, 344, 343, 342, 341, 340,
- 338, 337, 336, 335, 334, 275, 346, 346, 346, 346,
- 346, 346, 346, 346, 346, 346, 332, 276, 274, 274,
- 274, 274, 274, 274, 274, 274, 274, 274, 277, 330,
- 326, 325, 319, 317, 314, 313, 311, 301, 180, 180,
+ 100, 100, 100, 100, 180, 347, 412, 411, 410, 409,
+ 408, 407, 406, 405, 404, 403, 402, 274, 398, 397,
+
+ 382, 378, 373, 366, 362, 345, 344, 343, 342, 341,
+ 339, 338, 337, 336, 335, 275, 347, 347, 347, 347,
+ 347, 347, 347, 347, 347, 347, 333, 276, 274, 274,
+ 274, 274, 274, 274, 274, 274, 274, 274, 277, 331,
+ 327, 326, 320, 318, 315, 314, 312, 302, 180, 180,
180, 180, 180, 180, 180, 180, 180, 180, 180, 180,
180, 180, 180, 180, 180, 180, 180, 180, 180, 180,
- 180, 180, 180, 180, 180, 180, 275, 282, 273, 265,
+ 180, 180, 180, 180, 180, 180, 275, 283, 273, 265,
264, 263, 262, 261, 260, 259, 180, 187, 276, 279,
257, 256, 255, 254, 253, 252, 245, 244, 243, 277,
- 240, 226, 224, 223, 222, 170, 157, 275, 275, 275,
- 275, 275, 275, 275, 275, 275, 275, 156, 155, 276,
- 276, 276, 276, 276, 276, 276, 276, 276, 276, 388,
+ 240, 226, 224, 223, 222, 170, 162, 275, 275, 275,
+ 275, 275, 275, 275, 275, 275, 275, 157, 156, 276,
+ 276, 276, 276, 276, 276, 276, 276, 276, 276, 389,
277, 277, 277, 277, 277, 277, 277, 277, 277, 277,
- 280, 392, 154, 152, 151, 150, 149, 147, 142, 139,
+ 280, 393, 155, 154, 152, 151, 150, 149, 147, 142,
279, 187, 187, 187, 187, 187, 187, 187, 187, 187,
187, 187, 187, 187, 187, 187, 187, 187, 187, 187,
187, 187, 187, 187, 187, 187, 187, 187, 187, 194,
- 138, 279, 279, 279, 279, 279, 279, 279, 279, 279,
- 279, 137, 135, 134, 125, 124, 123, 121, 120, 116,
-
- 115, 280, 112, 111, 110, 109, 107, 105, 103, 77,
- 388, 388, 388, 388, 388, 388, 388, 388, 388, 388,
- 76, 75, 392, 392, 392, 392, 392, 392, 392, 392,
- 392, 392, 280, 280, 280, 280, 280, 280, 280, 280,
- 280, 280, 47, 194, 194, 194, 194, 194, 194, 194,
+ 139, 279, 279, 279, 279, 279, 279, 279, 279, 279,
+ 279, 138, 137, 135, 134, 125, 124, 123, 121, 120,
+
+ 116, 280, 115, 112, 111, 110, 109, 107, 105, 103,
+ 389, 389, 389, 389, 389, 389, 389, 389, 389, 389,
+ 77, 76, 393, 393, 393, 393, 393, 393, 393, 393,
+ 393, 393, 280, 280, 280, 280, 280, 280, 280, 280,
+ 280, 280, 75, 194, 194, 194, 194, 194, 194, 194,
194, 194, 194, 194, 194, 194, 194, 194, 194, 194,
194, 194, 194, 194, 194, 194, 194, 194, 194, 194,
194, 210, 210, 210, 210, 210, 210, 210, 210, 210,
210, 210, 210, 210, 210, 210, 210, 210, 210, 210,
210, 210, 210, 210, 210, 210, 210, 210, 210, 210,
- 210, 210, 210, 210, 210, 210, 210, 210, 281, 419,
- 38, 31, 30, 20, 19, 18, 15, 14, 347, 420,
- 13, 8, 7, 0, 0, 0, 0, 0, 348, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 349, 0,
- 419, 419, 419, 419, 419, 419, 419, 419, 419, 419,
+ 210, 210, 210, 210, 210, 210, 210, 210, 281, 420,
+ 47, 38, 31, 30, 20, 19, 18, 15, 348, 421,
+ 14, 13, 8, 7, 0, 0, 0, 0, 349, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 350, 0,
420, 420, 420, 420, 420, 420, 420, 420, 420, 420,
+ 421, 421, 421, 421, 421, 421, 421, 421, 421, 421,
0, 0, 0, 0, 0, 0, 0, 0, 0, 281,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 347,
- 350, 0, 0, 0, 0, 0, 0, 0, 0, 348,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 348,
351, 0, 0, 0, 0, 0, 0, 0, 0, 349,
+ 352, 0, 0, 0, 0, 0, 0, 0, 0, 350,
281, 281, 281, 281, 281, 281, 281, 281, 281, 281,
- 347, 347, 347, 347, 347, 347, 347, 347, 347, 347,
348, 348, 348, 348, 348, 348, 348, 348, 348, 348,
349, 349, 349, 349, 349, 349, 349, 349, 349, 349,
- 352, 350, 0, 0, 0, 0, 0, 0, 0, 0,
+ 350, 350, 350, 350, 350, 350, 350, 350, 350, 350,
353, 351, 0, 0, 0, 0, 0, 0, 0, 0,
- 354, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 355, 0, 350, 350, 350, 350, 350, 350, 350, 350,
- 350, 350, 351, 351, 351, 351, 351, 351, 351, 351,
- 351, 351, 0, 0, 0, 0, 0, 0, 0, 0,
+ 354, 352, 0, 0, 0, 0, 0, 0, 0, 0,
+ 355, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 356, 0, 351, 351, 351, 351, 351, 351, 351, 351,
+ 351, 351, 352, 352, 352, 352, 352, 352, 352, 352,
+ 352, 352, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 352, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 353, 356, 0, 0, 0, 0, 0, 0, 0,
+ 0, 353, 0, 0, 0, 0, 0, 0, 0, 0,
0, 354, 357, 0, 0, 0, 0, 0, 0, 0,
- 0, 355, 352, 352, 352, 352, 352, 352, 352, 352,
- 352, 352, 353, 353, 353, 353, 353, 353, 353, 353,
+ 0, 355, 358, 0, 0, 0, 0, 0, 0, 0,
+ 0, 356, 353, 353, 353, 353, 353, 353, 353, 353,
353, 353, 354, 354, 354, 354, 354, 354, 354, 354,
354, 354, 355, 355, 355, 355, 355, 355, 355, 355,
- 355, 355, 358, 356, 0, 0, 0, 0, 0, 0,
- 0, 0, 359, 357, 0, 0, 0, 0, 0, 0,
- 0, 0, 360, 0, 0, 0, 0, 0, 0, 0,
-
- 0, 0, 362, 0, 356, 356, 356, 356, 356, 356,
- 356, 356, 356, 356, 357, 357, 357, 357, 357, 357,
- 357, 357, 357, 357, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 358, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 359, 363, 0, 0, 0, 0, 0,
+ 355, 355, 356, 356, 356, 356, 356, 356, 356, 356,
+ 356, 356, 359, 357, 0, 0, 0, 0, 0, 0,
+ 0, 0, 360, 358, 0, 0, 0, 0, 0, 0,
+ 0, 0, 361, 0, 0, 0, 0, 0, 0, 0,
+
+ 0, 0, 363, 0, 357, 357, 357, 357, 357, 357,
+ 357, 357, 357, 357, 358, 358, 358, 358, 358, 358,
+ 358, 358, 358, 358, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 359, 0, 0, 0, 0, 0, 0,
0, 0, 0, 360, 364, 0, 0, 0, 0, 0,
- 0, 0, 0, 362, 358, 358, 358, 358, 358, 358,
- 358, 358, 358, 358, 359, 359, 359, 359, 359, 359,
+ 0, 0, 0, 361, 365, 0, 0, 0, 0, 0,
+ 0, 0, 0, 363, 359, 359, 359, 359, 359, 359,
359, 359, 359, 359, 360, 360, 360, 360, 360, 360,
- 360, 360, 360, 360, 362, 362, 362, 362, 362, 362,
-
- 362, 362, 362, 362, 366, 363, 0, 0, 0, 0,
- 0, 0, 0, 0, 367, 364, 0, 0, 0, 0,
- 0, 0, 0, 0, 368, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 369, 0, 363, 363, 363, 363,
- 363, 363, 363, 363, 363, 363, 364, 364, 364, 364,
- 364, 364, 364, 364, 364, 364, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 366, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 367, 370, 0, 0, 0,
+ 360, 360, 360, 360, 361, 361, 361, 361, 361, 361,
+ 361, 361, 361, 361, 363, 363, 363, 363, 363, 363,
+
+ 363, 363, 363, 363, 367, 364, 0, 0, 0, 0,
+ 0, 0, 0, 0, 368, 365, 0, 0, 0, 0,
+ 0, 0, 0, 0, 369, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 370, 0, 364, 364, 364, 364,
+ 364, 364, 364, 364, 364, 364, 365, 365, 365, 365,
+ 365, 365, 365, 365, 365, 365, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 367, 0, 0, 0, 0,
0, 0, 0, 0, 0, 368, 371, 0, 0, 0,
- 0, 0, 0, 0, 0, 369, 366, 366, 366, 366,
+ 0, 0, 0, 0, 0, 369, 372, 0, 0, 0,
+ 0, 0, 0, 0, 0, 370, 367, 367, 367, 367,
- 366, 366, 366, 366, 366, 366, 367, 367, 367, 367,
367, 367, 367, 367, 367, 367, 368, 368, 368, 368,
368, 368, 368, 368, 368, 368, 369, 369, 369, 369,
- 369, 369, 369, 369, 369, 369, 373, 370, 0, 0,
- 0, 0, 0, 0, 0, 0, 374, 371, 0, 0,
- 0, 0, 0, 0, 0, 0, 375, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 376, 0, 370, 370,
- 370, 370, 370, 370, 370, 370, 370, 370, 371, 371,
- 371, 371, 371, 371, 371, 371, 371, 371, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 373, 0, 0,
-
- 0, 0, 0, 0, 0, 0, 0, 374, 378, 0,
+ 369, 369, 369, 369, 369, 369, 370, 370, 370, 370,
+ 370, 370, 370, 370, 370, 370, 374, 371, 0, 0,
+ 0, 0, 0, 0, 0, 0, 375, 372, 0, 0,
+ 0, 0, 0, 0, 0, 0, 376, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 377, 0, 371, 371,
+ 371, 371, 371, 371, 371, 371, 371, 371, 372, 372,
+ 372, 372, 372, 372, 372, 372, 372, 372, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 374, 0, 0,
+
0, 0, 0, 0, 0, 0, 0, 375, 379, 0,
- 0, 0, 0, 0, 0, 0, 0, 376, 373, 373,
- 373, 373, 373, 373, 373, 373, 373, 373, 374, 374,
+ 0, 0, 0, 0, 0, 0, 0, 376, 380, 0,
+ 0, 0, 0, 0, 0, 0, 0, 377, 374, 374,
374, 374, 374, 374, 374, 374, 374, 374, 375, 375,
375, 375, 375, 375, 375, 375, 375, 375, 376, 376,
- 376, 376, 376, 376, 376, 376, 376, 376, 380, 378,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 379,
- 421, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 376, 376, 376, 376, 376, 376, 376, 376, 377, 377,
+ 377, 377, 377, 377, 377, 377, 377, 377, 381, 379,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 380,
422, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 423, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 378, 378, 378, 378, 378, 378, 378, 378, 378, 378,
379, 379, 379, 379, 379, 379, 379, 379, 379, 379,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 380,
- 0, 423, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 421, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 422, 0, 0, 0, 0, 0, 0, 0, 0,
380, 380, 380, 380, 380, 380, 380, 380, 380, 380,
- 389, 0, 421, 421, 421, 421, 421, 421, 421, 421,
- 421, 421, 422, 422, 422, 422, 422, 422, 422, 422,
- 422, 422, 423, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 381,
+ 0, 424, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 422, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 423, 0, 0, 0, 0, 0, 0, 0, 0,
+ 381, 381, 381, 381, 381, 381, 381, 381, 381, 381,
+ 390, 0, 422, 422, 422, 422, 422, 422, 422, 422,
+ 422, 422, 423, 423, 423, 423, 423, 423, 423, 423,
+ 423, 423, 424, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 424, 0, 0, 0, 0, 0, 0, 0,
0, 0, 425, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 426, 423, 423, 423, 423, 423, 423, 423,
- 423, 423, 423, 0, 389, 389, 389, 389, 389, 389,
- 389, 389, 389, 389, 389, 389, 389, 389, 389, 389,
- 389, 389, 389, 389, 389, 389, 389, 389, 389, 389,
- 389, 389, 393, 424, 427, 0, 0, 0, 0, 0,
- 0, 0, 0, 425, 0, 0, 0, 0, 0, 0,
+ 0, 0, 426, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 427, 424, 424, 424, 424, 424, 424, 424,
+ 424, 424, 424, 0, 390, 390, 390, 390, 390, 390,
+ 390, 390, 390, 390, 390, 390, 390, 390, 390, 390,
+ 390, 390, 390, 390, 390, 390, 390, 390, 390, 390,
+ 390, 390, 394, 425, 428, 0, 0, 0, 0, 0,
0, 0, 0, 426, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 428, 424, 424, 424, 424, 424, 424,
+ 0, 0, 0, 427, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 429, 425, 425, 425, 425, 425, 425,
- 424, 424, 424, 424, 425, 425, 425, 425, 425, 425,
425, 425, 425, 425, 426, 426, 426, 426, 426, 426,
- 426, 426, 426, 426, 0, 427, 393, 393, 393, 393,
- 393, 393, 393, 393, 393, 393, 393, 393, 393, 393,
- 393, 393, 393, 393, 393, 393, 393, 393, 393, 393,
- 393, 393, 393, 393, 428, 429, 427, 427, 427, 427,
- 427, 427, 427, 427, 427, 427, 430, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 431, 0, 0, 0,
- 0, 0, 0, 0, 0, 428, 428, 428, 428, 428,
- 428, 428, 428, 428, 428, 0, 432, 0, 0, 0,
+ 426, 426, 426, 426, 427, 427, 427, 427, 427, 427,
+ 427, 427, 427, 427, 0, 428, 394, 394, 394, 394,
+ 394, 394, 394, 394, 394, 394, 394, 394, 394, 394,
+ 394, 394, 394, 394, 394, 394, 394, 394, 394, 394,
+ 394, 394, 394, 394, 429, 430, 428, 428, 428, 428,
+ 428, 428, 428, 428, 428, 428, 431, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 432, 0, 0, 0,
+ 0, 0, 0, 0, 0, 429, 429, 429, 429, 429,
+ 429, 429, 429, 429, 429, 0, 433, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 429, 433, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 430, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 431, 434, 0,
- 0, 0, 0, 0, 0, 0, 0, 429, 429, 429,
- 429, 429, 429, 429, 429, 429, 429, 432, 430, 430,
- 430, 430, 430, 430, 430, 430, 430, 430, 431, 431,
- 431, 431, 431, 431, 431, 431, 431, 431, 433, 435,
- 0, 0, 0, 0, 0, 0, 0, 0, 432, 432,
- 432, 432, 432, 432, 432, 432, 432, 432, 436, 434,
-
- 0, 0, 0, 0, 0, 0, 0, 0, 437, 433,
- 433, 433, 433, 433, 433, 433, 433, 433, 433, 0,
- 438, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 434, 434, 434, 434, 434, 434, 434, 434, 434, 434,
- 435, 0, 0, 0, 0, 0, 0, 0, 0, 439,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 436,
- 0, 440, 0, 0, 0, 0, 0, 0, 0, 437,
- 0, 435, 435, 435, 435, 435, 435, 435, 435, 435,
- 435, 438, 0, 0, 0, 0, 0, 0, 0, 0,
- 436, 436, 436, 436, 436, 436, 436, 436, 436, 436,
-
+ 0, 0, 0, 0, 0, 0, 430, 434, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 431, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 432, 435, 0,
+ 0, 0, 0, 0, 0, 0, 0, 430, 430, 430,
+ 430, 430, 430, 430, 430, 430, 430, 433, 431, 431,
+ 431, 431, 431, 431, 431, 431, 431, 431, 432, 432,
+ 432, 432, 432, 432, 432, 432, 432, 432, 434, 436,
+ 0, 0, 0, 0, 0, 0, 0, 0, 433, 433,
+ 433, 433, 433, 433, 433, 433, 433, 433, 437, 435,
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 438, 434,
+ 434, 434, 434, 434, 434, 434, 434, 434, 434, 0,
+ 439, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 435, 435, 435, 435, 435, 435, 435, 435, 435, 435,
+ 436, 0, 0, 0, 0, 0, 0, 0, 0, 440,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 437,
+ 0, 441, 0, 0, 0, 0, 0, 0, 0, 438,
+ 0, 436, 436, 436, 436, 436, 436, 436, 436, 436,
+ 436, 439, 0, 0, 0, 0, 0, 0, 0, 0,
437, 437, 437, 437, 437, 437, 437, 437, 437, 437,
- 439, 441, 438, 438, 438, 438, 438, 438, 438, 438,
- 438, 438, 440, 442, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 443, 0, 0, 0, 0, 0, 0,
- 0, 439, 439, 439, 439, 439, 439, 439, 439, 439,
- 439, 0, 444, 440, 440, 440, 440, 440, 440, 440,
- 440, 440, 440, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 441, 0, 445, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 442, 0, 0, 0, 0, 0,
- 0, 0, 0, 446, 443, 0, 0, 0, 0, 0,
-
- 0, 0, 0, 441, 441, 441, 441, 441, 441, 441,
- 441, 441, 441, 444, 0, 442, 442, 442, 442, 442,
- 442, 442, 442, 442, 442, 443, 443, 443, 443, 443,
- 443, 443, 443, 443, 443, 445, 447, 0, 0, 0,
- 0, 0, 0, 0, 444, 444, 444, 444, 444, 444,
- 444, 444, 444, 444, 446, 448, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 445, 445, 445, 445,
- 445, 445, 445, 445, 445, 445, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 446, 446, 446, 446, 446,
- 446, 446, 446, 446, 446, 0, 0, 447, 0, 0,
+
+ 438, 438, 438, 438, 438, 438, 438, 438, 438, 438,
+ 440, 442, 439, 439, 439, 439, 439, 439, 439, 439,
+ 439, 439, 441, 443, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 444, 0, 0, 0, 0, 0, 0,
+ 0, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 0, 445, 441, 441, 441, 441, 441, 441, 441,
+ 441, 441, 441, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 442, 0, 446, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 443, 0, 0, 0, 0, 0,
+ 0, 0, 0, 447, 444, 0, 0, 0, 0, 0,
+
+ 0, 0, 0, 442, 442, 442, 442, 442, 442, 442,
+ 442, 442, 442, 445, 0, 443, 443, 443, 443, 443,
+ 443, 443, 443, 443, 443, 444, 444, 444, 444, 444,
+ 444, 444, 444, 444, 444, 446, 448, 0, 0, 0,
+ 0, 0, 0, 0, 445, 445, 445, 445, 445, 445,
+ 445, 445, 445, 445, 447, 449, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 446, 446, 446, 446,
+ 446, 446, 446, 446, 446, 446, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 447, 447, 447, 447, 447,
+ 447, 447, 447, 447, 447, 0, 0, 448, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 448, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 447, 447,
- 447, 447, 447, 447, 447, 447, 447, 447, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 448, 448, 448,
- 448, 448, 448, 448, 448, 448, 448, 573, 573, 573,
- 573, 573, 573, 573, 573, 573, 573, 573, 574, 574,
- 574, 574, 574, 574, 574, 574, 574, 574, 574, 575,
- 575, 575, 575, 575, 575, 575, 575, 575, 575, 575,
+ 0, 0, 0, 0, 0, 0, 449, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 448, 448,
+ 448, 448, 448, 448, 448, 448, 448, 448, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 449, 449, 449,
+ 449, 449, 449, 449, 449, 449, 449, 574, 574, 574,
+ 574, 574, 574, 574, 574, 574, 574, 574, 575, 575,
+ 575, 575, 575, 575, 575, 575, 575, 575, 575, 576,
576, 576, 576, 576, 576, 576, 576, 576, 576, 576,
-
- 576, 577, 577, 577, 577, 577, 577, 577, 577, 577,
- 577, 577, 578, 578, 578, 578, 578, 578, 578, 578,
- 579, 579, 579, 579, 579, 579, 579, 579, 580, 580,
- 581, 581, 581, 581, 581, 581, 581, 0, 581, 581,
- 581, 582, 0, 582, 582, 582, 582, 582, 582, 582,
- 582, 582, 583, 0, 583, 583, 583, 583, 583, 583,
- 583, 583, 583, 584, 584, 584, 584, 584, 584, 584,
- 584, 584, 584, 584, 585, 0, 585, 585, 585, 585,
- 585, 585, 585, 585, 585, 586, 0, 586, 586, 586,
- 586, 586, 586, 586, 586, 586, 587, 587, 587, 587,
-
- 587, 587, 587, 587, 588, 0, 588, 588, 588, 588,
- 588, 588, 588, 588, 588, 589, 0, 589, 589, 589,
- 589, 589, 589, 589, 589, 589, 590, 590, 590, 590,
- 590, 590, 590, 590, 0, 0, 590, 591, 0, 591,
- 591, 591, 591, 591, 591, 591, 591, 591, 592, 0,
- 592, 592, 592, 592, 592, 592, 592, 592, 592, 593,
- 593, 593, 593, 593, 593, 593, 593, 0, 0, 593,
- 594, 0, 594, 594, 594, 594, 594, 594, 594, 594,
- 594, 595, 0, 595, 595, 595, 595, 595, 595, 595,
- 595, 595, 596, 596, 596, 596, 596, 596, 596, 596,
-
- 596, 596, 596, 597, 597, 597, 597, 597, 597, 597,
- 597, 598, 0, 598, 598, 598, 598, 598, 598, 598,
- 598, 598, 599, 0, 599, 599, 599, 599, 599, 599,
- 599, 599, 599, 601, 0, 601, 601, 601, 601, 601,
- 601, 601, 601, 601, 602, 0, 602, 602, 602, 602,
- 602, 602, 602, 602, 602, 603, 0, 603, 603, 603,
- 603, 603, 603, 603, 603, 603, 604, 604, 604, 604,
- 604, 604, 604, 604, 605, 605, 606, 0, 606, 606,
- 606, 606, 606, 606, 606, 606, 606, 607, 0, 607,
- 607, 607, 607, 607, 607, 607, 607, 607, 608, 608,
-
- 608, 608, 608, 608, 608, 608, 608, 608, 608, 609,
- 0, 609, 609, 609, 609, 609, 609, 609, 609, 609,
- 610, 0, 610, 610, 610, 610, 610, 610, 610, 610,
- 610, 611, 611, 0, 611, 611, 611, 611, 611, 611,
- 611, 611, 612, 612, 612, 0, 612, 612, 612, 612,
- 612, 612, 612, 613, 613, 613, 613, 613, 613, 613,
- 613, 613, 613, 613, 614, 614, 614, 614, 614, 614,
- 614, 614, 614, 614, 614, 615, 615, 615, 615, 615,
- 615, 615, 615, 615, 615, 615, 616, 616, 616, 616,
- 616, 616, 616, 616, 616, 616, 616, 617, 0, 617,
-
- 617, 617, 617, 617, 617, 617, 617, 617, 618, 618,
- 618, 618, 618, 618, 618, 618, 619, 0, 619, 619,
- 619, 619, 619, 619, 619, 619, 619, 620, 0, 620,
- 620, 620, 620, 620, 620, 620, 620, 620, 621, 0,
- 621, 621, 621, 621, 621, 621, 621, 621, 621, 622,
- 622, 622, 622, 622, 622, 622, 622, 0, 0, 622,
- 623, 0, 623, 623, 623, 623, 623, 623, 623, 623,
- 623, 624, 0, 624, 624, 624, 624, 624, 624, 624,
- 624, 624, 625, 0, 625, 625, 625, 625, 625, 625,
- 625, 625, 625, 626, 626, 626, 626, 626, 626, 626,
-
- 626, 0, 0, 626, 627, 0, 627, 627, 627, 627,
- 627, 627, 627, 627, 627, 628, 0, 628, 628, 628,
- 628, 628, 628, 628, 628, 628, 629, 0, 629, 629,
- 629, 629, 629, 629, 629, 629, 629, 630, 0, 630,
- 630, 630, 630, 630, 630, 630, 630, 630, 631, 0,
- 631, 631, 631, 631, 631, 631, 631, 631, 631, 633,
- 0, 633, 633, 633, 633, 633, 633, 633, 633, 633,
- 634, 0, 634, 634, 634, 634, 634, 634, 634, 634,
- 634, 635, 635, 635, 635, 635, 635, 635, 635, 635,
- 635, 635, 636, 0, 636, 636, 636, 636, 636, 636,
-
- 636, 636, 636, 637, 0, 637, 637, 637, 637, 637,
- 637, 637, 637, 637, 638, 0, 638, 638, 638, 638,
- 638, 638, 638, 638, 638, 639, 0, 639, 639, 639,
- 639, 639, 639, 639, 639, 639, 640, 640, 640, 640,
- 640, 640, 640, 640, 640, 0, 640, 641, 641, 641,
- 641, 641, 641, 641, 641, 641, 641, 641, 642, 642,
- 642, 642, 642, 642, 642, 642, 642, 642, 642, 643,
- 643, 643, 643, 643, 643, 643, 643, 643, 643, 643,
+ 577, 577, 577, 577, 577, 577, 577, 577, 577, 577,
+
+ 577, 578, 578, 578, 578, 578, 578, 578, 578, 578,
+ 578, 578, 579, 579, 579, 579, 579, 579, 579, 579,
+ 580, 580, 580, 580, 580, 580, 580, 580, 581, 581,
+ 582, 582, 582, 582, 582, 582, 582, 0, 582, 582,
+ 582, 583, 0, 583, 583, 583, 583, 583, 583, 583,
+ 583, 583, 584, 0, 584, 584, 584, 584, 584, 584,
+ 584, 584, 584, 585, 585, 585, 585, 585, 585, 585,
+ 585, 585, 585, 585, 586, 0, 586, 586, 586, 586,
+ 586, 586, 586, 586, 586, 587, 0, 587, 587, 587,
+ 587, 587, 587, 587, 587, 587, 588, 588, 588, 588,
+
+ 588, 588, 588, 588, 589, 0, 589, 589, 589, 589,
+ 589, 589, 589, 589, 589, 590, 0, 590, 590, 590,
+ 590, 590, 590, 590, 590, 590, 591, 591, 591, 591,
+ 591, 591, 591, 591, 0, 0, 591, 592, 0, 592,
+ 592, 592, 592, 592, 592, 592, 592, 592, 593, 0,
+ 593, 593, 593, 593, 593, 593, 593, 593, 593, 594,
+ 594, 594, 594, 594, 594, 594, 594, 0, 0, 594,
+ 595, 0, 595, 595, 595, 595, 595, 595, 595, 595,
+ 595, 596, 0, 596, 596, 596, 596, 596, 596, 596,
+ 596, 596, 597, 597, 597, 597, 597, 597, 597, 597,
+
+ 597, 597, 597, 598, 598, 598, 598, 598, 598, 598,
+ 598, 599, 0, 599, 599, 599, 599, 599, 599, 599,
+ 599, 599, 600, 0, 600, 600, 600, 600, 600, 600,
+ 600, 600, 600, 602, 0, 602, 602, 602, 602, 602,
+ 602, 602, 602, 602, 603, 0, 603, 603, 603, 603,
+ 603, 603, 603, 603, 603, 604, 0, 604, 604, 604,
+ 604, 604, 604, 604, 604, 604, 605, 605, 605, 605,
+ 605, 605, 605, 605, 606, 606, 607, 0, 607, 607,
+ 607, 607, 607, 607, 607, 607, 607, 608, 0, 608,
+ 608, 608, 608, 608, 608, 608, 608, 608, 609, 609,
+
+ 609, 609, 609, 609, 609, 609, 609, 609, 609, 610,
+ 0, 610, 610, 610, 610, 610, 610, 610, 610, 610,
+ 611, 0, 611, 611, 611, 611, 611, 611, 611, 611,
+ 611, 612, 612, 0, 612, 612, 612, 612, 612, 612,
+ 612, 612, 613, 613, 613, 0, 613, 613, 613, 613,
+ 613, 613, 613, 614, 614, 614, 614, 614, 614, 614,
+ 614, 614, 614, 614, 615, 615, 615, 615, 615, 615,
+ 615, 615, 615, 615, 615, 616, 616, 616, 616, 616,
+ 616, 616, 616, 616, 616, 616, 617, 617, 617, 617,
+ 617, 617, 617, 617, 617, 617, 617, 618, 0, 618,
+
+ 618, 618, 618, 618, 618, 618, 618, 618, 619, 619,
+ 619, 619, 619, 619, 619, 619, 620, 0, 620, 620,
+ 620, 620, 620, 620, 620, 620, 620, 621, 0, 621,
+ 621, 621, 621, 621, 621, 621, 621, 621, 622, 0,
+ 622, 622, 622, 622, 622, 622, 622, 622, 622, 623,
+ 623, 623, 623, 623, 623, 623, 623, 0, 0, 623,
+ 624, 0, 624, 624, 624, 624, 624, 624, 624, 624,
+ 624, 625, 0, 625, 625, 625, 625, 625, 625, 625,
+ 625, 625, 626, 0, 626, 626, 626, 626, 626, 626,
+ 626, 626, 626, 627, 627, 627, 627, 627, 627, 627,
+
+ 627, 0, 0, 627, 628, 0, 628, 628, 628, 628,
+ 628, 628, 628, 628, 628, 629, 0, 629, 629, 629,
+ 629, 629, 629, 629, 629, 629, 630, 0, 630, 630,
+ 630, 630, 630, 630, 630, 630, 630, 631, 0, 631,
+ 631, 631, 631, 631, 631, 631, 631, 631, 632, 0,
+ 632, 632, 632, 632, 632, 632, 632, 632, 632, 634,
+ 0, 634, 634, 634, 634, 634, 634, 634, 634, 634,
+ 635, 0, 635, 635, 635, 635, 635, 635, 635, 635,
+ 635, 636, 636, 636, 636, 636, 636, 636, 636, 636,
+ 636, 636, 637, 0, 637, 637, 637, 637, 637, 637,
+
+ 637, 637, 637, 638, 0, 638, 638, 638, 638, 638,
+ 638, 638, 638, 638, 639, 0, 639, 639, 639, 639,
+ 639, 639, 639, 639, 639, 640, 0, 640, 640, 640,
+ 640, 640, 640, 640, 640, 640, 641, 641, 641, 641,
+ 641, 641, 641, 641, 641, 0, 641, 642, 642, 642,
+ 642, 642, 642, 642, 642, 642, 642, 642, 643, 643,
+ 643, 643, 643, 643, 643, 643, 643, 643, 643, 644,
644, 644, 644, 644, 644, 644, 644, 644, 644, 644,
- 644, 645, 645, 645, 645, 645, 645, 645, 645, 645,
-
- 645, 645, 646, 646, 646, 646, 646, 646, 646, 646,
- 646, 646, 646, 647, 647, 647, 647, 647, 647, 647,
- 647, 647, 647, 647, 648, 648, 648, 648, 648, 648,
- 648, 648, 648, 648, 648, 649, 649, 649, 649, 649,
- 649, 649, 649, 649, 649, 649, 650, 0, 650, 650,
- 650, 650, 650, 650, 650, 650, 650, 651, 651, 651,
- 651, 651, 651, 651, 651, 651, 651, 651, 652, 652,
- 652, 652, 652, 652, 652, 652, 652, 652, 652, 653,
- 653, 653, 653, 653, 653, 653, 653, 653, 653, 653,
- 654, 0, 654, 654, 654, 654, 654, 654, 654, 654,
-
- 654, 655, 0, 655, 655, 655, 655, 655, 655, 655,
- 655, 655, 656, 0, 656, 656, 656, 656, 656, 656,
- 656, 656, 656, 657, 0, 657, 657, 657, 657, 657,
- 657, 657, 657, 657, 658, 0, 658, 658, 658, 658,
- 658, 658, 658, 658, 658, 659, 0, 659, 659, 659,
- 659, 659, 659, 659, 659, 659, 660, 0, 660, 660,
- 660, 660, 660, 660, 660, 660, 660, 661, 661, 661,
- 661, 661, 661, 0, 661, 0, 0, 661, 662, 662,
- 662, 662, 662, 662, 0, 662, 0, 0, 662, 663,
- 663, 0, 663, 663, 663, 663, 663, 663, 663, 663,
-
- 664, 664, 664, 664, 664, 664, 664, 664, 664, 664,
- 664, 665, 665, 665, 665, 665, 665, 665, 665, 665,
- 665, 665, 666, 666, 666, 666, 666, 666, 666, 666,
- 666, 666, 666, 667, 667, 667, 667, 667, 667, 667,
- 667, 667, 667, 667, 668, 668, 668, 668, 668, 668,
- 668, 668, 668, 668, 668, 669, 669, 669, 669, 669,
- 669, 669, 669, 669, 669, 669, 670, 670, 670, 670,
- 670, 670, 670, 670, 670, 670, 670, 671, 671, 671,
- 671, 671, 671, 671, 671, 671, 671, 671, 672, 672,
- 672, 672, 672, 672, 672, 672, 672, 672, 672, 673,
-
- 673, 673, 673, 673, 673, 673, 673, 673, 673, 673,
+ 645, 645, 645, 645, 645, 645, 645, 645, 645, 645,
+ 645, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+
+ 646, 646, 647, 647, 647, 647, 647, 647, 647, 647,
+ 647, 647, 647, 648, 648, 648, 648, 648, 648, 648,
+ 648, 648, 648, 648, 649, 649, 649, 649, 649, 649,
+ 649, 649, 649, 649, 649, 650, 650, 650, 650, 650,
+ 650, 650, 650, 650, 650, 650, 651, 0, 651, 651,
+ 651, 651, 651, 651, 651, 651, 651, 652, 652, 652,
+ 652, 652, 652, 652, 652, 652, 652, 652, 653, 653,
+ 653, 653, 653, 653, 653, 653, 653, 653, 653, 654,
+ 654, 654, 654, 654, 654, 654, 654, 654, 654, 654,
+ 655, 0, 655, 655, 655, 655, 655, 655, 655, 655,
+
+ 655, 656, 0, 656, 656, 656, 656, 656, 656, 656,
+ 656, 656, 657, 0, 657, 657, 657, 657, 657, 657,
+ 657, 657, 657, 658, 0, 658, 658, 658, 658, 658,
+ 658, 658, 658, 658, 659, 0, 659, 659, 659, 659,
+ 659, 659, 659, 659, 659, 660, 0, 660, 660, 660,
+ 660, 660, 660, 660, 660, 660, 661, 0, 661, 661,
+ 661, 661, 661, 661, 661, 661, 661, 662, 662, 662,
+ 662, 662, 662, 0, 662, 0, 0, 662, 663, 663,
+ 663, 663, 663, 663, 0, 663, 0, 0, 663, 664,
+ 664, 0, 664, 664, 664, 664, 664, 664, 664, 664,
+
+ 665, 665, 665, 665, 665, 665, 665, 665, 665, 665,
+ 665, 666, 666, 666, 666, 666, 666, 666, 666, 666,
+ 666, 666, 667, 667, 667, 667, 667, 667, 667, 667,
+ 667, 667, 667, 668, 668, 668, 668, 668, 668, 668,
+ 668, 668, 668, 668, 669, 669, 669, 669, 669, 669,
+ 669, 669, 669, 669, 669, 670, 670, 670, 670, 670,
+ 670, 670, 670, 670, 670, 670, 671, 671, 671, 671,
+ 671, 671, 671, 671, 671, 671, 671, 672, 672, 672,
+ 672, 672, 672, 672, 672, 672, 672, 672, 673, 673,
+ 673, 673, 673, 673, 673, 673, 673, 673, 673, 674,
+
674, 674, 674, 674, 674, 674, 674, 674, 674, 674,
- 674, 675, 675, 675, 675, 675, 675, 675, 675, 675,
- 675, 675, 676, 676, 676, 676, 676, 676, 676, 676,
- 676, 676, 676, 677, 677, 677, 677, 677, 677, 677,
- 677, 677, 677, 677, 678, 678, 678, 678, 678, 678,
- 678, 678, 678, 678, 678, 679, 679, 679, 679, 679,
- 679, 679, 679, 679, 679, 679, 680, 680, 680, 680,
- 680, 680, 680, 680, 680, 680, 680, 681, 0, 681,
- 681, 681, 681, 681, 681, 681, 681, 681, 682, 682,
-
- 682, 682, 682, 682, 682, 682, 0, 0, 682, 683,
- 0, 683, 683, 683, 683, 683, 683, 683, 683, 683,
- 684, 0, 684, 684, 684, 684, 684, 684, 684, 684,
- 684, 685, 685, 685, 685, 685, 685, 685, 685, 0,
- 0, 685, 686, 0, 686, 686, 686, 686, 686, 686,
- 686, 686, 686, 687, 0, 687, 687, 687, 687, 687,
- 687, 687, 687, 687, 688, 688, 688, 688, 688, 688,
- 688, 688, 688, 688, 688, 689, 689, 0, 689, 689,
- 689, 689, 689, 689, 689, 689, 690, 690, 690, 690,
- 690, 690, 690, 690, 690, 690, 690, 691, 691, 691,
-
- 691, 691, 691, 691, 691, 691, 691, 691, 692, 692,
- 692, 692, 692, 692, 692, 692, 692, 692, 692, 693,
- 693, 693, 693, 693, 693, 693, 693, 693, 693, 693,
+ 675, 675, 675, 675, 675, 675, 675, 675, 675, 675,
+ 675, 676, 676, 676, 676, 676, 676, 676, 676, 676,
+ 676, 676, 677, 677, 677, 677, 677, 677, 677, 677,
+ 677, 677, 677, 678, 678, 678, 678, 678, 678, 678,
+ 678, 678, 678, 678, 679, 679, 679, 679, 679, 679,
+ 679, 679, 679, 679, 679, 680, 680, 680, 680, 680,
+ 680, 680, 680, 680, 680, 680, 681, 681, 681, 681,
+ 681, 681, 681, 681, 681, 681, 681, 682, 0, 682,
+ 682, 682, 682, 682, 682, 682, 682, 682, 683, 683,
+
+ 683, 683, 683, 683, 683, 683, 0, 0, 683, 684,
+ 0, 684, 684, 684, 684, 684, 684, 684, 684, 684,
+ 685, 0, 685, 685, 685, 685, 685, 685, 685, 685,
+ 685, 686, 686, 686, 686, 686, 686, 686, 686, 0,
+ 0, 686, 687, 0, 687, 687, 687, 687, 687, 687,
+ 687, 687, 687, 688, 0, 688, 688, 688, 688, 688,
+ 688, 688, 688, 688, 689, 689, 689, 689, 689, 689,
+ 689, 689, 689, 689, 689, 690, 690, 0, 690, 690,
+ 690, 690, 690, 690, 690, 690, 691, 691, 691, 691,
+ 691, 691, 691, 691, 691, 691, 691, 692, 692, 692,
+
+ 692, 692, 692, 692, 692, 692, 692, 692, 693, 693,
+ 693, 693, 693, 693, 693, 693, 693, 693, 693, 694,
694, 694, 694, 694, 694, 694, 694, 694, 694, 694,
- 694, 695, 695, 695, 695, 695, 695, 695, 695, 695,
- 695, 695, 696, 696, 696, 696, 696, 696, 696, 696,
- 696, 696, 696, 697, 697, 697, 697, 697, 697, 697,
- 697, 697, 697, 697, 698, 698, 698, 698, 698, 698,
- 698, 698, 698, 698, 698, 699, 699, 699, 699, 699,
- 699, 699, 699, 699, 699, 699, 700, 700, 700, 700,
-
- 700, 700, 700, 700, 700, 700, 700, 701, 701, 701,
- 701, 701, 701, 701, 701, 701, 701, 701, 702, 702,
- 702, 702, 702, 702, 702, 702, 702, 702, 702, 703,
- 703, 703, 703, 703, 703, 703, 703, 703, 703, 703,
+ 695, 695, 695, 695, 695, 695, 695, 695, 695, 695,
+ 695, 696, 696, 696, 696, 696, 696, 696, 696, 696,
+ 696, 696, 697, 697, 697, 697, 697, 697, 697, 697,
+ 697, 697, 697, 698, 698, 698, 698, 698, 698, 698,
+ 698, 698, 698, 698, 699, 699, 699, 699, 699, 699,
+ 699, 699, 699, 699, 699, 700, 700, 700, 700, 700,
+ 700, 700, 700, 700, 700, 700, 701, 701, 701, 701,
+
+ 701, 701, 701, 701, 701, 701, 701, 702, 702, 702,
+ 702, 702, 702, 702, 702, 702, 702, 702, 703, 703,
+ 703, 703, 703, 703, 703, 703, 703, 703, 703, 704,
704, 704, 704, 704, 704, 704, 704, 704, 704, 704,
- 704, 705, 705, 705, 705, 705, 705, 705, 705, 705,
- 705, 705, 706, 706, 706, 706, 706, 706, 706, 706,
- 706, 706, 706, 707, 0, 707, 707, 707, 707, 707,
- 707, 707, 707, 707, 708, 0, 708, 708, 708, 708,
- 708, 708, 708, 708, 708, 709, 0, 709, 709, 709,
-
- 709, 709, 709, 709, 709, 709, 710, 0, 710, 710,
- 710, 710, 710, 710, 710, 710, 710, 711, 711, 0,
- 711, 711, 711, 711, 711, 711, 711, 711, 712, 712,
- 712, 712, 712, 712, 712, 712, 712, 712, 712, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
-
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572, 572, 572, 572, 572, 572,
- 572, 572, 572, 572, 572
+ 705, 705, 705, 705, 705, 705, 705, 705, 705, 705,
+ 705, 706, 706, 706, 706, 706, 706, 706, 706, 706,
+ 706, 706, 707, 707, 707, 707, 707, 707, 707, 707,
+ 707, 707, 707, 708, 0, 708, 708, 708, 708, 708,
+ 708, 708, 708, 708, 709, 0, 709, 709, 709, 709,
+ 709, 709, 709, 709, 709, 710, 0, 710, 710, 710,
+
+ 710, 710, 710, 710, 710, 710, 711, 0, 711, 711,
+ 711, 711, 711, 711, 711, 711, 711, 712, 712, 0,
+ 712, 712, 712, 712, 712, 712, 712, 712, 713, 713,
+ 713, 713, 713, 713, 713, 713, 713, 713, 713, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573, 573, 573, 573, 573, 573,
+ 573, 573, 573, 573, 573
} ;
/* Table of booleans, true if rule could match eol. */
-static yyconst flex_int32_t yy_rule_can_match_eol[129] =
+static yyconst flex_int32_t yy_rule_can_match_eol[130] =
{ 0,
1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -1672,7 +1672,7 @@ static yyconst flex_int32_t yy_rule_can_match_eol[129] =
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 1, 1, 0, 0, 0, };
+ 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, };
/* The intent behind this definition is that it'll catch
* any uses of REJECT which flex missed.
@@ -1682,18 +1682,18 @@ static yyconst flex_int32_t yy_rule_can_match_eol[129] =
#define YY_MORE_ADJ 0
#define YY_RESTORE_YY_MORE_OFFSET
#line 1 "regex.l"
-/* Foma: a finite-state toolkit and library. */
-/* Copyright © 2008-2014 Mans Hulden */
-/* This file is part of foma. */
-/* Foma is free software: you can redistribute it and/or modify */
-/* it under the terms of the GNU General Public License version 2 as */
-/* published by the Free Software Foundation. */
-/* Foma is distributed in the hope that it will be useful, */
-/* but WITHOUT ANY WARRANTY; without even the implied warranty of */
-/* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the */
-/* GNU General Public License for more details. */
-/* You should have received a copy of the GNU General Public License */
-/* along with foma. If not, see <http://www.gnu.org/licenses/>. */
+/* Foma: a finite-state toolkit and library. */
+/* Copyright © 2008-2015 Mans Hulden */
+/* This file is part of foma. */
+/* Licensed under the Apache License, Version 2.0 (the "License"); */
+/* you may not use this file except in compliance with the License. */
+/* You may obtain a copy of the License at */
+/* http://www.apache.org/licenses/LICENSE-2.0 */
+/* Unless required by applicable law or agreed to in writing, software */
+/* distributed under the License is distributed on an "AS IS" BASIS, */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */
+/* See the License for the specific language governing permissions and */
+/* limitations under the License. */
#define YY_NO_INPUT 1
#line 23 "regex.l"
#include <stdio.h>
@@ -2136,7 +2136,7 @@ yy_match:
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 573 )
+ if ( yy_current_state >= 574 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
@@ -2829,166 +2829,166 @@ YY_RULE_SETUP
case 94:
YY_RULE_SETUP
#line 426 "regex.l"
-{ return HIGH_CROSS_PRODUCT; }
+{ yylval_param->type = OP_TWO_LEVEL_REPLACE; return DIRECTION; }
YY_BREAK
case 95:
YY_RULE_SETUP
#line 427 "regex.l"
-{ return CROSS_PRODUCT; }
+{ return HIGH_CROSS_PRODUCT; }
YY_BREAK
case 96:
YY_RULE_SETUP
#line 428 "regex.l"
-{ return COMMA; }
+{ return CROSS_PRODUCT; }
YY_BREAK
case 97:
YY_RULE_SETUP
#line 429 "regex.l"
-{ return DOUBLE_COMMA; }
+{ return COMMA; }
YY_BREAK
case 98:
YY_RULE_SETUP
#line 430 "regex.l"
-{ return IGNORE_INTERNAL; }
+{ return DOUBLE_COMMA; }
YY_BREAK
case 99:
YY_RULE_SETUP
#line 431 "regex.l"
-{ return IGNORE_ALL; }
+{ return IGNORE_INTERNAL; }
YY_BREAK
case 100:
YY_RULE_SETUP
#line 432 "regex.l"
-{ return RIGHT_QUOTIENT; }
+{ return IGNORE_ALL; }
YY_BREAK
case 101:
YY_RULE_SETUP
#line 433 "regex.l"
-{ return LEFT_QUOTIENT; }
+{ return RIGHT_QUOTIENT; }
YY_BREAK
case 102:
YY_RULE_SETUP
#line 434 "regex.l"
-{ return INTERLEAVE_QUOTIENT; }
+{ return LEFT_QUOTIENT; }
YY_BREAK
case 103:
YY_RULE_SETUP
#line 435 "regex.l"
-{ return TERM_NEGATION; }
+{ return INTERLEAVE_QUOTIENT; }
YY_BREAK
case 104:
YY_RULE_SETUP
#line 436 "regex.l"
-{ return MINUS; }
+{ return TERM_NEGATION; }
YY_BREAK
case 105:
YY_RULE_SETUP
#line 437 "regex.l"
-{ return CONTAINS_OPT_ONE; }
+{ return MINUS; }
YY_BREAK
case 106:
YY_RULE_SETUP
#line 438 "regex.l"
-{ return CONTAINS_ONE; }
+{ return CONTAINS_OPT_ONE; }
YY_BREAK
case 107:
YY_RULE_SETUP
#line 439 "regex.l"
-{ return CONTAINS; }
+{ return CONTAINS_ONE; }
YY_BREAK
case 108:
YY_RULE_SETUP
#line 440 "regex.l"
-{ return KLEENE_PLUS; }
+{ return CONTAINS; }
YY_BREAK
case 109:
YY_RULE_SETUP
#line 441 "regex.l"
-{ return KLEENE_STAR; }
+{ return KLEENE_PLUS; }
YY_BREAK
case 110:
YY_RULE_SETUP
#line 442 "regex.l"
-{ return INVERSE; }
+{ return KLEENE_STAR; }
YY_BREAK
case 111:
YY_RULE_SETUP
#line 443 "regex.l"
-{ return REVERSE; }
+{ return INVERSE; }
YY_BREAK
case 112:
YY_RULE_SETUP
#line 444 "regex.l"
-{ return LBRACKET; }
+{ return REVERSE; }
YY_BREAK
case 113:
YY_RULE_SETUP
#line 445 "regex.l"
-{ return RBRACKET; }
+{ return LBRACKET; }
YY_BREAK
case 114:
YY_RULE_SETUP
#line 446 "regex.l"
-{ return PRECEDES; }
+{ return RBRACKET; }
YY_BREAK
case 115:
YY_RULE_SETUP
#line 447 "regex.l"
-{ return FOLLOWS; }
+{ return PRECEDES; }
YY_BREAK
case 116:
YY_RULE_SETUP
#line 448 "regex.l"
-{ return NEQ; }
+{ return FOLLOWS; }
YY_BREAK
case 117:
YY_RULE_SETUP
#line 449 "regex.l"
-{ return LPAREN; }
+{ return NEQ; }
YY_BREAK
case 118:
YY_RULE_SETUP
#line 450 "regex.l"
-{ return RPAREN; }
+{ return LPAREN; }
YY_BREAK
case 119:
YY_RULE_SETUP
#line 451 "regex.l"
-{ return ENDM; }
+{ return RPAREN; }
YY_BREAK
case 120:
YY_RULE_SETUP
#line 452 "regex.l"
-{ return ENDD; }
+{ return ENDM; }
YY_BREAK
case 121:
YY_RULE_SETUP
#line 453 "regex.l"
-{ return END; }
+{ return ENDD; }
YY_BREAK
case 122:
YY_RULE_SETUP
#line 454 "regex.l"
-{ return UNION; }
+{ return END; }
YY_BREAK
case 123:
YY_RULE_SETUP
#line 455 "regex.l"
-{ return INTERSECT; }
+{ return UNION; }
YY_BREAK
case 124:
-/* rule 124 can match eol */
YY_RULE_SETUP
-#line 457 "regex.l"
-{ yycolumn = 1; }
+#line 456 "regex.l"
+{ return INTERSECT; }
YY_BREAK
case 125:
/* rule 125 can match eol */
YY_RULE_SETUP
#line 458 "regex.l"
-{ }
+{ yycolumn = 1; }
YY_BREAK
case 126:
+/* rule 126 can match eol */
YY_RULE_SETUP
#line 459 "regex.l"
{ }
@@ -2996,14 +2996,19 @@ YY_RULE_SETUP
case 127:
YY_RULE_SETUP
#line 460 "regex.l"
-{ }
+{ }
YY_BREAK
case 128:
YY_RULE_SETUP
#line 461 "regex.l"
+{ }
+ YY_BREAK
+case 129:
+YY_RULE_SETUP
+#line 462 "regex.l"
ECHO;
YY_BREAK
-#line 3007 "lex.yy.c"
+#line 3012 "lex.yy.c"
case YY_STATE_EOF(INITIAL):
case YY_STATE_EOF(DEFI):
case YY_STATE_EOF(QTD):
@@ -3303,7 +3308,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 573 )
+ if ( yy_current_state >= 574 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
@@ -3332,11 +3337,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 573 )
+ if ( yy_current_state >= 574 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
- yy_is_jam = (yy_current_state == 572);
+ yy_is_jam = (yy_current_state == 573);
return yy_is_jam ? 0 : yy_current_state;
}
@@ -4199,4 +4204,4 @@ void yyfree (void * ptr , yyscan_t yyscanner)
#define YYTABLES_NAME "yytables"
-#line 461 "regex.l"
+#line 462 "regex.l"
diff --git a/back-ends/foma/lexc.l b/back-ends/foma/lexc.l
index b844c0e..e326246 100644
--- a/back-ends/foma/lexc.l
+++ b/back-ends/foma/lexc.l
@@ -52,8 +52,8 @@ struct fsm *fsm_lexc_parse_string(char *string, int verbose) {
if (lexclex() != 1) {
if (lexentries != -1) {
printf("%i\n",lexentries);
- }
- }
+ }
+ }
lexc_delete_buffer(my_string_buffer);
g_defines = olddefines;
return(lexc_to_fsm());
@@ -69,13 +69,13 @@ void lexc_trim(char *s) {
/* Remove trailing ; and = and space and initial space */
int i,j;
for (i = strlen(s)-1; *(s+i) == ';' || *(s+i) == '=' || *(s+i) == ' ' || *(s+i) == '\t'; i--)
- *(s+i) = '\0';
+ *(s+i) = '\0';
for (i=0; *(s+i) == ' ' || *(s+i) == '\t' || *(s+i) == '\n'; i++) {
}
for (j=0; *(s+i) != '\0'; i++, j++) {
*(s+j) = *(s+i);
}
- *(s+j) = *(s+i);
+ *(s+j) = *(s+i);
}
%}
@@ -126,7 +126,7 @@ ANY [\001-\177]|[\300-\337][\200-\277]|[\340-\357][\200-\277][\200-\277]|[\36
if (lexentries != -1) {
printf("%i, ",lexentries);
}
- printf("%s...",lexctext+8);
+ printf("%s...",lexctext+8);
fflush(stdout);
lexentries = 0;
lexc_set_current_lexicon(lexctext+8, SOURCE_LEXICON);
@@ -180,7 +180,7 @@ ANY [\001-\177]|[\300-\337][\200-\277]|[\340-\357][\200-\277][\200-\277]|[\36
*(lexctext+lexcleng-1) = ';';
if (my_yyparse(lexctext, lexclineno, g_defines, NULL) == 0) {
lexc_set_network(current_parse);
- }
+ }
BEGIN(LEXENTRIES);
}
diff --git a/back-ends/foma/lexcread.c b/back-ends/foma/lexcread.c
index 3535dc4..2872625 100644
--- a/back-ends/foma/lexcread.c
+++ b/back-ends/foma/lexcread.c
@@ -34,12 +34,12 @@ extern int g_lexc_align;
struct multichar_symbols {
char *symbol;
- short int sigma_number;
+ short sigma_number;
struct multichar_symbols *next;
};
struct lexstates { /* Separate list of LEXICON states */
- char *name;
+ char *name;
struct states *state;
struct lexstates *next;
unsigned char targeted;
@@ -48,8 +48,8 @@ struct lexstates { /* Separate list of LEXICON states */
struct states {
struct trans {
- short int in;
- short int out;
+ short in;
+ short out;
struct states *target;
struct trans *next;
} *trans;
@@ -58,7 +58,7 @@ struct states {
unsigned int hashval; /* Hash for remaining symbols until next lexstate */
unsigned char mergeable; /* Can this state be merged with other suffix */
/* 0 = NO, 1 = YES, 2 = DELETED/MERGED */
- unsigned short int distance; /* Number of remaining symbols until lexstate */
+ unsigned short distance; /* Number of remaining symbols until lexstate */
struct states *merge_with;
};
@@ -85,7 +85,7 @@ static struct lexc_hashtable *hashtable;
static struct fsm *current_regex_network;
static int cwordin[1000], cwordout[1000], medcwordin[2000], medcwordout[2000], carity, lexc_statecount, maxlen, hasfinal, current_entry, net_has_unknown;
-static _Bool *mchash;
+static Boolean *mchash;
static struct lexstates *clexicon, *ctarget;
static char *mystrncpy(char *dest, char *src, int len);
@@ -173,7 +173,7 @@ void lexc_init() {
maxlen = 0;
- mchash = xxcalloc(256*256, sizeof(_Bool));
+ mchash = xxcalloc(256*256, sizeof(Boolean));
for (i=0; i< SIGMA_HASH_TABLESIZE; i++) {
(hashtable+i)->symbol = NULL;
(hashtable+i)->sigma_number = -1;
@@ -188,7 +188,7 @@ void lexc_clear_current_word() {
}
void lexc_add_state(struct states *s) {
- struct statelist *sl;
+ struct statelist *sl;
sl = xxmalloc(sizeof(struct statelist));
sl->state = s;
s->number = -1;
@@ -222,7 +222,7 @@ void lexc_update_unknowns(int sigma_number) {
t->next = newtrans;
}
}
- }
+ }
}
void lexc_add_network() {
@@ -391,7 +391,7 @@ void lexc_set_current_lexicon(char *name, int which) {
if (which == 0) {
clexicon = l;
l->has_outgoing = 1;
- } else {
+ } else {
ctarget = l;
}
}
@@ -585,7 +585,7 @@ void lexc_pad() {
break;
}
if (*(cwordin+i) == -1 && *(cwordout+i) != -1) {
- pad = 1; /* Pad upper */
+ pad = 1; /* Pad upper */
}
else if (*(cwordin+i) != -1 && *(cwordout+i) == -1) {
pad = 2; /* Pad lower */
@@ -685,7 +685,7 @@ void lexc_add_mc(char *symbol) {
mcprev->next = mcnew;
s = sigma_add(symbol, lexsigma);
- mchashval = (unsigned int) ((unsigned char) *(symbol)) * 256 + (unsigned int) ((unsigned char) *(symbol+1));
+ mchashval = (unsigned int) ((unsigned char) *(symbol)) * 256 + (unsigned int) ((unsigned char) *(symbol+1));
lexc_add_sigma_hash(symbol, s);
*(mchash+mchashval) = 1;
mcnew->sigma_number = s;
@@ -795,10 +795,10 @@ void lexc_number_states() {
}
/* If there is no Root lexicon, the first lexicon mentioned is Root */
if (!hasroot) {
- for (s = statelist; s != NULL; s = s->next) {
+ for (s = statelist; s != NULL; s = s->next) {
if (s->next == NULL) {
s->state->number = 0;
-#ifdef ORIGINAL
+#ifdef ORIGINAL
fprintf(stderr,"*Warning: no Root lexicon, using '%s' as Root.\n",s->state->lexstate->name);
#else
if (verbose_lexc_ == 1)
@@ -824,7 +824,7 @@ void lexc_number_states() {
}
}
- for (s = statelist; s != NULL; s = s->next) {
+ for (s = statelist; s != NULL; s = s->next) {
if (s->state->number == -1) {
s->state->number = n;
n++;
@@ -837,11 +837,11 @@ void lexc_number_states() {
fprintf(stderr,"*Warning: lexicon '%s' defined but not used\n",l->name);
fflush(stdout);
#else
- if (verbose_lexc_)
+ if (verbose_lexc_)
{
fprintf(stderr,"*Warning: lexicon '%s' defined but not used\n",l->name);
fflush(stderr);
- }
+ }
#endif
}
if (l->has_outgoing == 0 && strcmp(l->name, "#") != 0) {
@@ -899,7 +899,7 @@ void lexc_merge_states() {
/* Find a suitable prime for hashing: proportional to the size of the */
/* number of mergeable states */
- for (i = 0; primes[i] < numstates/4; i++) { }
+ for (i = 0; primes[i] < numstates/4; i++) { }
tablesize = primes[i];
hashstates = xxcalloc(tablesize,sizeof(struct hashstates));
@@ -914,7 +914,7 @@ void lexc_merge_states() {
newl->state = s->state;
newl->next = currentl->next;
currentl->next = newl;
- }
+ }
s->state->hashval = s->state->hashval % tablesize;
currenth = hashstates+s->state->hashval;
if (currenth->state == NULL) {
@@ -923,7 +923,7 @@ void lexc_merge_states() {
newh = xxcalloc(1,sizeof(struct hashstates));
newh->state = s->state;
newh->next = currenth->next;
- currenth->next = newh;
+ currenth->next = newh;
}
}
}
@@ -968,7 +968,7 @@ void lexc_merge_states() {
for (s = statelist, sprev = NULL; s != NULL; ) {
if (s->state->mergeable == 2) {
if (sprev != NULL) {
- sprev->next = s->next;
+ sprev->next = s->next;
} else {
statelist = s;
}
@@ -1019,7 +1019,7 @@ struct fsm *lexc_to_fsm() {
fprintf(stderr,"Building lexicon...\n");
fflush(stdout);
#else
- if (verbose_lexc_)
+ if (verbose_lexc_)
{
fprintf(stderr,"Building lexicon...\n");
fflush(stderr);
diff --git a/back-ends/foma/mem.c b/back-ends/foma/mem.c
index 02ac396..0b7db45 100644
--- a/back-ends/foma/mem.c
+++ b/back-ends/foma/mem.c
@@ -93,6 +93,12 @@ INLINE void *xxcalloc(size_t nmemb, size_t size) {
}
INLINE char *xxstrdup(const char *s) {
- return(strdup(s));
+ // return(strdup(s));
+ size_t size = strlen(s) + 1;
+ char *p = malloc(size);
+ if (p) {
+ memcpy(p, s, size);
+ }
+ return p;
}
diff --git a/back-ends/foma/minimize.c b/back-ends/foma/minimize.c
index 20e5df4..240e0a7 100644
--- a/back-ends/foma/minimize.c
+++ b/back-ends/foma/minimize.c
@@ -27,7 +27,7 @@ static struct fsm *rebuild_machine(struct fsm *net);
static int *single_sigma_array, *double_sigma_array, *memo_table, *temp_move, *temp_group, maxsigma, epsilon_symbol, num_states, num_symbols, num_finals, mainloop, total_states;
-static _Bool *finals;
+static Boolean *finals;
struct statesym {
int target;
@@ -63,7 +63,7 @@ struct e {
struct agenda {
struct p *p;
struct agenda *next;
- _Bool index;
+ Boolean index;
};
// HFST MODIFICATIONS: struct trans_list -> struct trans_list_struct
@@ -108,8 +108,8 @@ struct fsm *fsm_minimize(struct fsm *net) {
if (g_minimize_hopcroft != 0) {
net = fsm_minimize_hop(net);
}
- else
- net = fsm_minimize_brz(net);
+ else
+ net = fsm_minimize_brz(net);
fsm_update_flags(net,YES,YES,YES,YES,UNK,UNK);
}
return(net);
@@ -124,7 +124,7 @@ static struct fsm *fsm_minimize_hop(struct fsm *net) {
struct e *temp_E;
struct trans_array_struct *tptr;
struct trans_list_struct *transitions;
- int i,j,minsym,next_minsym,current_i, stateno, thissize, source;
+ int i,j,minsym,next_minsym,current_i, stateno, thissize, source;
unsigned int tail;
fsm_count(net);
@@ -137,7 +137,7 @@ static struct fsm *fsm_minimize_hop(struct fsm *net) {
P = NULL;
- /*
+ /*
1. generate the inverse lookup table
2. generate P and E (partitions, states linked list)
3. Init Agenda = {Q, Q-F}
@@ -313,7 +313,7 @@ static INLINE int refine_states(int invstates) {
struct e *thise;
struct p *tP, *newP = NULL;
- /*
+ /*
1. add inverse(P,a) to table of inverses, disallowing duplicates
2. first pass on S, touch each state once, increasing P->t_count
3. for each P where counter != count, split and add to agenda
@@ -346,7 +346,7 @@ static INLINE int refine_states(int invstates) {
continue;
}
- if ((tP->t_count != tP->count) && (tP->count > 1) && (tP->t_count > 0)) {
+ if ((tP->t_count != tP->count) && (tP->count > 1) && (tP->t_count > 0)) {
/* Check if we already split this */
newP = tP->current_split;
@@ -395,7 +395,7 @@ static INLINE int refine_states(int invstates) {
selfsplit = 1;
} else {
/* If the block is not on the agenda, we add */
- /* the smaller of tP, newP and start the symloop from 0 */
+ /* the smaller of tP, newP and start the symloop from 0 */
agenda_add((tP->inv_count < tP->inv_t_count ? tP : newP),0);
}
/* Add to middle of P-chain */
@@ -462,7 +462,7 @@ static void agenda_add(struct p *pptr, int start) {
static void init_PE() {
/* Create two members of P
(nonfinals,finals)
- and put both of them on the agenda
+ and put both of them on the agenda
*/
int i;
@@ -590,7 +590,7 @@ static void generate_inverse(struct fsm *net) {
if ((fsm+i)->target == -1) {
continue;
}
- symbol = symbol_pair_to_single_symbol((fsm+i)->in,(fsm+i)->out);
+ symbol = symbol_pair_to_single_symbol((fsm+i)->in,(fsm+i)->out);
source = (fsm+i)->state_no;
target = (fsm+i)->target;
tptr = trans_array + target;
@@ -614,7 +614,7 @@ static void sigma_to_pairs(struct fsm *net) {
fsm = net->states;
- epsilon_symbol = -1;
+ epsilon_symbol = -1;
maxsigma = sigma_max(net->sigma);
maxsigma++;
@@ -642,7 +642,7 @@ static void sigma_to_pairs(struct fsm *net) {
/* Table for checking whether a state is final */
- finals = xxcalloc(num_states, sizeof(_Bool));
+ finals = xxcalloc(num_states, sizeof(Boolean));
x = 0; num_finals = 0;
net->arity = 1;
for (i=0; (fsm+i)->state_no != -1; i++) {
diff --git a/back-ends/foma/regex.c b/back-ends/foma/regex.c
index 4606c96..2a57cf1 100644
--- a/back-ends/foma/regex.c
+++ b/back-ends/foma/regex.c
@@ -158,6 +158,7 @@ void clear_rewrite_ruleset(struct rewrite_set *rewrite_rules) {
fsm_destroy(r->left);
fsm_destroy(r->right);
fsm_destroy(r->right2);
+ fsm_destroy(r->cross_product);
rp = r->next;
xxfree(r);
}
@@ -185,7 +186,6 @@ void add_rewrite_rule() {
new_rewrite_rule->rewrite_contexts = contexts;
new_rewrite_rule->next = rewrite_rules;
new_rewrite_rule->rule_direction = rule_direction;
- new_rewrite_rule->cpunion = NULL;
rewrite_rules = new_rewrite_rule;
rules = NULL;
@@ -194,6 +194,18 @@ void add_rewrite_rule() {
}
}
+void add_eprule(struct fsm *R, struct fsm *R2, int type) {
+ struct fsmrules *newrule;
+ rewrite = 1;
+ newrule = xxmalloc(sizeof(struct fsmrules));
+ newrule->left = fsm_empty_string();
+ newrule->right = R;
+ newrule->right2 = R2;
+ newrule->arrow_type = type;
+ newrule->next = rules;
+ rules = newrule;
+}
+
void add_rule(struct fsm *L, struct fsm *R, struct fsm *R2, int type) {
struct fsm *test;
struct fsmrules *newrule;
@@ -236,7 +248,7 @@ void add_rule(struct fsm *L, struct fsm *R, struct fsm *R2, int type) {
/* Line 268 of yacc.c */
-#line 240 "regex.c"
+#line 252 "regex.c"
/* Enabling traces. */
#ifndef YYDEBUG
@@ -357,7 +369,7 @@ typedef union YYSTYPE
{
/* Line 293 of yacc.c */
-#line 186 "regex.y"
+#line 198 "regex.y"
char *string;
struct fsm *net;
@@ -366,7 +378,7 @@ typedef union YYSTYPE
/* Line 293 of yacc.c */
-#line 370 "regex.c"
+#line 382 "regex.c"
} YYSTYPE;
# define YYSTYPE_IS_TRIVIAL 1
# define yystype YYSTYPE /* obsolescent; will be withdrawn */
@@ -391,7 +403,7 @@ typedef struct YYLTYPE
/* Line 343 of yacc.c */
-#line 395 "regex.c"
+#line 407 "regex.c"
#ifdef short
# undef short
@@ -756,20 +768,20 @@ static const yytype_int8 yyrhs[] =
/* YYRLINE[YYN] -- source line where rule number YYN was defined. */
static const yytype_uint16 yyrline[] =
{
- 0, 240, 240, 241, 244, 246, 247, 248, 249, 251,
- 253, 254, 255, 256, 257, 258, 259, 260, 261, 262,
- 263, 264, 265, 266, 268, 269, 270, 271, 272, 273,
- 274, 275, 276, 277, 279, 281, 282, 283, 284, 285,
- 286, 287, 288, 289, 290, 291, 292, 295, 296, 297,
- 298, 300, 302, 304, 305, 306, 307, 308, 309, 310,
- 311, 313, 314, 315, 317, 318, 319, 320, 322, 323,
- 324, 325, 326, 327, 329, 330, 331, 332, 333, 335,
- 337, 338, 339, 340, 341, 342, 343, 344, 345, 347,
- 348, 349, 350, 352, 353, 355, 356, 357, 358, 359,
- 360, 361, 362, 363, 364, 366, 367, 369, 370, 371,
- 372, 373, 374, 375, 376, 377, 378, 379, 380, 381,
- 382, 383, 384, 385, 386, 387, 388, 389, 390, 391,
- 393, 395, 398, 399, 401, 403, 405
+ 0, 252, 252, 253, 256, 258, 259, 260, 261, 263,
+ 265, 266, 267, 268, 269, 270, 271, 272, 273, 274,
+ 275, 276, 277, 278, 280, 281, 282, 283, 284, 285,
+ 286, 287, 288, 289, 291, 293, 294, 295, 296, 297,
+ 298, 299, 300, 301, 302, 303, 304, 307, 308, 309,
+ 310, 312, 314, 316, 317, 318, 319, 320, 321, 322,
+ 323, 325, 326, 327, 329, 330, 331, 332, 334, 335,
+ 336, 337, 338, 339, 341, 342, 343, 344, 345, 347,
+ 349, 350, 351, 352, 353, 354, 355, 356, 357, 359,
+ 360, 361, 362, 364, 365, 367, 368, 369, 370, 371,
+ 372, 373, 374, 375, 376, 378, 379, 381, 382, 383,
+ 384, 385, 386, 387, 388, 389, 390, 391, 392, 393,
+ 394, 395, 396, 397, 398, 399, 400, 401, 402, 403,
+ 405, 407, 410, 411, 413, 415, 417
};
#endif
@@ -1958,7 +1970,7 @@ YYLTYPE yylloc;
/* User initialization code. */
/* Line 1590 of yacc.c */
-#line 199 "regex.y"
+#line 211 "regex.y"
{
clear_quantifiers();
rewrite = 0;
@@ -1970,7 +1982,7 @@ YYLTYPE yylloc;
}
/* Line 1590 of yacc.c */
-#line 1974 "regex.c"
+#line 1986 "regex.c"
goto yysetstate;
@@ -2157,938 +2169,938 @@ yyreduce:
case 4:
/* Line 1806 of yacc.c */
-#line 244 "regex.y"
+#line 256 "regex.y"
{ current_parse = (yyvsp[(1) - (2)].net); }
break;
case 5:
/* Line 1806 of yacc.c */
-#line 246 "regex.y"
+#line 258 "regex.y"
{ }
break;
case 6:
/* Line 1806 of yacc.c */
-#line 247 "regex.y"
+#line 259 "regex.y"
{ (yyval.net) = fsm_compose((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net)); }
break;
case 7:
/* Line 1806 of yacc.c */
-#line 248 "regex.y"
+#line 260 "regex.y"
{ (yyval.net) = fsm_lenient_compose((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net)); }
break;
case 8:
/* Line 1806 of yacc.c */
-#line 249 "regex.y"
+#line 261 "regex.y"
{ (yyval.net) = fsm_cross_product((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net)); }
break;
case 9:
/* Line 1806 of yacc.c */
-#line 251 "regex.y"
+#line 263 "regex.y"
{ if (rewrite) { add_rewrite_rule(); (yyval.net) = fsm_rewrite(rewrite_rules); clear_rewrite_ruleset(rewrite_rules); } rewrite = 0; contexts = NULL; rules = NULL; rewrite_rules = NULL; }
break;
case 10:
/* Line 1806 of yacc.c */
-#line 253 "regex.y"
+#line 265 "regex.y"
{ }
break;
case 11:
/* Line 1806 of yacc.c */
-#line 254 "regex.y"
+#line 266 "regex.y"
{ (yyval.net) = NULL; add_context_pair((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net));}
break;
case 12:
/* Line 1806 of yacc.c */
-#line 255 "regex.y"
+#line 267 "regex.y"
{ add_context_pair((yyvsp[(1) - (2)].net),fsm_empty_string()); }
break;
case 13:
/* Line 1806 of yacc.c */
-#line 256 "regex.y"
+#line 268 "regex.y"
{ add_context_pair((yyvsp[(1) - (4)].net),fsm_empty_string()); }
break;
case 14:
/* Line 1806 of yacc.c */
-#line 257 "regex.y"
+#line 269 "regex.y"
{ add_context_pair(fsm_empty_string(),fsm_empty_string());}
break;
case 15:
/* Line 1806 of yacc.c */
-#line 258 "regex.y"
+#line 270 "regex.y"
{ add_rewrite_rule(); add_context_pair(fsm_empty_string(),fsm_empty_string());}
break;
case 16:
/* Line 1806 of yacc.c */
-#line 259 "regex.y"
+#line 271 "regex.y"
{ add_rewrite_rule(); add_context_pair((yyvsp[(1) - (4)].net),fsm_empty_string());}
break;
case 17:
/* Line 1806 of yacc.c */
-#line 260 "regex.y"
+#line 272 "regex.y"
{ add_context_pair(fsm_empty_string(),fsm_empty_string());}
break;
case 18:
/* Line 1806 of yacc.c */
-#line 261 "regex.y"
+#line 273 "regex.y"
{ add_context_pair(fsm_empty_string(),(yyvsp[(2) - (2)].net)); }
break;
case 19:
/* Line 1806 of yacc.c */
-#line 262 "regex.y"
+#line 274 "regex.y"
{ add_context_pair(fsm_empty_string(),(yyvsp[(2) - (4)].net)); }
break;
case 20:
/* Line 1806 of yacc.c */
-#line 263 "regex.y"
+#line 275 "regex.y"
{ add_context_pair((yyvsp[(1) - (5)].net),(yyvsp[(3) - (5)].net)); }
break;
case 21:
/* Line 1806 of yacc.c */
-#line 264 "regex.y"
+#line 276 "regex.y"
{ (yyval.net) = fsm_context_restrict((yyvsp[(1) - (3)].net),contexts); fsm_clear_contexts(contexts);}
break;
case 22:
/* Line 1806 of yacc.c */
-#line 265 "regex.y"
+#line 277 "regex.y"
{ add_rule((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net),NULL,(yyvsp[(2) - (3)].type)); if ((yyvsp[(1) - (3)].net)->arity == 2) { printf("Error: LHS is transducer\n"); YYERROR;}}
break;
case 23:
/* Line 1806 of yacc.c */
-#line 266 "regex.y"
+#line 278 "regex.y"
{ add_rule((yyvsp[(1) - (2)].net),NULL,NULL,(yyvsp[(2) - (2)].type)); }
break;
case 24:
/* Line 1806 of yacc.c */
-#line 268 "regex.y"
+#line 280 "regex.y"
{ add_rule((yyvsp[(2) - (5)].net),(yyvsp[(5) - (5)].net),NULL,(yyvsp[(4) - (5)].type)|ARROW_DOTTED); if ((yyvsp[(5) - (5)].net) == NULL) { YYERROR;}}
break;
case 25:
/* Line 1806 of yacc.c */
-#line 269 "regex.y"
- { add_rule(fsm_empty_string(),(yyvsp[(4) - (4)].net),NULL,(yyvsp[(3) - (4)].type)|ARROW_DOTTED); if ((yyvsp[(4) - (4)].net) == NULL) { YYERROR;}}
+#line 281 "regex.y"
+ { add_eprule((yyvsp[(4) - (4)].net),NULL,(yyvsp[(3) - (4)].type)|ARROW_DOTTED); if ((yyvsp[(4) - (4)].net) == NULL) { YYERROR;}}
break;
case 26:
/* Line 1806 of yacc.c */
-#line 270 "regex.y"
+#line 282 "regex.y"
{ add_rule((yyvsp[(2) - (7)].net),(yyvsp[(5) - (7)].net),NULL,(yyvsp[(4) - (7)].type)|ARROW_DOTTED);}
break;
case 27:
/* Line 1806 of yacc.c */
-#line 271 "regex.y"
- { add_rule(fsm_empty_string(),(yyvsp[(4) - (6)].net),NULL,(yyvsp[(3) - (6)].type)|ARROW_DOTTED);}
+#line 283 "regex.y"
+ { add_eprule((yyvsp[(4) - (6)].net),NULL,(yyvsp[(3) - (6)].type)|ARROW_DOTTED);}
break;
case 28:
/* Line 1806 of yacc.c */
-#line 272 "regex.y"
+#line 284 "regex.y"
{ add_rule((yyvsp[(2) - (7)].net),(yyvsp[(5) - (7)].net),NULL,(yyvsp[(4) - (7)].type)|ARROW_DOTTED); rule_direction = (yyvsp[(6) - (7)].type);}
break;
case 29:
/* Line 1806 of yacc.c */
-#line 273 "regex.y"
- { add_rule(fsm_empty_string(),(yyvsp[(4) - (6)].net),NULL,(yyvsp[(3) - (6)].type)|ARROW_DOTTED); rule_direction = (yyvsp[(5) - (6)].type);}
+#line 285 "regex.y"
+ { add_eprule((yyvsp[(4) - (6)].net),NULL,(yyvsp[(3) - (6)].type)|ARROW_DOTTED); rule_direction = (yyvsp[(5) - (6)].type);}
break;
case 30:
/* Line 1806 of yacc.c */
-#line 274 "regex.y"
+#line 286 "regex.y"
{ add_rule((yyvsp[(1) - (5)].net),(yyvsp[(3) - (5)].net),NULL,(yyvsp[(2) - (5)].type));}
break;
case 31:
/* Line 1806 of yacc.c */
-#line 275 "regex.y"
+#line 287 "regex.y"
{ add_rule((yyvsp[(1) - (4)].net),NULL,NULL,(yyvsp[(2) - (4)].type));}
break;
case 32:
/* Line 1806 of yacc.c */
-#line 276 "regex.y"
+#line 288 "regex.y"
{ add_rule((yyvsp[(1) - (5)].net),(yyvsp[(3) - (5)].net),NULL,(yyvsp[(2) - (5)].type)); rule_direction = (yyvsp[(4) - (5)].type);}
break;
case 33:
/* Line 1806 of yacc.c */
-#line 277 "regex.y"
+#line 289 "regex.y"
{ add_rule((yyvsp[(1) - (4)].net),NULL,NULL,(yyvsp[(2) - (4)].type)); rule_direction = (yyvsp[(3) - (4)].type);}
break;
case 34:
/* Line 1806 of yacc.c */
-#line 279 "regex.y"
+#line 291 "regex.y"
{ add_rewrite_rule();}
break;
case 35:
/* Line 1806 of yacc.c */
-#line 281 "regex.y"
+#line 293 "regex.y"
{ add_rule((yyvsp[(1) - (3)].net),fsm_empty_string(),fsm_empty_string(),(yyvsp[(2) - (3)].type));}
break;
case 36:
/* Line 1806 of yacc.c */
-#line 282 "regex.y"
+#line 294 "regex.y"
{ add_rule((yyvsp[(1) - (4)].net),fsm_empty_string(),(yyvsp[(4) - (4)].net),(yyvsp[(2) - (4)].type));}
break;
case 37:
/* Line 1806 of yacc.c */
-#line 283 "regex.y"
+#line 295 "regex.y"
{ add_rule((yyvsp[(1) - (4)].net),(yyvsp[(3) - (4)].net),fsm_empty_string(),(yyvsp[(2) - (4)].type));}
break;
case 38:
/* Line 1806 of yacc.c */
-#line 284 "regex.y"
+#line 296 "regex.y"
{ add_rule((yyvsp[(1) - (5)].net),(yyvsp[(3) - (5)].net),(yyvsp[(5) - (5)].net),(yyvsp[(2) - (5)].type));}
break;
case 39:
/* Line 1806 of yacc.c */
-#line 285 "regex.y"
+#line 297 "regex.y"
{ add_rule((yyvsp[(1) - (7)].net),(yyvsp[(3) - (7)].net),(yyvsp[(5) - (7)].net),(yyvsp[(2) - (7)].type));}
break;
case 40:
/* Line 1806 of yacc.c */
-#line 286 "regex.y"
+#line 298 "regex.y"
{ add_rule((yyvsp[(1) - (6)].net),(yyvsp[(3) - (6)].net),fsm_empty_string(),(yyvsp[(2) - (6)].type));}
break;
case 41:
/* Line 1806 of yacc.c */
-#line 287 "regex.y"
+#line 299 "regex.y"
{ add_rule((yyvsp[(1) - (6)].net),fsm_empty_string(),(yyvsp[(4) - (6)].net),(yyvsp[(2) - (6)].type));}
break;
case 42:
/* Line 1806 of yacc.c */
-#line 288 "regex.y"
+#line 300 "regex.y"
{ add_rule((yyvsp[(1) - (5)].net),fsm_empty_string(),fsm_empty_string(),(yyvsp[(2) - (5)].type));}
break;
case 43:
/* Line 1806 of yacc.c */
-#line 289 "regex.y"
+#line 301 "regex.y"
{ add_rule((yyvsp[(1) - (7)].net),(yyvsp[(3) - (7)].net),(yyvsp[(5) - (7)].net),(yyvsp[(2) - (7)].type)); rule_direction = (yyvsp[(6) - (7)].type);}
break;
case 44:
/* Line 1806 of yacc.c */
-#line 290 "regex.y"
+#line 302 "regex.y"
{ add_rule((yyvsp[(1) - (6)].net),fsm_empty_string(),(yyvsp[(4) - (6)].net),(yyvsp[(2) - (6)].type)); rule_direction = (yyvsp[(5) - (6)].type);}
break;
case 45:
/* Line 1806 of yacc.c */
-#line 291 "regex.y"
+#line 303 "regex.y"
{ add_rule((yyvsp[(1) - (6)].net),(yyvsp[(3) - (6)].net),fsm_empty_string(),(yyvsp[(2) - (6)].type)); rule_direction = (yyvsp[(5) - (6)].type);}
break;
case 46:
/* Line 1806 of yacc.c */
-#line 292 "regex.y"
+#line 304 "regex.y"
{ add_rule((yyvsp[(1) - (5)].net),fsm_empty_string(),fsm_empty_string(),(yyvsp[(2) - (5)].type)); rule_direction = (yyvsp[(4) - (5)].type);}
break;
case 47:
/* Line 1806 of yacc.c */
-#line 295 "regex.y"
+#line 307 "regex.y"
{ }
break;
case 48:
/* Line 1806 of yacc.c */
-#line 296 "regex.y"
+#line 308 "regex.y"
{ (yyval.net) = fsm_shuffle((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net)); }
break;
case 49:
/* Line 1806 of yacc.c */
-#line 297 "regex.y"
+#line 309 "regex.y"
{ (yyval.net) = fsm_precedes((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net)); }
break;
case 50:
/* Line 1806 of yacc.c */
-#line 298 "regex.y"
+#line 310 "regex.y"
{ (yyval.net) = fsm_follows((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net)); }
break;
case 51:
/* Line 1806 of yacc.c */
-#line 300 "regex.y"
+#line 312 "regex.y"
{ }
break;
case 52:
/* Line 1806 of yacc.c */
-#line 302 "regex.y"
+#line 314 "regex.y"
{ }
break;
case 53:
/* Line 1806 of yacc.c */
-#line 304 "regex.y"
+#line 316 "regex.y"
{ }
break;
case 54:
/* Line 1806 of yacc.c */
-#line 305 "regex.y"
+#line 317 "regex.y"
{ (yyval.net) = fsm_union((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net)); }
break;
case 55:
/* Line 1806 of yacc.c */
-#line 306 "regex.y"
+#line 318 "regex.y"
{ (yyval.net) = fsm_priority_union_upper((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net)); }
break;
case 56:
/* Line 1806 of yacc.c */
-#line 307 "regex.y"
+#line 319 "regex.y"
{ (yyval.net) = fsm_priority_union_lower((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net)); }
break;
case 57:
/* Line 1806 of yacc.c */
-#line 308 "regex.y"
+#line 320 "regex.y"
{ (yyval.net) = fsm_intersect((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net)); }
break;
case 58:
/* Line 1806 of yacc.c */
-#line 309 "regex.y"
+#line 321 "regex.y"
{ (yyval.net) = fsm_minus((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net)); }
break;
case 59:
/* Line 1806 of yacc.c */
-#line 310 "regex.y"
+#line 322 "regex.y"
{ (yyval.net) = fsm_union(fsm_complement((yyvsp[(1) - (3)].net)),(yyvsp[(3) - (3)].net)); }
break;
case 60:
/* Line 1806 of yacc.c */
-#line 311 "regex.y"
+#line 323 "regex.y"
{ (yyval.net) = fsm_intersect(fsm_union(fsm_complement(fsm_copy((yyvsp[(1) - (3)].net))),fsm_copy((yyvsp[(3) - (3)].net))), fsm_union(fsm_complement(fsm_copy((yyvsp[(3) - (3)].net))),fsm_copy((yyvsp[(1) - (3)].net)))); fsm_destroy((yyvsp[(1) - (3)].net)); fsm_destroy((yyvsp[(3) - (3)].net));}
break;
case 61:
/* Line 1806 of yacc.c */
-#line 313 "regex.y"
+#line 325 "regex.y"
{ }
break;
case 62:
/* Line 1806 of yacc.c */
-#line 314 "regex.y"
+#line 326 "regex.y"
{ (yyval.net) = fsm_concat((yyvsp[(1) - (2)].net),(yyvsp[(2) - (2)].net)); }
break;
case 63:
/* Line 1806 of yacc.c */
-#line 315 "regex.y"
+#line 327 "regex.y"
{ (yyval.net) = fsm_ignore(fsm_contains(fsm_concat(fsm_symbol((yyvsp[(1) - (3)].string)),fsm_concat((yyvsp[(3) - (3)].net),fsm_symbol((yyvsp[(1) - (3)].string))))),union_quantifiers(),OP_IGNORE_ALL); }
break;
case 64:
/* Line 1806 of yacc.c */
-#line 317 "regex.y"
+#line 329 "regex.y"
{ (yyval.net) = fsm_logical_eq((yyvsp[(1) - (3)].string),(yyvsp[(3) - (3)].string)); }
break;
case 65:
/* Line 1806 of yacc.c */
-#line 318 "regex.y"
+#line 330 "regex.y"
{ (yyval.net) = fsm_complement(fsm_logical_eq((yyvsp[(1) - (3)].string),(yyvsp[(3) - (3)].string))); }
break;
case 66:
/* Line 1806 of yacc.c */
-#line 319 "regex.y"
+#line 331 "regex.y"
{ (yyval.net) = fsm_logical_precedence((yyvsp[(1) - (3)].string),(yyvsp[(3) - (3)].string)); }
break;
case 67:
/* Line 1806 of yacc.c */
-#line 320 "regex.y"
+#line 332 "regex.y"
{ (yyval.net) = fsm_logical_precedence((yyvsp[(3) - (3)].string),(yyvsp[(1) - (3)].string)); }
break;
case 68:
/* Line 1806 of yacc.c */
-#line 322 "regex.y"
+#line 334 "regex.y"
{ }
break;
case 69:
/* Line 1806 of yacc.c */
-#line 323 "regex.y"
+#line 335 "regex.y"
{ (yyval.net) = fsm_ignore((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net), OP_IGNORE_ALL); }
break;
case 70:
/* Line 1806 of yacc.c */
-#line 324 "regex.y"
+#line 336 "regex.y"
{ (yyval.net) = fsm_ignore((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net), OP_IGNORE_INTERNAL); }
break;
case 71:
/* Line 1806 of yacc.c */
-#line 325 "regex.y"
+#line 337 "regex.y"
{ (yyval.net) = fsm_quotient_right((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net)); }
break;
case 72:
/* Line 1806 of yacc.c */
-#line 326 "regex.y"
+#line 338 "regex.y"
{ (yyval.net) = fsm_quotient_left((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net)); }
break;
case 73:
/* Line 1806 of yacc.c */
-#line 327 "regex.y"
+#line 339 "regex.y"
{ (yyval.net) = fsm_quotient_interleave((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net)); }
break;
case 74:
/* Line 1806 of yacc.c */
-#line 329 "regex.y"
+#line 341 "regex.y"
{ }
break;
case 75:
/* Line 1806 of yacc.c */
-#line 330 "regex.y"
+#line 342 "regex.y"
{ (yyval.net) = fsm_complement((yyvsp[(2) - (2)].net)); }
break;
case 76:
/* Line 1806 of yacc.c */
-#line 331 "regex.y"
+#line 343 "regex.y"
{ (yyval.net) = fsm_contains((yyvsp[(2) - (2)].net)); }
break;
case 77:
/* Line 1806 of yacc.c */
-#line 332 "regex.y"
+#line 344 "regex.y"
{ (yyval.net) = fsm_contains_one((yyvsp[(2) - (2)].net)); }
break;
case 78:
/* Line 1806 of yacc.c */
-#line 333 "regex.y"
+#line 345 "regex.y"
{ (yyval.net) = fsm_contains_opt_one((yyvsp[(2) - (2)].net)); }
break;
case 79:
/* Line 1806 of yacc.c */
-#line 335 "regex.y"
+#line 347 "regex.y"
{ }
break;
case 80:
/* Line 1806 of yacc.c */
-#line 337 "regex.y"
+#line 349 "regex.y"
{ }
break;
case 81:
/* Line 1806 of yacc.c */
-#line 338 "regex.y"
+#line 350 "regex.y"
{ (yyval.net) = fsm_kleene_star(fsm_minimize((yyvsp[(1) - (2)].net))); }
break;
case 82:
/* Line 1806 of yacc.c */
-#line 339 "regex.y"
+#line 351 "regex.y"
{ (yyval.net) = fsm_kleene_plus((yyvsp[(1) - (2)].net)); }
break;
case 83:
/* Line 1806 of yacc.c */
-#line 340 "regex.y"
+#line 352 "regex.y"
{ (yyval.net) = fsm_determinize(fsm_reverse((yyvsp[(1) - (2)].net))); }
break;
case 84:
/* Line 1806 of yacc.c */
-#line 341 "regex.y"
+#line 353 "regex.y"
{ (yyval.net) = fsm_invert((yyvsp[(1) - (2)].net)); }
break;
case 85:
/* Line 1806 of yacc.c */
-#line 342 "regex.y"
+#line 354 "regex.y"
{ (yyval.net) = fsm_upper((yyvsp[(1) - (2)].net)); }
break;
case 86:
/* Line 1806 of yacc.c */
-#line 343 "regex.y"
+#line 355 "regex.y"
{ (yyval.net) = fsm_lower((yyvsp[(1) - (2)].net)); }
break;
case 87:
/* Line 1806 of yacc.c */
-#line 344 "regex.y"
+#line 356 "regex.y"
{ (yyval.net) = flag_eliminate((yyvsp[(1) - (2)].net), NULL); }
break;
case 88:
/* Line 1806 of yacc.c */
-#line 345 "regex.y"
+#line 357 "regex.y"
{ (yyval.net) = fsm_cross_product((yyvsp[(1) - (3)].net),(yyvsp[(3) - (3)].net)); }
break;
case 89:
/* Line 1806 of yacc.c */
-#line 347 "regex.y"
+#line 359 "regex.y"
{ (yyval.net) = fsm_concat_n((yyvsp[(1) - (2)].net),atoi((yyvsp[(2) - (2)].string))); }
break;
case 90:
/* Line 1806 of yacc.c */
-#line 348 "regex.y"
+#line 360 "regex.y"
{ (yyval.net) = fsm_concat(fsm_concat_n(fsm_copy((yyvsp[(1) - (2)].net)), atoi((yyvsp[(2) - (2)].string))),fsm_kleene_plus(fsm_copy((yyvsp[(1) - (2)].net)))); fsm_destroy((yyvsp[(1) - (2)].net)); }
break;
case 91:
/* Line 1806 of yacc.c */
-#line 349 "regex.y"
+#line 361 "regex.y"
{ (yyval.net) = fsm_concat_m_n((yyvsp[(1) - (2)].net),0,atoi((yyvsp[(2) - (2)].string))-1); }
break;
case 92:
/* Line 1806 of yacc.c */
-#line 350 "regex.y"
+#line 362 "regex.y"
{ (yyval.net) = fsm_concat_m_n((yyvsp[(1) - (2)].net),atoi((yyvsp[(2) - (2)].string)),atoi(strstr((yyvsp[(2) - (2)].string),",")+1)); }
break;
case 93:
/* Line 1806 of yacc.c */
-#line 352 "regex.y"
+#line 364 "regex.y"
{ }
break;
case 94:
/* Line 1806 of yacc.c */
-#line 353 "regex.y"
+#line 365 "regex.y"
{ (yyval.net) = fsm_term_negation((yyvsp[(2) - (2)].net)); }
break;
case 95:
/* Line 1806 of yacc.c */
-#line 355 "regex.y"
+#line 367 "regex.y"
{ (yyval.net) = (yyvsp[(1) - (1)].net);}
break;
case 96:
/* Line 1806 of yacc.c */
-#line 356 "regex.y"
+#line 368 "regex.y"
{ (yyval.net) = (yyvsp[(1) - (1)].net); }
break;
case 97:
/* Line 1806 of yacc.c */
-#line 357 "regex.y"
+#line 369 "regex.y"
{ (yyval.net) = fsm_complement(fsm_substitute_symbol(fsm_intersect(fsm_quantifier((yyvsp[(1) - (4)].string)),fsm_complement((yyvsp[(3) - (4)].net))),(yyvsp[(1) - (4)].string),"@_EPSILON_SYMBOL_@")); purge_quantifier((yyvsp[(1) - (4)].string)); }
break;
case 98:
/* Line 1806 of yacc.c */
-#line 358 "regex.y"
+#line 370 "regex.y"
{ (yyval.net) = fsm_substitute_symbol(fsm_intersect(fsm_quantifier((yyvsp[(1) - (2)].string)),(yyvsp[(2) - (2)].net)),(yyvsp[(1) - (2)].string),"@_EPSILON_SYMBOL_@"); purge_quantifier((yyvsp[(1) - (2)].string)); }
break;
case 99:
/* Line 1806 of yacc.c */
-#line 359 "regex.y"
+#line 371 "regex.y"
{ if (count_quantifiers()) (yyval.net) = (yyvsp[(2) - (3)].net); else {(yyval.net) = fsm_optionality((yyvsp[(2) - (3)].net));} }
break;
case 100:
/* Line 1806 of yacc.c */
-#line 360 "regex.y"
+#line 372 "regex.y"
{ (yyval.net) = (yyvsp[(2) - (3)].net); }
break;
case 101:
/* Line 1806 of yacc.c */
-#line 361 "regex.y"
+#line 373 "regex.y"
{(yyval.net) = fsm_concat(fsm_universal(),fsm_concat(fsm_symbol((yyvsp[(2) - (5)].string)),fsm_concat(fsm_universal(),fsm_concat(fsm_symbol((yyvsp[(2) - (5)].string)),fsm_concat(union_quantifiers(),fsm_concat(fsm_symbol((yyvsp[(4) - (5)].string)),fsm_concat(fsm_universal(),fsm_concat(fsm_symbol((yyvsp[(4) - (5)].string)),fsm_universal())))))))); }
break;
case 102:
/* Line 1806 of yacc.c */
-#line 362 "regex.y"
+#line 374 "regex.y"
{(yyval.net) = fsm_concat(fsm_universal(),fsm_concat(fsm_symbol((yyvsp[(2) - (5)].string)),fsm_concat(fsm_universal(),fsm_concat(fsm_symbol((yyvsp[(2) - (5)].string)),fsm_concat(fsm_ignore((yyvsp[(4) - (5)].net),union_quantifiers(),OP_IGNORE_ALL),fsm_universal()))))); }
break;
case 103:
/* Line 1806 of yacc.c */
-#line 363 "regex.y"
+#line 375 "regex.y"
{(yyval.net) = fsm_concat(fsm_universal(),fsm_concat(fsm_ignore((yyvsp[(2) - (5)].net),union_quantifiers(),OP_IGNORE_ALL),fsm_concat(fsm_symbol((yyvsp[(4) - (5)].string)),fsm_concat(fsm_universal(),fsm_concat(fsm_symbol((yyvsp[(4) - (5)].string)),fsm_universal()))))); }
break;
case 104:
/* Line 1806 of yacc.c */
-#line 364 "regex.y"
+#line 376 "regex.y"
{(yyval.net) = fsm_substitute_symbol((yyvsp[(1) - (2)].net),subval1,subval2); substituting = 0; xxfree(subval1); xxfree(subval2); subval1 = subval2 = NULL;}
break;
case 105:
/* Line 1806 of yacc.c */
-#line 366 "regex.y"
+#line 378 "regex.y"
{ (yyval.net) = (yyvsp[(3) - (4)].net); substituting = 1; }
break;
case 106:
/* Line 1806 of yacc.c */
-#line 367 "regex.y"
+#line 379 "regex.y"
{ subval1 = (yyvsp[(2) - (4)].string); subval2 = (yyvsp[(4) - (4)].string); }
break;
case 107:
/* Line 1806 of yacc.c */
-#line 369 "regex.y"
+#line 381 "regex.y"
{ (yyval.net) = (yyvsp[(1) - (1)].net); }
break;
case 108:
/* Line 1806 of yacc.c */
-#line 370 "regex.y"
+#line 382 "regex.y"
{ (yyval.net) = fsm_boolean(fsm_isidentity((yyvsp[(2) - (3)].net))); }
break;
case 109:
/* Line 1806 of yacc.c */
-#line 371 "regex.y"
+#line 383 "regex.y"
{ (yyval.net) = fsm_boolean(fsm_isfunctional((yyvsp[(2) - (3)].net))); }
break;
case 110:
/* Line 1806 of yacc.c */
-#line 372 "regex.y"
+#line 384 "regex.y"
{ (yyval.net) = fsm_boolean(fsm_isunambiguous((yyvsp[(2) - (3)].net))); }
break;
case 111:
/* Line 1806 of yacc.c */
-#line 373 "regex.y"
+#line 385 "regex.y"
{ (yyval.net) = fsm_extract_nonidentity(fsm_copy((yyvsp[(2) - (3)].net))); }
break;
case 112:
/* Line 1806 of yacc.c */
-#line 374 "regex.y"
+#line 386 "regex.y"
{ (yyval.net) = fsm_lowerdet(fsm_copy((yyvsp[(2) - (3)].net))); }
break;
case 113:
/* Line 1806 of yacc.c */
-#line 375 "regex.y"
+#line 387 "regex.y"
{ (yyval.net) = fsm_lowerdeteps(fsm_copy((yyvsp[(2) - (3)].net))); }
break;
case 114:
/* Line 1806 of yacc.c */
-#line 376 "regex.y"
+#line 388 "regex.y"
{ (yyval.net) = fsm_markallfinal(fsm_copy((yyvsp[(2) - (3)].net))); }
break;
case 115:
/* Line 1806 of yacc.c */
-#line 377 "regex.y"
+#line 389 "regex.y"
{ (yyval.net) = fsm_extract_unambiguous(fsm_copy((yyvsp[(2) - (3)].net))); }
break;
case 116:
/* Line 1806 of yacc.c */
-#line 378 "regex.y"
+#line 390 "regex.y"
{ (yyval.net) = fsm_extract_ambiguous(fsm_copy((yyvsp[(2) - (3)].net))); }
break;
case 117:
/* Line 1806 of yacc.c */
-#line 379 "regex.y"
+#line 391 "regex.y"
{ (yyval.net) = fsm_extract_ambiguous_domain(fsm_copy((yyvsp[(2) - (3)].net))); }
break;
case 118:
/* Line 1806 of yacc.c */
-#line 380 "regex.y"
+#line 392 "regex.y"
{ (yyval.net) = fsm_letter_machine(fsm_copy((yyvsp[(2) - (3)].net))); }
break;
case 119:
/* Line 1806 of yacc.c */
-#line 381 "regex.y"
+#line 393 "regex.y"
{ (yyval.net) = fsm_mark_fsm_tail((yyvsp[(2) - (5)].net),(yyvsp[(4) - (5)].net)); }
break;
case 120:
/* Line 1806 of yacc.c */
-#line 382 "regex.y"
+#line 394 "regex.y"
{ (yyval.net) = fsm_add_loop((yyvsp[(2) - (5)].net),(yyvsp[(4) - (5)].net),1); }
break;
case 121:
/* Line 1806 of yacc.c */
-#line 383 "regex.y"
+#line 395 "regex.y"
{ (yyval.net) = fsm_add_loop((yyvsp[(2) - (5)].net),(yyvsp[(4) - (5)].net),0); }
break;
case 122:
/* Line 1806 of yacc.c */
-#line 384 "regex.y"
+#line 396 "regex.y"
{ (yyval.net) = fsm_add_loop((yyvsp[(2) - (5)].net),(yyvsp[(4) - (5)].net),2); }
break;
case 123:
/* Line 1806 of yacc.c */
-#line 385 "regex.y"
+#line 397 "regex.y"
{ (yyval.net) = fsm_add_sink((yyvsp[(2) - (3)].net),1); }
break;
case 124:
/* Line 1806 of yacc.c */
-#line 386 "regex.y"
+#line 398 "regex.y"
{ (yyval.net) = fsm_left_rewr((yyvsp[(2) - (5)].net),(yyvsp[(4) - (5)].net)); }
break;
case 125:
/* Line 1806 of yacc.c */
-#line 387 "regex.y"
+#line 399 "regex.y"
{ (yyval.net) = fsm_flatten((yyvsp[(2) - (5)].net),(yyvsp[(4) - (5)].net)); }
break;
case 126:
/* Line 1806 of yacc.c */
-#line 388 "regex.y"
+#line 400 "regex.y"
{ (yyval.net) = fsm_substitute_label((yyvsp[(2) - (7)].net), fsm_network_to_char((yyvsp[(4) - (7)].net)), (yyvsp[(6) - (7)].net)); }
break;
case 127:
/* Line 1806 of yacc.c */
-#line 389 "regex.y"
+#line 401 "regex.y"
{ (yyval.net) = fsm_close_sigma(fsm_copy((yyvsp[(2) - (3)].net)), 0); }
break;
case 128:
/* Line 1806 of yacc.c */
-#line 390 "regex.y"
+#line 402 "regex.y"
{ (yyval.net) = fsm_close_sigma(fsm_copy((yyvsp[(2) - (3)].net)), 1); }
break;
case 129:
/* Line 1806 of yacc.c */
-#line 391 "regex.y"
+#line 403 "regex.y"
{ (yyval.net) = fsm_equal_substrings((yyvsp[(2) - (7)].net),(yyvsp[(4) - (7)].net),(yyvsp[(6) - (7)].net)); }
break;
case 130:
/* Line 1806 of yacc.c */
-#line 394 "regex.y"
+#line 406 "regex.y"
{ frec++; fargptr[frec] = 0 ;declare_function_name((yyvsp[(1) - (3)].string)) ; add_function_argument((yyvsp[(2) - (3)].net)); }
break;
case 131:
/* Line 1806 of yacc.c */
-#line 396 "regex.y"
+#line 408 "regex.y"
{ frec++; fargptr[frec] = 0 ;declare_function_name((yyvsp[(1) - (2)].string)) ; add_function_argument((yyvsp[(2) - (2)].net)); }
break;
case 132:
/* Line 1806 of yacc.c */
-#line 398 "regex.y"
+#line 410 "regex.y"
{ add_function_argument((yyvsp[(2) - (3)].net)); }
break;
case 133:
/* Line 1806 of yacc.c */
-#line 399 "regex.y"
+#line 411 "regex.y"
{ add_function_argument((yyvsp[(2) - (3)].net)); }
break;
case 134:
/* Line 1806 of yacc.c */
-#line 402 "regex.y"
+#line 414 "regex.y"
{ add_function_argument((yyvsp[(2) - (3)].net)); if (((yyval.net) = function_apply(defined_nets, defined_funcs)) == NULL) YYERROR; }
break;
case 135:
/* Line 1806 of yacc.c */
-#line 404 "regex.y"
+#line 416 "regex.y"
{ add_function_argument((yyvsp[(2) - (3)].net)); if (((yyval.net) = function_apply(defined_nets, defined_funcs)) == NULL) YYERROR; }
break;
case 136:
/* Line 1806 of yacc.c */
-#line 406 "regex.y"
+#line 418 "regex.y"
{ if (((yyval.net) = function_apply(defined_nets, defined_funcs)) == NULL) YYERROR;}
break;
/* Line 1806 of yacc.c */
-#line 3092 "regex.c"
+#line 3104 "regex.c"
default: break;
}
/* User semantic actions sometimes alter yychar, and that requires
@@ -3326,6 +3338,6 @@ yyreturn:
/* Line 2067 of yacc.c */
-#line 408 "regex.y"
+#line 420 "regex.y"
diff --git a/back-ends/foma/regex.l b/back-ends/foma/regex.l
index bb8c08e..7a28c43 100644
--- a/back-ends/foma/regex.l
+++ b/back-ends/foma/regex.l
@@ -181,7 +181,7 @@ BRACED [{]([^}]|[\300-\337].|[\340-\357]..|[\360-\367]...)+[}]
yylval_param->net = fsm_parse_regex_string(tempstr2);
if (yylval_param->net != NULL) {
return NET;
- }
+ }
}
}
@@ -270,7 +270,7 @@ BRACED [{]([^}]|[\300-\337].|[\340-\357]..|[\360-\367]...)+[}]
/* Stuff that goes inside " ", including UTF8 \uHHHH sequences */
<QTD>([\300-\337].|[\340-\357]..|[\360-\367]...|[\001-\041]|[\043-\177])+/[\042] {
- decode_quoted(yytext);
+ decode_quoted(yytext);
yylval_param->net = fsm_symbol(yytext);
BEGIN(QTDEND);
return NET;
@@ -292,14 +292,14 @@ BRACED [{]([^}]|[\300-\337].|[\340-\357]..|[\360-\367]...)+[}]
/* Sigma */
[\316][\243] {
- yylval_param->net = fsm_identity();
+ yylval_param->net = fsm_identity();
return NET;
}
(_S\() { return SUCCESSOR_OF; }
(_isunambiguous\() { return ISUNAMBIGUOUS; }
-(_isidentity\() { return ISIDENTITY; }
+(_isidentity\() { return ISIDENTITY; }
(_isfunctional\() { return ISFUNCTIONAL; }
(_notid\() { return NOTID; }
(_lm\() { return LETTERMACHINE; }
@@ -448,7 +448,7 @@ BRACED [{]([^}]|[\300-\337].|[\340-\357]..|[\360-\367]...)+[}]
[\342][\211][\240] { return NEQ; }
(\() { return LPAREN; }
(\)) { return RPAREN; }
-(;m) { return ENDM; }
+(;m) { return ENDM; }
(;d) { return ENDD; }
(;) { return END; }
(\||[\342][\210][\250]|[\342][\210][\252]) { return UNION; }
diff --git a/back-ends/foma/regex.y b/back-ends/foma/regex.y
index 0e65af9..e86eea6 100644
--- a/back-ends/foma/regex.y
+++ b/back-ends/foma/regex.y
@@ -324,7 +324,7 @@ network6: network7 { }
| network6 IGNORE_INTERNAL network7 { $$ = fsm_ignore($1,$3, OP_IGNORE_INTERNAL); }
| network6 RIGHT_QUOTIENT network7 { $$ = fsm_quotient_right($1,$3); }
| network6 LEFT_QUOTIENT network7 { $$ = fsm_quotient_left($1,$3); }
-| network6 INTERLEAVE_QUOTIENT network7 { $$ = fsm_quotient_interleave($1,$3); }
+| network6 INTERLEAVE_QUOTIENT network7 { $$ = fsm_quotient_interleave($1,$3); }
network7: network8 { }
| COMPLEMENT network7 { $$ = fsm_complement($2); }
@@ -392,17 +392,17 @@ network12: fend { $$ = $1; } |
fstart: FUNCTION network COMMA
{ frec++; fargptr[frec] = 0 ;declare_function_name($1) ; add_function_argument($2); }
-| FUNCTION network
+| FUNCTION network
{ frec++; fargptr[frec] = 0 ;declare_function_name($1) ; add_function_argument($2); }
fmid: fstart network COMMA { add_function_argument($2); }
| fmid network COMMA { add_function_argument($2); }
-fend: fmid network RPAREN
+fend: fmid network RPAREN
{ add_function_argument($2); if (($$ = function_apply(defined_nets, defined_funcs)) == NULL) YYERROR; }
-| fstart network RPAREN
+| fstart network RPAREN
{ add_function_argument($2); if (($$ = function_apply(defined_nets, defined_funcs)) == NULL) YYERROR; }
-| fstart RPAREN
+| fstart RPAREN
{ if (($$ = function_apply(defined_nets, defined_funcs)) == NULL) YYERROR;}
%%
diff --git a/back-ends/foma/rewrite.c b/back-ends/foma/rewrite.c
index a4304d8..8ac140a 100644
--- a/back-ends/foma/rewrite.c
+++ b/back-ends/foma/rewrite.c
@@ -1,621 +1,525 @@
-/* Foma: a finite-state toolkit and library. */
-/* Copyright © 2008-2012 Mans Hulden */
+/* Foma: a finite-state toolkit and library. */
+/* Copyright © 2008-2015 Mans Hulden */
-/* This file is part of foma. */
+/* This file is part of foma. */
-/* Foma is free software: you can redistribute it and/or modify */
-/* it under the terms of the GNU General Public License version 2 as */
-/* published by the Free Software Foundation. */
+/* Licensed under the Apache License, Version 2.0 (the "License"); */
+/* you may not use this file except in compliance with the License. */
+/* You may obtain a copy of the License at */
-/* Foma is distributed in the hope that it will be useful, */
-/* but WITHOUT ANY WARRANTY; without even the implied warranty of */
-/* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the */
-/* GNU General Public License for more details. */
+/* http://www.apache.org/licenses/LICENSE-2.0 */
-/* You should have received a copy of the GNU General Public License */
-/* along with foma. If not, see <http://www.gnu.org/licenses/>. */
+/* Unless required by applicable law or agreed to in writing, software */
+/* distributed under the License is distributed on an "AS IS" BASIS, */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */
+/* See the License for the specific language governing permissions and */
+/* limitations under the License. */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "foma.h"
-/*
- * Short version:
-
- * 1. We construct a language with centers interspersed with arbitrary single-symbol
- * identity relations (Insert). Centers are A:B cross products (for a rule A -> B)
- * 2. We constrain the centers to occur only in the proper contexts. (Context)
- * 3. We constrain the centers to have to occur if there is a proper context (Coerce)
- * 4. We add a few Additional constraints if the rules is longest-match, dotted, etc.
- * 5. We create [Insert & Context & Coerce & Additional] which is the collection
- of our rules.
- * 6. Of course, if we have multiple parallel rules and contexts, we intersect
- all those in the above Context, Coerce, Additional...
- * 7. Voilá
-
- * Longer version (how it actually works):
-
- * The actual encoding of rewrite rules is as follows:
- * 1. An automaton is interpreted as a transducer with sections marked inside brackets
- * [...<...>...] as a transduction, and outside sections as identity relations.
- * For example the string: "ab[a>b<a>Z]ab"
- * represents a transducer a:a b:b a:b a:0 a:a b:b
- * where Z is a "hard zero"
- *
- * So [ signifies : start of rewrite rule center
- * > signifies : the following symbol is on the lower side only
- * < signifies : the following symbol is on the upper side only
- * ] signifies : end of rewrite rule center
- * Any symbol not preceded by [,<,> is an Id symbol
- *
- * 2. A rewriting transducer is constructed as follows:
- * a. call *rewrite_cp() to build a center A:B string
- * e.g. A = abc , B = de
- * becomes CP = a>d<b>e<c>0
- * if the relation is not equal length, the shorter string
- * is padded with a special hard zero symbol
- *
- * b. construct the language Insert = @#@ NoSpecial* /[ %[ CP %] ] @#@
-* NoSpecial being any single non-auxiliary symbol
- * i.e. identity relations interspersed with cross-products
- * and where every string begins and ends with the special symbol @#@
- * Now we have strings such as aaa[a>0]a[a>b]va
- * Where the bracketed sequences are rewrite centers
- * Naturally, for many parallel rules, we take the union of *all* rule
- * cross-products in CP.
- * Two additions:
- * i) if the rule is dotted, e.g. [..] -> b
- * we get stuff like "xxx[[0>b]]xxx" ...
- * with two opening and closing brackets
- * ii) if the rule is a chunking rule A -> B ...C
- * we get stuff like "xxx[0>BAAA0>C]xxx" where the A's are Id(a)
- *
- * c. constrain the bracketed sequences as per the type of rewrite rule
- * Replace = [Insert & Context & Coerce]
- * Context is a context restriction statement on the legitimacy of the [ ] sequence
- * Coerce comes in two varieties depending on the rewrite arrow:
- * CoerceU forces every proper context to have a rewrite ->
- * CoerceL is the <- equivalent (CoerceU&CoerceL encodes <->)
- * - LR is a left-to-right constraint, and LM is a longest-match constraint
- * - SM is shortest match. These are added as intersections in Replace
- * depending on the rule type.
- * - Optional rules have no "Coerce"
- * - Dotted rules [.A.] -> B are separated into the empty part
- * A&0 and the non empty A-0 and are treated as separate parallel rules
- * the empty part has a separate Coerce & Context statement
- * restricting and coercing the rule center to occur in the proper
- * context if also flanked by i) and Id or ii) some symbol on the
- * upper side (i.e. not [[ which indicates dotted centers
- *
- * d. Construct a transducer from the bracketed language by calling
- * rewrite_cp_to_fst() which converts single-tape languages with strings
- * such as a[c>d]e to a:a c:d e:e and removes the auxiliaries.
- * We now remove the @#@ symbols as well
- *
- *
- * More detail:
-
- * Insert = @#@ NoSpecial/[ %[ UnionCP %] ] @#@
- * i.e. as described above. UnionCP = C1|C2|C3...
- * cross products where C1 = [A:B] for a rule A op B
-
- * Context = [ => Dir(L) _ C1 Dir(R) , Dir(L2) _ C1 Dir(R2) , ...
- * i.e. the opening bracket for a rule center [ is only allowed
- * if there is a legitimate center followed by a legitimate right context
- * to the right, and a legimate left context to the left.
- * Since contexts may be one-sided || or \/ etc. we use the function Dir(L)
- * to ignore either the upper or the lower side of the context, depending
- * on the context pair's directionality
-
- * Coerce = ~[[?* -[?* %>]] Dir(L) A [Dir(R) ?* & OSR ?*] ?*]
- * OSR = NoSpecial* %[ | @#@
- * i.e. Coerce is the language that does not contain a upper-side center A
- * occurring outside brackets (that's what OSR is for) if the contexts
- * are correct. If the rule arrow is <-, replace A with B, yielding
- * a coercion on the lower part of the A -> B.
- * the [?* -[?* %>]] part could be replaced with ?* but at a huge
- * cost of nondeterminism, so it's really an efficiency tweak.
- * For coerce, we take the intersection of every center
- * and context pair (to which the center applies).
-
- * For the empty center part of dotted rules, we use: CoerceD =
- * ~[[EndOutside & [?* [Id|Sigl]] & [?* Dir(L)] ] [ [Dir(R) ?*] & [Id|Sigr] ?*]]
- * where Sigl = \%] %];
- * and Sigr = %[ \%[;
- * meaning the language that never contains a L R sequence
- * if both L and R are identity symbols or regular [ rules
- * ruling out sequences such as x [a>0]
- * ^
- * if x is the left context and a the right one.
-
- * LR = ~[[[EndOutside] & [?* Dir(L)]] [[Dir(C) - [%[ ?*]] & $%[ ] Dir(R) ?*]
- * This is our left-to-right restriction. EndOutside = ~[?* %[ \%]*] - [?* %>]
- * (this again is a tweak: endoutside could be defined without the subtraction).
- * The logic of LR is simple: we shouldn't gave a Center inside C ... [ ] sequence
- * where C extends inside the bracket: meaning, we could have started rewriting
- * earlier.
-
- * LM = ~$[Dir(L) [ %[ [ [ Upper(C) - [%[ ?* | ?* %] ] ] & $[ %] Upper(NoSpecial) ] ] Dir(R)];
- * Longest-matched is simply the language where we don't find a L [ C R sequence
- * where the C contains inside it a ], indicating we could have chosen a longer
- * rewrite.
-
- * SM = ~$[Dir(L) %[ C/Low1 [NoSpecial/Low1 ?* & Dir(R) ?*]]
- * Here Low1 is the language we use to ignore the lower side inside [ ] brackets
- * The logic is this: We don't allow a L [ C R sequence where the C doesn't end
- * in ], i.e. is followed by at least one symbol which is not ]
- * (indicating we could have chosen a longer rewrite).
-
- * Additional notes: we have a special function to perform the context restriction
- * involved in constructing Context for two reasons:
- * 1) We want to maintain the @#@ symbols and not treat them as special which
- * the generic context_restrict doesn't do.
- * 2) We need some efficiency tweaks, such as are done in Coerce to
- * minimize nondeterminism.
-
- * If we have dotted rules we extend Context with [ => [ _ also.
- */
-
-
-struct fsm *rewrite_nospecial();
-struct fsm *rewr_context_restrict(struct fsm *X, struct fsmcontexts *LR);
-struct fsm *rewrite_pad(struct fsm *net);
-
-void rewrite_add_special_syms(struct fsm *net) {
- if (net == NULL)
- return;
- sigma_substitute(".#.", "@#@", net->sigma); /* We convert boundaries to our interal rep. */
- /* This is because sigma merging is handled */
- /* in a special way for .#., which we don't want. */
- if (sigma_find("@#@", net->sigma) == -1)
- sigma_add("@#@",net->sigma);
-
- sigma_add("@Z@",net->sigma);
- sigma_add("@<@",net->sigma);
- sigma_add("@>@",net->sigma);
- sigma_add("@]@",net->sigma);
- sigma_add("@[@",net->sigma);
- sigma_sort(net);
-}
+// Lower(X) puts X on output tape (may also be represented by @ID@ on input tape)
+// Upper(X) puts X on input tape
+// Unrewritten(X) X on input tape, not rewritten (aligned with @O@ symbols)
+// NotContain(X) MT does not contain MT configuration X
-struct fsm *rewr_igupp(struct fsm *net) {
- struct fsm *uppsym, *NoSpecial;
- NoSpecial = rewrite_nospecial();
-
- uppsym = fsm_minimize(fsm_kleene_star(fsm_union(fsm_symbol("@Z@"),fsm_union(fsm_symbol("@>@"),fsm_union(fsm_symbol("@]@"),fsm_union(fsm_concat(fsm_symbol("@<@"),fsm_copy(NoSpecial)),fsm_union(fsm_concat(fsm_kleene_plus(fsm_symbol("@[@")),fsm_copy(NoSpecial)),fsm_union(fsm_concat(fsm_symbol("@<@"),fsm_symbol("@Z@")),fsm_concat(fsm_kleene_plus(fsm_symbol("@]@")),fsm_symbol("@Z@"))))))))));
- fsm_destroy(NoSpecial);
- return(fsm_ignore(net,uppsym,OP_IGNORE_ALL));
-}
-
-struct fsm *rewr_iglow(struct fsm *net) {
- struct fsm *lowsym, *NoSpecial;
- NoSpecial = rewrite_nospecial();
-
- lowsym = fsm_minimize(fsm_kleene_star(fsm_union(fsm_symbol("@Z@"),fsm_union(fsm_symbol("@<@"),fsm_union(fsm_kleene_star(fsm_symbol("@[@")),fsm_union(fsm_kleene_star(fsm_symbol("@]@")),fsm_union(fsm_concat(fsm_symbol("@>@"),NoSpecial),fsm_concat(fsm_symbol("@>@"),fsm_symbol("@Z@")))))))));
-
- return(fsm_ignore(net,lowsym,OP_IGNORE_ALL));
-}
+// Boundary: every MT word begins and ends with boundary, i.e. the @#@ symbol on the input tape, output tape, and relevant semantic symbols
-struct fsm *rewr_iglow_inside(struct fsm *net) {
- struct fsm *lowsym, *NoSpecial;
- NoSpecial = rewrite_nospecial();
+/*
- lowsym = fsm_minimize(fsm_union(fsm_symbol("@Z@"),fsm_union(fsm_symbol("@<@"),fsm_union(fsm_concat(fsm_symbol("@>@"),NoSpecial),fsm_concat(fsm_symbol("@>@"),fsm_symbol("@Z@"))))));
-
- return(fsm_ignore(net,lowsym,OP_IGNORE_ALL));
-}
+ [ @O@ ] [ @I[@ ] [ @I@ ] [ @I]@ ] [ @I[]@ ]
+ [ @0@ ] [ @#0001@ ] [ @#0001@ ] [ @#0001@ ] [ @#0001@ ]
+ [ @#@ ] [ ANY|@0@ ] [ ANY|@0@ ] [ ANY|@0@ ] [ ANY|@0@ ]
+ [ @ID@ ] [ ANY|@ID|@0@ ] [ ANY|@ID|@0@ ] [ ANY|@ID|@0@ ] [ ANY|@ID|@0@ ]
+
+
+*/
+/* Special symbols used:
+ @0@ Epsilon
+ @O@ Outside rewrite
+ @I@ Inside rewrite
+ @I[@ Beginning of rewrite
+ @I[]@ Beginning and end of rewrite
+ @I]@ End of rewrite
+ @ID@ Identity symbol (= repeat symbol on previous tape at this position)
+ @#@ Boundary (the symbol .#. is mapped to this in contexts before the compilation procedure)
+ @#X@ X = rule number (one for each rule, starting with @#0001@)
+*/
+
+struct rewrite_batch {
+
+ struct rewrite_set *rewrite_set;
+ struct fsm *Rulenames;
+ struct fsm *ISyms;
+ struct fsm *ANY;
+ struct fsm *IOpen;
+ struct fsm *IClose;
+ struct fsm *ITape;
+ struct fsm *Any4Tape;
+ struct fsm *Epextend;
+ int num_rules;
+ char (*namestrings)[8];
+
+};
+
+char *specialsymbols[] = {"@0@","@O@","@I@","@I[@","@I[]@","@I]@","@ID@","@#@", NULL};
+
+void rewrite_add_special_syms(struct rewrite_batch *rb, struct fsm *net);
+struct fsm *rewrite_upper(struct rewrite_batch *rb, struct fsm *upper);
+struct fsm *rewrite_lower(struct rewrite_batch *rb, struct fsm *lower);
+struct fsm *rewrite_two_level(struct rewrite_batch *rb, struct fsm *lang, int rightside);
+struct fsm *rewr_context_restrict(struct rewrite_batch *rb, struct fsm *X, struct fsmcontexts *LR);
+struct fsm *rewr_contains(struct rewrite_batch *rb, struct fsm *lang);
+struct fsm *rewr_unrewritten(struct rewrite_batch *rb, struct fsm *lang);
+struct fsm *rewr_notleftmost(struct rewrite_batch *rb, struct fsm *lang, int rule_number, int arrow_type);
+struct fsm *rewr_notshortest(struct rewrite_batch *rb, struct fsm *lang, int rule_number);
+struct fsm *rewr_notlongest(struct rewrite_batch *rb, struct fsm *lang, int rule_number, int arrow_type);
+struct fsm *rewrite_tape_m_to_n_of_k(struct fsm *lang, int m, int n, int k);
+struct fsm *rewrite_cp(struct rewrite_batch *rb, struct fsm *upper, struct fsm *lower, int rule_number);
+struct fsm *rewrite_cp_transducer(struct rewrite_batch *rb, struct fsm *t, int rule_number);
+struct fsm *rewrite_cp_markup(struct rewrite_batch *rb, struct fsm *upper, struct fsm *lower1, struct fsm *lower2, int rule_number);
+struct fsm *rewrite_epextend(struct rewrite_batch *rb);
+struct fsm *rewrite_any_4tape(struct rewrite_batch *rb);
+struct fsm *rewrite_itape(struct rewrite_batch *rb);
+void rewrite_cleanup(struct rewrite_batch *rb);
-struct fsm *rewrite_nospecial() {
- struct fsm *net;
- net = fsm_create("");
- net->states = xxmalloc(sizeof(struct fsm_state)*3);
- sigma_add_special(IDENTITY,net->sigma);
- sigma_add("@Z@",net->sigma);
- sigma_add("@#@",net->sigma);
- sigma_add("@<@",net->sigma);
- sigma_add("@>@",net->sigma);
- sigma_add("@]@",net->sigma);
- sigma_add("@[@",net->sigma);
- add_fsm_arc(net->states,0,0,IDENTITY,IDENTITY,1,0,1);
- add_fsm_arc(net->states,1,1,-1,-1,-1,1,0);
- add_fsm_arc(net->states,2,-1,-1,-1,-1,-1,-1);
- sigma_sort(net);
- return(net);
-}
struct fsm *fsm_rewrite(struct rewrite_set *all_rules) {
+ struct rewrite_batch *rb;
struct rewrite_set *ruleset;
struct fsmrules *rules;
- struct fsmcontexts *contexts, *allcontexts, *newcontext;
- struct fsm *UnionCP, *RuleCP, *UnionCPI, *Insert, *NoSpecial, *Context, *ContextD, *Result, *Coerce, *CoerceLR, *CoerceLM, *CoerceSM, *thisCoerce, *SigL, *SigR, *Id, *Outside, *EndOutside, *CoerceCenter = NULL;
- int dir, c, minimal_old, dottedrules, num_parallel_rules;
+ struct fsmcontexts *contexts;
+ struct fsm *RuleCP, *Base, *Boundary, *Outside, *CP, *C, *LeftExtend, *RightExtend, *Center;
+ int i, num_rules, rule_number, dir;
+ /* Count parallel rules */
+ for (ruleset = all_rules, num_rules = 0; ruleset != NULL; ruleset = ruleset->next) {
+ for (rules = ruleset->rewrite_rules; rules != NULL; rules = rules->next) {
+ num_rules++;
+ }
+ }
- extern int g_minimal;
+ rb = xxcalloc(1, sizeof(struct rewrite_batch));
+ rb->rewrite_set = all_rules;
+ rb->num_rules = num_rules;
+ rb->namestrings = xxmalloc(sizeof rb->namestrings * num_rules);
+ for (i = 0; i < rb->num_rules; i++) {
+ sprintf(rb->namestrings[i], "@#%04i@", i+1);
+ }
- dottedrules = 0;
+ rb->ISyms = fsm_minimize(fsm_union(fsm_symbol("@I@"), fsm_union(fsm_symbol("@I[]@"), fsm_union(fsm_symbol("@I[@"), fsm_symbol("@I]@")))));
+ rb->Rulenames = fsm_empty_set();
+ for (i = 1; i <= num_rules; i++) {
+ rb->Rulenames = fsm_minimize(fsm_union(rb->Rulenames, fsm_symbol(rb->namestrings[i-1])));
+ }
+ rb->ANY = fsm_identity();
+ rewrite_add_special_syms(rb, rb->ANY);
- /* Preprocess by adding all special symbols to sigmas */
- num_parallel_rules = 0;
+ /* Add auxiliary symbols to all alphabets */
for (ruleset = all_rules; ruleset != NULL; ruleset = ruleset->next) {
- num_parallel_rules++;
for (rules = ruleset->rewrite_rules; rules != NULL; rules = rules->next) {
- rewrite_add_special_syms(rules->left);
- rewrite_add_special_syms(rules->right);
- rewrite_add_special_syms(rules->right2);
+ rewrite_add_special_syms(rb, rules->left);
+ rewrite_add_special_syms(rb, rules->right);
+ rewrite_add_special_syms(rb, rules->right2);
}
for (contexts = ruleset->rewrite_contexts; contexts != NULL; contexts = contexts->next) {
- rewrite_add_special_syms(contexts->left);
- rewrite_add_special_syms(contexts->right);
+ rewrite_add_special_syms(rb, contexts->left);
+ rewrite_add_special_syms(rb, contexts->right);
}
}
- /* Do the CP for all rules and store it in the set */
- /* Do the union of all rules */
- UnionCP = fsm_empty_set();
- //printf("Initial CP and Ignore\n"); fflush(stdout);
-
- NoSpecial = rewrite_nospecial();
- Id = fsm_minimize(fsm_union(fsm_copy(NoSpecial),fsm_symbol("@#@")));
-
- for (ruleset = all_rules; ruleset != NULL; ruleset = ruleset->next) {
+ /* Get cross-product of every rule, according to its type */
+ RuleCP = fsm_empty_set();
+ for (ruleset = all_rules, rule_number = 1; ruleset != NULL; ruleset = ruleset->next) {
+ dir = ruleset->rule_direction;
for (rules = ruleset->rewrite_rules; rules != NULL; rules = rules->next) {
- /* insert transducer instead of cross-product */
- if (rules->right == NULL) {
- /* Make rules->left upper side of T */
- /* Convert transducer to single-tape format */
- rules->cross_product = rewrite_pad(fsm_flatten(fsm_copy(rules->left), fsm_symbol("@Z@")));
+ if (rules->right == NULL) {
+ /* T(x)-type rule */
+ CP = rewrite_cp_transducer(rb, fsm_copy(rules->left), rule_number);
+ rules->cross_product = fsm_copy(CP);
rules->right = fsm_minimize(fsm_lower(fsm_copy(rules->left)));
- rules->left = fsm_minimize(fsm_upper(rules->left));
- rewrite_add_special_syms(rules->right);
- }
- else if (rules->right2 == NULL) {
+ rules->left = fsm_minimize(fsm_upper(fsm_copy(rules->left)));
+ rewrite_add_special_syms(rb, rules->right);
+ rewrite_add_special_syms(rb, rules->left);
+ } else if (rules->right2 == NULL) {
/* Regular rewrite rule */
- rules->cross_product = rewrite_cp(fsm_copy(rules->left),fsm_copy(rules->right));
- } else {
- /* A -> B ... C type rule */
- /* 0>BA0>C */
- rules->cross_product = fsm_minimize(fsm_concat(rewrite_cp(fsm_empty_string(),fsm_copy(rules->right)),fsm_concat(fsm_copy(rules->left),rewrite_cp(fsm_empty_string(),fsm_copy(rules->right2)))));
- }
- if ((rules->arrow_type & ARROW_DOTTED) != 0) {
- dottedrules++;
- //printf("RET\n");
- rules->cross_product = fsm_minimize(fsm_concat(fsm_symbol("@[@"),fsm_concat(rules->cross_product,fsm_symbol("@]@"))));
- }
- UnionCP = fsm_minimize(fsm_union(fsm_copy(rules->cross_product),UnionCP));
- }
+ CP = rewrite_cp(rb, fsm_copy(rules->left), fsm_copy(rules->right), rule_number);
+ rules->cross_product = fsm_copy(CP);
+ } else {
+ /* A -> B ... C -type rule */
+ CP = rewrite_cp_markup(rb, fsm_copy(rules->left), fsm_copy(rules->right), fsm_copy(rules->right2), rule_number);
+ rules->cross_product = fsm_copy(CP);
+ }
+ RuleCP = fsm_minimize(fsm_union(RuleCP, CP));
+ rule_number++;
+ }
+ }
- /* Transform every context to ignore Upper/Lower */
- dir = ruleset->rule_direction;
- for (c = 0, contexts = ruleset->rewrite_contexts; contexts != NULL; contexts = contexts->next) {
- c = 1;
- if (dir == OP_UPWARD_REPLACE) {
- contexts->cpleft = rewr_iglow(fsm_copy(contexts->left));
- contexts->cpright = rewr_iglow(fsm_copy(contexts->right));
- }
- if (dir == OP_DOWNWARD_REPLACE) {
- contexts->cpleft = rewr_igupp(fsm_copy(contexts->left));
- contexts->cpright = rewr_igupp(fsm_copy(contexts->right));
- }
- if (dir == OP_LEFTWARD_REPLACE) {
- contexts->cpleft = rewr_iglow(fsm_copy(contexts->left));
- contexts->cpright = rewr_igupp(fsm_copy(contexts->right));
- }
- if (dir == OP_RIGHTWARD_REPLACE) {
- contexts->cpleft = rewr_igupp(fsm_copy(contexts->left));
- contexts->cpright = rewr_iglow(fsm_copy(contexts->right));
- }
- }
- if (c == 0) {
- ruleset->rewrite_contexts = xxcalloc(1,sizeof(struct fsmcontexts));
- ruleset->rewrite_contexts->cpleft = fsm_empty_string();
- ruleset->rewrite_contexts->cpright = fsm_empty_string();
- ruleset->rewrite_contexts->next = NULL;
- }
+ /* Create Base language */
+ Boundary = fsm_parse_regex("\"@O@\" \"@0@\" \"@#@\" \"@ID@\"", NULL, NULL);
+ Outside = fsm_minimize(fsm_concat(fsm_symbol("@O@"), fsm_concat(fsm_symbol("@0@"), fsm_concat(fsm_copy(rb->ANY), fsm_symbol("@ID@")))));
+ Base = fsm_minimize(fsm_concat(fsm_copy(Boundary), fsm_concat(fsm_kleene_star(fsm_union(RuleCP, Outside)), fsm_copy(Boundary))));
+ fsm_destroy(Boundary);
+ for (ruleset = all_rules, rule_number = 1; ruleset != NULL; ruleset = ruleset->next) {
+ dir = ruleset->rule_direction;
+ /* Replace all context spec with Upper/Lower, depending on rule_direction */
+ for (contexts = ruleset->rewrite_contexts; contexts != NULL; contexts = contexts->next) {
+ switch(dir) {
+ case OP_UPWARD_REPLACE:
+ contexts->cpleft = rewrite_upper(rb, fsm_copy(contexts->left));
+ contexts->cpright = rewrite_upper(rb, fsm_copy(contexts->right));
+ break;
+ case OP_RIGHTWARD_REPLACE:
+ contexts->cpleft = rewrite_lower(rb, fsm_copy(contexts->left));
+ contexts->cpright = rewrite_upper(rb, fsm_copy(contexts->right));
+ break;
+ case OP_LEFTWARD_REPLACE:
+ contexts->cpleft = rewrite_upper(rb, fsm_copy(contexts->left));
+ contexts->cpright = rewrite_lower(rb, fsm_copy(contexts->right));
+ break;
+ case OP_DOWNWARD_REPLACE:
+ contexts->cpleft = rewrite_lower(rb, fsm_copy(contexts->left));
+ contexts->cpright = rewrite_lower(rb, fsm_copy(contexts->right));
+ break;
+ case OP_TWO_LEVEL_REPLACE:
+ contexts->cpleft = rewrite_two_level(rb, fsm_copy(contexts->left), 0);
+ contexts->cpright = rewrite_two_level(rb, fsm_copy(contexts->right), 1);
+ break;
+ }
+ }
+ for (rules = ruleset->rewrite_rules; rules != NULL; rules = rules->next) {
+ /* Just the rule center w/ number without CP() contests */
+ /* Actually, maybe better to include CP(U,L) in this, very slow with e.g. a -> a || _ b^15 */
+ if (rules->arrow_type & ARROW_DOTTED) {
+ /* define EP Tape1of4("@O@") | [ Tape1of4("@I[@" "@I@"* "@I]@" | "@I[]@") & Tape3of4(~["@0@"*]) ] ; */
+ /* Additional constraint: 0->x is only allowed between EP _ EP */
+ /* The left and right sides can be checked separately */
+ /* ~[?* Center ~[EP ?*]] & ~[~[?* EP] Center ?*] */
+ Center = fsm_copy(rules->cross_product);
+ Base = fsm_intersect(fsm_intersect(Base, fsm_complement(fsm_concat(rewrite_any_4tape(rb), fsm_concat(fsm_copy(Center), fsm_complement(fsm_concat(rewrite_epextend(rb), rewrite_any_4tape(rb))))))), fsm_complement(fsm_concat(fsm_complement(fsm_concat(rewrite_any_4tape(rb), rewrite_epextend(rb))), fsm_concat(fsm_copy(Center), rewrite_any_4tape(rb)))));
+ }
+ if (ruleset->rewrite_contexts) {
+ Base = fsm_intersect(Base, rewr_context_restrict(rb, rules->cross_product, ruleset->rewrite_contexts));
+ }
+ /* Determine C (based on rule type) */
+ C = fsm_empty_set();
+ if ((rules->arrow_type & ARROW_RIGHT) && !(rules->arrow_type & ARROW_OPTIONAL)) {
+ C = fsm_union(C, rewr_unrewritten(rb, fsm_minimize(fsm_minus(fsm_copy(rules->left), fsm_empty_string()))));
+ }
+ if ((rules->arrow_type & ARROW_LEFT) && !(rules->arrow_type & ARROW_OPTIONAL)) {
+ C = fsm_union(C, rewr_unrewritten(rb, fsm_minimize(fsm_minus(fsm_copy(rules->right), fsm_empty_string()))));
+ }
+ if (rules->arrow_type & ARROW_LONGEST_MATCH) {
+ if (rules->arrow_type & ARROW_RIGHT) {
+ C = fsm_union(C, rewr_notleftmost(rb, rewrite_upper(rb, fsm_copy(rules->left)), rule_number, rules->arrow_type));
+ C = fsm_union(C, rewr_notlongest(rb, rewrite_upper(rb, fsm_copy(rules->left)), rule_number, rules->arrow_type));
+ }
+ if (rules->arrow_type & ARROW_LEFT) {
+ C = fsm_union(C, rewr_notleftmost(rb, rewrite_lower(rb, fsm_copy(rules->right)), rule_number, rules->arrow_type));
+ C = fsm_union(C, rewr_notlongest(rb, rewrite_lower(rb, fsm_copy(rules->right)), rule_number, rules->arrow_type));
+ }
+ }
+ if (rules->arrow_type & ARROW_SHORTEST_MATCH) {
+ if (rules->arrow_type & ARROW_RIGHT) {
+ C = fsm_union(C, rewr_notleftmost(rb, rewrite_upper(rb, fsm_copy(rules->left)), rule_number, rules->arrow_type));
+ C = fsm_union(C, rewr_notshortest(rb, rewrite_upper(rb, fsm_copy(rules->left)), rule_number));
+ }
+ if (rules->arrow_type & ARROW_LEFT) {
+ C = fsm_union(C, rewr_notleftmost(rb, rewrite_lower(rb, fsm_copy(rules->right)), rule_number, rules->arrow_type));
+ C = fsm_union(C, rewr_notshortest(rb, rewrite_lower(rb, fsm_copy(rules->right)), rule_number));
+ }
+ }
+ if (!ruleset->rewrite_contexts) {
+ if (rules->arrow_type & ARROW_DOTTED && !(rules->arrow_type & ARROW_OPTIONAL)) {
+ Base = fsm_minus(Base, rewr_contains(rb, fsm_concat(rewrite_epextend(rb), rewrite_epextend(rb))));
+ } else {
+ Base = fsm_minus(Base, rewr_contains(rb, fsm_copy(C)));
+ }
+ }
+ for (contexts = ruleset->rewrite_contexts; contexts != NULL; contexts = contexts->next) {
+ /* Constraints: running intersect w/ Base */
+ /* NotContain(LC [Unrewritten|LM|...] RC) */
+ if (rules->arrow_type & ARROW_DOTTED && !(rules->arrow_type & ARROW_OPTIONAL)) {
+ /* Extend left and right */
+ LeftExtend = fsm_minimize(fsm_intersect(fsm_concat(rewrite_any_4tape(rb), fsm_copy(contexts->cpleft)), fsm_concat(rewrite_any_4tape(rb), rewrite_epextend(rb))));
+ RightExtend = fsm_minimize(fsm_intersect(fsm_concat(rewrite_epextend(rb), rewrite_any_4tape(rb)), fsm_concat(fsm_copy(contexts->cpright), rewrite_any_4tape(rb))));
+ Base = fsm_minus(Base, rewr_contains(rb, fsm_minimize(fsm_concat(LeftExtend, RightExtend))));
+ } else {
+ Base = fsm_minus(Base, rewr_contains(rb, fsm_concat(fsm_copy(contexts->cpleft), fsm_concat(fsm_copy(C), fsm_copy(contexts->cpright)))));
+ }
+ }
+ rule_number++;
+ fsm_destroy(C);
+ }
}
+ Base = fsm_minimize(fsm_lower(fsm_compose(Base, fsm_parse_regex("[?:0]^4 [?:0 ?:0 ? ?]* [?:0]^4", NULL, NULL))));
+ Base = fsm_unflatten(Base, "@0@", "@ID@");
- /* UnionCP now holds _all_ cross products needed for Insert */
- /* Insert = .#. NoSpecial/[ %[ UnionCP %] ] .#. */
+ for (i = 0; specialsymbols[i] != NULL; i++) {
+ Base->sigma = sigma_remove(specialsymbols[i], Base->sigma);
+ }
+ for (rule_number = 1; rule_number <= num_rules; rule_number++)
+ Base->sigma = sigma_remove(rb->namestrings[rule_number-1], Base->sigma);
- UnionCPI = fsm_minimize(fsm_concat(fsm_symbol("@[@"),fsm_minimize(fsm_concat(fsm_copy(UnionCP),fsm_symbol("@]@")))));
+ fsm_compact(Base);
+ sigma_sort(Base);
+ rewrite_cleanup(rb);
+ return Base;
+}
- Insert = fsm_minimize(fsm_ignore(fsm_kleene_star(fsm_copy(NoSpecial)), UnionCPI, OP_IGNORE_ALL));
- Insert = fsm_minimize(fsm_concat(fsm_symbol("@#@"),fsm_minimize(fsm_concat(Insert,fsm_symbol("@#@")))));
-
- /* Context = [ => Lower(L) _ C1|...|Cn ] Upper(R) */
- //printf("RuleCP\n"); fflush(stdout);
- for (ruleset = all_rules; ruleset != NULL; ruleset = ruleset->next) {
- RuleCP = fsm_empty_set();
- for (rules = ruleset->rewrite_rules; rules != NULL; rules = rules->next) {
- RuleCP = fsm_minimize(fsm_union(RuleCP,fsm_copy(rules->cross_product)));
- fsm_destroy(rules->cross_product);
- }
- ruleset->cpunion = RuleCP; /* Store every rule's individual cross-product */
- }
+void rewrite_cleanup(struct rewrite_batch *rb) {
+
+ if (rb->Rulenames != NULL)
+ fsm_destroy(rb->Rulenames);
+ if (rb->ISyms != NULL)
+ fsm_destroy(rb->ISyms);
+ if (rb->ANY != NULL)
+ fsm_destroy(rb->ANY);
+ if (rb->IOpen != NULL)
+ fsm_destroy(rb->IOpen);
+ if (rb->IClose != NULL)
+ fsm_destroy(rb->IClose);
+ if (rb->ITape != NULL)
+ fsm_destroy(rb->ITape);
+ if (rb->Any4Tape != NULL)
+ fsm_destroy(rb->Any4Tape);
+ if (rb->Epextend != NULL)
+ fsm_destroy(rb->Epextend);
+ if (rb->namestrings != NULL)
+ xxfree(rb->namestrings);
+ xxfree(rb);
+ return;
+}
- allcontexts = NULL;
- /* Do Context */
- //printf("Context\n"); fflush(stdout);
- for (ruleset = all_rules; ruleset != NULL; ruleset = ruleset->next) {
- /* For every context pair, add its rule union to list of restricts */
- for (contexts = ruleset->rewrite_contexts; contexts != NULL; contexts = contexts->next) {
- newcontext = xxcalloc(1,sizeof(struct fsmcontexts));
- /* left = L */
- newcontext->left = fsm_copy(contexts->cpleft);
- /* right = Center ] R */
- newcontext->right = fsm_minimize(fsm_concat(fsm_copy(ruleset->cpunion),fsm_concat(fsm_symbol("@]@"),fsm_copy(contexts->cpright))));
- newcontext->next = allcontexts;
- allcontexts = newcontext;
- }
- fsm_destroy(ruleset->cpunion);
- ruleset->cpunion = RuleCP;
+struct fsm *rewr_notlongest(struct rewrite_batch *rb, struct fsm *lang, int rule_number, int arrow_type) {
+ /* define NotLongest(X) [Upper(X)/Lower(X) & Tape1of4(IOpen Tape1Sig* ["@O@" | IOpen] Tape1Sig*)] */
+ struct fsm *nl, *flt, *rulenum;
+ nl = fsm_parse_regex("[\"@I[@\"|\"@I[]@\"] [\"@I[@\"|\"@I[]@\"|\"@I]@\"|\"@I@\"|\"@O@\"]* [\"@O@\"|\"@I[@\"|\"@I[]@\"] [\"@I[@\"|\"@I[]@\"|\"@I]@\"|\"@I@\"|\"@O@\"]*", NULL, NULL);
+ nl = rewrite_tape_m_to_n_of_k(nl, 1, 1, 4);
+ rulenum = fsm_minimize(fsm_concat(fsm_identity(), fsm_concat(fsm_symbol(rb->namestrings[rule_number-1]), fsm_concat(fsm_identity(), fsm_concat(fsm_identity(), fsm_universal())))));
+ nl = fsm_intersect(nl, rulenum);
+ /* lang can't end in @0@ */
+ if (arrow_type & ARROW_RIGHT) {
+ flt = fsm_parse_regex("[? ? ? ?]* [? ? [?-\"@0@\"] ?]", NULL, NULL);
+ } else {
+ flt = fsm_parse_regex("[? ? ? ?]* [? ? ? [?-\"@0@\"]]", NULL, NULL);
}
- //printf("Doing context\n");
-
+ return fsm_minimize(fsm_intersect(fsm_intersect(nl, fsm_copy(lang)), flt));
+}
- /* Suspend minimization for Context */
- minimal_old = g_minimal;
- g_minimal = 0;
+struct fsm *rewr_notshortest(struct rewrite_batch *rb, struct fsm *lang, int rule_number) {
+ /* define NotShortest(X) [Upper/Lower(X) & Tape1of4("@I[@" \IClose*)] */
+ struct fsm *ns, *rulenum;
+ ns = fsm_parse_regex("[\"@I[@\"] \\[\"@I]@\"]*", NULL, NULL);
+ rulenum = fsm_minimize(fsm_concat(fsm_identity(), fsm_concat(fsm_symbol(rb->namestrings[rule_number-1]), fsm_concat(fsm_identity(), fsm_concat(fsm_identity(), fsm_universal())))));
+ ns = rewrite_tape_m_to_n_of_k(ns, 1, 1, 4);
+ ns = fsm_intersect(ns, rulenum);
+ return fsm_minimize(fsm_intersect(ns, fsm_copy(lang)));
+}
- if (allcontexts == NULL) {
- Context = fsm_universal();
+struct fsm *rewr_notleftmost(struct rewrite_batch *rb, struct fsm *lang, int rule_number, int arrow_type) {
+ struct fsm *nl, *flt, *rulenum;
+ /* define Leftmost(X) [Upper/Lower(X) & Tape1of4("@O@" Tape1Sig* IOpen Tape1Sig*) ] */
+ nl = fsm_parse_regex("\"@O@\" [\"@O@\"]* [\"@I[@\"|\"@I[]@\"] [\"@I[@\"|\"@I[]@\"|\"@I]@\"|\"@I@\"|\"@O@\"]*", NULL, NULL);
+ nl = rewrite_tape_m_to_n_of_k(nl, 1, 1, 4);
+ rulenum = fsm_minimize(fsm_concat(fsm_concat(fsm_symbol("@O@"), fsm_concat(fsm_identity(), fsm_concat(fsm_identity(), fsm_identity()))), fsm_concat(fsm_kleene_star(fsm_concat(fsm_symbol("@O@"), fsm_concat(fsm_identity(), fsm_concat(fsm_identity(), fsm_identity())))), fsm_concat(fsm_union(fsm_symbol("@I[@"), fsm_symbol("@I[]@")), fsm_concat(fsm_symbol(rb->namestrings[rule_number-1]), fsm_universal())))));
+ nl = fsm_intersect(nl, rulenum);
+ if (arrow_type & ARROW_RIGHT) {
+ flt = fsm_parse_regex("[? ? ? ?]* [? ? [?-\"@0@\"] ?]", NULL, NULL);
} else {
- if (dottedrules == 0) {
- Context = rewr_context_restrict(fsm_symbol("@[@"),allcontexts);
- } else {
- //printf("Doing dottedcontext\n");
-
- newcontext = xxcalloc(1,sizeof(struct fsmcontexts));
- newcontext->left = fsm_symbol("@[@");
- newcontext->right = fsm_empty_string();
- newcontext->next = allcontexts;
- allcontexts = newcontext;
- Context = rewr_context_restrict(fsm_symbol("@[@"),allcontexts);
-
- newcontext = xxcalloc(1,sizeof(struct fsmcontexts));
- newcontext->left = fsm_minimize(fsm_union(fsm_copy(Id),fsm_concat(fsm_term_negation(fsm_symbol("@]@")),fsm_symbol("@]@"))));
- newcontext->right = fsm_empty_string();
- newcontext->next = NULL;
-
- ContextD = fsm_minimize(fsm_context_restrict(fsm_minimize(fsm_concat(fsm_symbol("@[@"),fsm_symbol("@[@"))), newcontext));
- sigma_remove("@#@",ContextD->sigma);
- Context = fsm_intersect(Context,ContextD);
- fsm_destroy(newcontext->left);
- fsm_destroy(newcontext->right);
- xxfree(newcontext);
- }
+ flt = fsm_parse_regex("[? ? ? ?]* [? ? ? [?-\"@0@\"]]", NULL, NULL);
}
+ return fsm_minimize(fsm_intersect(fsm_intersect(nl, fsm_copy(lang)), flt));
+}
- g_minimal = minimal_old;
-
- /* For all L C R combos */
- /* ~[?* - [?* @>@] Dir(L) A [Dir(R) ?* & Outside ?*] ?*] for -> */
- /* ~[?* - [?* @>@] Dir(L) B [Dir(R) ?* & Outside ?*] ?*] for <- */
-
- /* Outside = [NoSpecial* [%[|%#]]; */
- Outside = fsm_minimize(fsm_concat(fsm_kleene_star(fsm_copy(NoSpecial)),fsm_union(fsm_symbol("@[@"),fsm_symbol("@#@"))));
- /* EndOutside = ~[?* %[ \%]*] - [?* %>] */
- EndOutside = fsm_complement(fsm_concat(fsm_universal(),fsm_concat(fsm_symbol("@[@"),fsm_kleene_star(fsm_term_negation(fsm_symbol("@]@"))))));
- EndOutside = fsm_minus(EndOutside, fsm_concat(fsm_universal(),fsm_symbol("@>@")));
-
- Coerce = NULL;
- Coerce = fsm_copy(Insert);
+struct fsm *rewr_unrewritten(struct rewrite_batch *rb, struct fsm *lang) {
+ /* define Unrewritten(X) [X .o. [0:"@O@" 0:"@0@" ? 0:"@ID@"]*].l; */
+ struct fsm *C;
+ C = fsm_minimize(fsm_kleene_star(fsm_concat(fsm_cross_product(fsm_empty_string(), fsm_symbol("@O@")), fsm_concat(fsm_cross_product(fsm_empty_string(), fsm_symbol("@0@")), fsm_concat(fsm_copy(rb->ANY), fsm_cross_product(fsm_empty_string(), fsm_symbol("@ID@")))))));
+ return fsm_minimize(fsm_lower(fsm_compose(lang, C)));
+}
- for (ruleset = all_rules; ruleset != NULL; ruleset = ruleset->next) {
- for (rules = ruleset->rewrite_rules; rules != NULL; rules = rules->next) {
- for (contexts = ruleset->rewrite_contexts; contexts != NULL; contexts = contexts->next) {
-
- if ((rules->arrow_type & (ARROW_LEFT | ARROW_RIGHT)) == (ARROW_LEFT|ARROW_RIGHT) )
- CoerceCenter = fsm_union(fsm_copy(rules->left),fsm_copy(rules->right));
- else if ((rules->arrow_type & ARROW_RIGHT) != 0)
- CoerceCenter = fsm_copy(rules->left);
- else if ((rules->arrow_type & ARROW_LEFT) != 0) {
- CoerceCenter = fsm_copy(rules->right);
- }
- //printf("Coercing\n");
- /* Can't coerce empty string */
- CoerceCenter = fsm_minus(CoerceCenter, fsm_empty_string());
-
- /* ~[ [?* -[?* %>]] Upper(L) C [Upper(L) ?* & OSR ?*] ?*]; */
- /* TODO: Try replacing this with logic version for efficiency */
- if ((rules->arrow_type & ARROW_OPTIONAL) == 0) {
-
- if ((rules->arrow_type & ARROW_DOTTED) == 0) {
- thisCoerce = fsm_complement(fsm_concat(fsm_minus(fsm_universal(), fsm_concat(fsm_universal(), fsm_symbol("@>@"))),fsm_concat(fsm_copy(contexts->cpleft),fsm_concat(fsm_copy(CoerceCenter),fsm_concat(fsm_intersect(fsm_concat(fsm_copy(contexts->cpright),fsm_universal()),fsm_concat(fsm_copy(Outside),fsm_universal())),fsm_universal())))));
-
- } else {
- //printf("Coercing dotted\n");
- /* It's the empty part of a [..] dotted rule */
- SigL = fsm_concat(fsm_term_negation(fsm_symbol("@]@")),fsm_symbol("@]@"));
- SigR = fsm_concat(fsm_symbol("@[@"),fsm_term_negation(fsm_symbol("@[@")));
- thisCoerce = fsm_complement(fsm_concat(fsm_intersect(fsm_copy(EndOutside),fsm_intersect(fsm_concat(fsm_universal(),fsm_copy(contexts->cpleft)),fsm_concat(fsm_universal(),fsm_union(fsm_copy(Id),SigL)))),fsm_intersect(fsm_concat(fsm_union(fsm_copy(Id),SigR),fsm_universal()),fsm_concat(fsm_copy(contexts->cpright),fsm_universal()))));
- }
- }
- if ((rules->arrow_type & ARROW_OPTIONAL) != 0) {
- thisCoerce = fsm_universal();
- }
- if ((rules->arrow_type & ARROW_LEFT_TO_RIGHT) != 0) {
- /* LR ~[[[EndOutside] & [?* L/Upp2]] [ [A/Low2 - [%[ ?*] ] & $%[ ] R/Upp2 ?*] */
- //printf("Adding LR\n"); fflush(stdout);
- CoerceLR = fsm_complement(fsm_concat(fsm_intersect(fsm_copy(EndOutside),fsm_concat(fsm_universal(),fsm_copy(contexts->cpleft))),fsm_concat(fsm_intersect(fsm_minus(rewr_iglow(fsm_copy(CoerceCenter)), fsm_concat(fsm_symbol("@[@"),fsm_universal())),fsm_contains(fsm_concat(fsm_symbol("@[@"),rewr_iglow(fsm_copy(NoSpecial))))),fsm_concat(fsm_copy(contexts->cpright),fsm_universal()))));
-
- thisCoerce = fsm_intersect(thisCoerce, CoerceLR);
- }
- /* LM = ~$[ Dir(L) [ %[ [ A/Low2 - [ %[ ?* | ?* %] ] ] & $[ %] NoSpecial/Low2 ] ] Dir(R) ] */
- if ((rules->arrow_type & ARROW_LONGEST_MATCH) != 0) {
- //printf("Adding LM\n");
-
-
- /* For single rule (don't look at left context) */
- /* LM = ~$[ Dir(L) [ %[ [ A/Low2 - [ %[ ?* | ?* %] ] ] & $[ %] NoSpecial/Low2 ] ] Dir(R) ] */
-
- if (num_parallel_rules == 1)
- CoerceLM = fsm_complement(fsm_contains(fsm_concat(fsm_empty_string(),fsm_concat(fsm_concat(fsm_symbol("@[@"),fsm_intersect(fsm_minus(rewr_iglow(fsm_copy(CoerceCenter)),fsm_union(fsm_concat(fsm_symbol("@[@"),fsm_universal()),fsm_concat(fsm_universal(),fsm_symbol("@]@")))),fsm_contains(fsm_concat(fsm_symbol("@]@"),rewr_iglow(fsm_copy(NoSpecial)))))),fsm_copy(contexts->cpright)))));
- else
- CoerceLM = fsm_complement(fsm_contains(fsm_concat(fsm_copy(contexts->cpleft),fsm_concat(fsm_concat(fsm_symbol("@[@"),fsm_intersect(fsm_minus(rewr_iglow(fsm_copy(CoerceCenter)),fsm_union(fsm_concat(fsm_symbol("@[@"),fsm_universal()),fsm_concat(fsm_universal(),fsm_symbol("@]@")))),fsm_contains(fsm_concat(fsm_symbol("@]@"),rewr_iglow(fsm_copy(NoSpecial)))))),fsm_copy(contexts->cpright)))));
-
- thisCoerce = fsm_intersect(thisCoerce, CoerceLM);
- }
- if ((rules->arrow_type & ARROW_SHORTEST_MATCH) != 0) {
- /* ~$[L/Low2 %[ C/Low1 [NoSpecial/Low1 ?* & R/Low2 ?*]] */
-
- CoerceSM = fsm_complement(fsm_contains(fsm_concat(fsm_copy(contexts->cpleft),fsm_concat(fsm_symbol("@[@"),fsm_concat(rewr_iglow_inside(fsm_copy(CoerceCenter)),fsm_intersect(fsm_concat(rewr_iglow_inside(fsm_copy(NoSpecial)),fsm_universal()),fsm_concat(fsm_copy(contexts->cpright),fsm_universal())))))));
-
- thisCoerce = fsm_intersect(thisCoerce, CoerceSM);
- }
- if (Coerce != NULL) {
- Coerce = fsm_intersect(Coerce, thisCoerce);
- } else {
- Coerce = thisCoerce;
- }
- fsm_destroy(CoerceCenter);
- }
- }
+struct fsm *rewr_contains(struct rewrite_batch *rb, struct fsm *lang) {
+ /* define NotContain(X) ~[[Tape1Sig Tape2Sig Tape3Sig Tape4Sig]* X ?*]; */
+ return fsm_minimize(fsm_concat(rewrite_any_4tape(rb), fsm_concat(lang, rewrite_any_4tape(rb))));
+}
+
+struct fsm *rewrite_tape_m_to_n_of_k(struct fsm *lang, int m, int n, int k) {
+ /* [X .o. [0:?^(m-1) ?^(n-m+1) 0:?^(k-n)]*].l */
+ return fsm_minimize(fsm_lower(fsm_compose(lang, fsm_kleene_star(fsm_concat(fsm_concat_n(fsm_cross_product(fsm_empty_string(), fsm_identity()), m-1), fsm_concat(fsm_concat_n(fsm_identity(), n-m+1), fsm_concat_n(fsm_cross_product(fsm_empty_string(), fsm_identity()), k-n)))))));
+}
+
+struct fsm *rewrite_two_level(struct rewrite_batch *rb, struct fsm *lang, int rightside) {
+ struct fsm *Lower, *Upper, *Result;
+ Lower = rewrite_lower(rb, fsm_minimize(fsm_lower(fsm_copy(lang))));
+ Upper = rewrite_upper(rb, fsm_minimize(fsm_upper(lang)));
+ if (rightside == 1) {
+ Result = fsm_minimize(fsm_intersect(fsm_concat(Lower, rewrite_any_4tape(rb)), fsm_concat(Upper, rewrite_any_4tape(rb))));
+ } else {
+ Result = fsm_minimize(fsm_intersect(fsm_concat(rewrite_any_4tape(rb), Lower), fsm_concat(rewrite_any_4tape(rb), Upper)));
}
- fsm_destroy(NoSpecial);
- fsm_destroy(Id);
- fsm_destroy(Outside);
- fsm_destroy(EndOutside);
- fsm_destroy(Insert);
-
- //printf("Have result\n"); fflush(stdout);
- //Result = fsm_intersect(Insert,fsm_intersect(Context,Coerce));
- Result = fsm_intersect(Context,Coerce);
- Result = fsm_substitute_symbol(Result, "@#@", "@_EPSILON_SYMBOL_@");
- Result = fsm_substitute_symbol(Result, "@<@", "@_EPSILON_SYMBOL_@");
- Result = fsm_minimize(Result);
- Result = rewrite_cp_to_fst(fsm_minimize(Result), "@>@", "@Z@");
- Result = fsm_substitute_symbol(Result, "@[@", "@_EPSILON_SYMBOL_@");
- Result = fsm_substitute_symbol(Result, "@]@", "@_EPSILON_SYMBOL_@");
- sigma_remove("@>@", Result->sigma);
- sigma_remove("@Z@", Result->sigma);
- // sigma_sort(Result);
- Result = fsm_minimize(Result);
- fsm_compact(Result);
- sigma_sort(Result);
- fsm_clear_contexts(allcontexts);
- fsm_destroy(UnionCP);
- return(Result);
+ return Result;
}
-struct fsm *rewr_context_restrict(struct fsm *X, struct fsmcontexts *LR) {
+struct fsm *rewrite_lower(struct rewrite_batch *rb, struct fsm *lower) {
- struct fsm *Var, *Notvar, *UnionL, *UnionP, *Result;
- struct fsmcontexts *pairs;
+ /*
+ Lower:
- Var = fsm_symbol("@VARX@");
- Notvar = fsm_minimize(fsm_kleene_star(fsm_term_negation(fsm_symbol("@VARX@"))));
+ [ @O@ | ISyms | ISyms ]*
+ [ @0@ | Rulenums | Rulenums ]
+ [ <R>,@#@ | @0@,R | R ]
+ [ @ID@ | <R> | @0@ ]
- /* We add the variable symbol to all alphabets to avoid ? mathing it */
- /* which would cause extra nondeterminism */
+ R = any real symbol
+ <R> = any real symbol, not inserted
- /* Also, if any L or R is undeclared we add 0 */
- for (pairs = LR; pairs != NULL; pairs = pairs->next) {
- if (pairs->left == NULL) {
- pairs->left = fsm_empty_string();
- } else {
- sigma_add("@VARX@",pairs->left->sigma);
- sigma_sort(pairs->left);
- }
- if (pairs->right == NULL) {
- pairs->right = fsm_empty_string();
-
- } else {
- sigma_add("@VARX@",pairs->right->sigma);
- sigma_sort(pairs->right);
- }
- }
+ */
- UnionP = fsm_empty_set();
+ struct fsm *One, *Two, *Three, *Filter;
- for (pairs = LR; pairs != NULL ; pairs = pairs->next) {
- UnionP = fsm_minimize(fsm_union(fsm_minimize(fsm_concat(fsm_copy(pairs->left),fsm_concat(fsm_copy(Var),fsm_concat(fsm_copy(Notvar),fsm_concat(fsm_copy(Var),fsm_copy(pairs->right)))))), UnionP));
- }
-
- UnionL = fsm_minimize(fsm_concat(fsm_copy(Notvar),fsm_concat(fsm_copy(Var), fsm_concat(fsm_copy(X), fsm_concat(fsm_copy(Var),fsm_copy(Notvar))))));
+ One = fsm_minimize(fsm_concat(fsm_cross_product(fsm_empty_string(), fsm_symbol("@O@")), fsm_concat(fsm_cross_product(fsm_empty_string(), fsm_symbol("@0@")), fsm_concat(fsm_union(fsm_symbol("@#@"), fsm_copy(rb->ANY)), fsm_cross_product(fsm_empty_string(),fsm_symbol("@ID@"))))));
- Result = fsm_intersect(UnionL,fsm_complement(fsm_concat(fsm_minus(fsm_copy(Notvar),fsm_concat(fsm_universal(),fsm_symbol("@>@"))),fsm_minimize(fsm_concat(fsm_copy(UnionP),fsm_copy(Notvar))))));
+ Two = fsm_minimize(fsm_concat(fsm_cross_product(fsm_empty_string(), fsm_copy(rb->ISyms)), fsm_concat(fsm_cross_product(fsm_empty_string(), fsm_copy(rb->Rulenames)), fsm_concat(fsm_cross_product(fsm_empty_string(), fsm_union(fsm_copy(rb->ANY), fsm_symbol("@0@"))), fsm_copy(rb->ANY)))));
- if (sigma_find("@VARX@", Result->sigma) != -1) {
- Result = fsm_complement(fsm_substitute_symbol(Result, "@VARX@","@_EPSILON_SYMBOL_@"));
- } else {
- Result = fsm_complement(Result);
+ Three = fsm_minimize(fsm_concat(fsm_cross_product(fsm_empty_string(), fsm_copy(rb->ISyms)), fsm_concat(fsm_cross_product(fsm_empty_string(), fsm_copy(rb->Rulenames)), fsm_concat(fsm_cross_product(fsm_empty_string(), fsm_copy(rb->ANY)), fsm_cross_product(fsm_empty_string(), fsm_symbol("@0@"))))));
+
+ Filter = fsm_minimize(fsm_kleene_star(fsm_union(One, fsm_union(Two, Three))));
+ return fsm_minimize(fsm_lower(fsm_compose(lower, Filter)));
+}
+
+struct fsm *rewrite_any_4tape(struct rewrite_batch *rb) {
+
+ /*
+ Upper:
+
+ [ @O@ | ISyms ]*
+ [ @0@ | Rulenums ]
+ [ <R>,@#@ | @0@,R ]
+ [ @ID@ | R, at ID@, at 0@ ]
+
+ R = any real symbol
+ <R> = any real symbol, not inserted
+ */
+ if (rb->Any4Tape == NULL) {
+ rb->Any4Tape = fsm_minimize(fsm_kleene_star(fsm_union(fsm_concat(fsm_symbol("@O@"), fsm_concat(fsm_symbol("@0@"), fsm_concat(fsm_union(fsm_copy(rb->ANY), fsm_symbol("@#@")), fsm_symbol("@ID@")))), fsm_concat(fsm_copy(rb->ISyms), fsm_concat(fsm_copy(rb->Rulenames), fsm_concat(fsm_union(fsm_copy(rb->ANY), fsm_symbol("@0@")), fsm_union(fsm_copy(rb->ANY), fsm_union(fsm_symbol("@ID@"), fsm_symbol("@0@")))))))));
}
- fsm_destroy(UnionP);
- fsm_destroy(Var);
- fsm_destroy(Notvar);
- fsm_destroy(X);
- return(Result);
+ return fsm_copy(rb->Any4Tape);
}
-struct fsm *fsm_context_restrict(struct fsm *X, struct fsmcontexts *LR) {
+struct fsm *rewrite_upper(struct rewrite_batch *rb, struct fsm *upper) {
+ /*
+ Upper:
- struct fsm *Var, *Notvar, *UnionL, *UnionP, *Result, *Word;
- struct fsmcontexts *pairs;
+ [ @O@ | ISyms | ISyms ]*
+ [ @0@ | Rulenums | Rulenums ]
+ [ <R>,@#@ | @0@ | <R> ]
+ [ @ID@ | R | R, at ID@, at 0@ ]
- /* [.#. \.#.* .#.]-`[[ [\X* X C X \X*]&~[\X* [L1 X \X* X R1|...|Ln X \X* X Rn] \X*]],X,0] */
- /* Where X = variable symbol */
- /* The above only works if we do the subtraction iff the right hand side contains .#. in */
- /* its alphabet */
- /* A more generic formula is the following: */
+ R = any real symbol
+ <R> = any real symbol, not inserted
+ */
- /* `[[[(?) \.#.* (?)] - `[[[\X* X C X \X*] - [\X* [L1 X \X* X R1|...|Ln X \X* X Rn] \X*] ],X,0],.#.,0]; */
- /* Here, the LHS is another way of saying ~[?+ .#. ?+] */
+ struct fsm *One, *Two, *Three, *Filter;
- Var = fsm_symbol("@VARX@");
- Notvar = fsm_minimize(fsm_kleene_star(fsm_term_negation(fsm_symbol("@VARX@"))));
+ One = fsm_minimize(fsm_concat(fsm_cross_product(fsm_empty_string(), fsm_symbol("@O@")), fsm_concat(fsm_cross_product(fsm_empty_string(), fsm_symbol("@0@")), fsm_concat(fsm_union(fsm_symbol("@#@"), fsm_copy(rb->ANY)), fsm_cross_product(fsm_empty_string(),fsm_symbol("@ID@"))))));
- /* We add the variable symbol to all alphabets to avoid ? mathing it */
- /* which would cause extra nondeterminism */
- sigma_add("@VARX@",X->sigma);
- sigma_sort(X);
-
- /* Also, if any L or R is undeclared we add 0 */
- for (pairs = LR; pairs != NULL; pairs = pairs->next) {
- if (pairs->left == NULL) {
- pairs->left = fsm_empty_string();
- } else {
- sigma_add("@VARX@",pairs->left->sigma);
- sigma_substitute(".#.", "@#@", pairs->left->sigma);
- sigma_sort(pairs->left);
- }
- if (pairs->right == NULL) {
- pairs->right = fsm_empty_string();
- } else {
- sigma_add("@VARX@",pairs->right->sigma);
- sigma_substitute(".#.", "@#@", pairs->right->sigma);
- sigma_sort(pairs->right);
- }
- }
+ Two = fsm_minimize(fsm_concat(fsm_cross_product(fsm_empty_string(), fsm_copy(rb->ISyms)), fsm_concat(fsm_cross_product(fsm_empty_string(), fsm_copy(rb->Rulenames)), fsm_concat(fsm_cross_product(fsm_empty_string(), fsm_symbol("@0@")), fsm_cross_product(fsm_empty_string(), fsm_copy(rb->ANY))))));
- UnionP = fsm_empty_set();
-
- for (pairs = LR; pairs != NULL ; pairs = pairs->next) {
- UnionP = fsm_minimize(fsm_union(fsm_minimize(fsm_concat(fsm_copy(pairs->left),fsm_concat(fsm_copy(Var),fsm_concat(fsm_copy(Notvar),fsm_concat(fsm_copy(Var),fsm_copy(pairs->right)))))), UnionP));
- }
-
- UnionL = fsm_minimize(fsm_concat(fsm_copy(Notvar),fsm_concat(fsm_copy(Var), fsm_concat(fsm_copy(X), fsm_concat(fsm_copy(Var),fsm_copy(Notvar))))));
+ Three = fsm_minimize(fsm_concat(fsm_cross_product(fsm_empty_string(), fsm_copy(rb->ISyms)), fsm_concat(fsm_cross_product(fsm_empty_string(), fsm_copy(rb->Rulenames)), fsm_concat(fsm_copy(rb->ANY), fsm_cross_product(fsm_empty_string(), fsm_union(fsm_union(fsm_symbol("@0@"), fsm_copy(rb->ANY)), fsm_symbol("@ID@")))))));
- Result = fsm_intersect(UnionL, fsm_complement(fsm_concat(fsm_copy(Notvar),fsm_minimize(fsm_concat(fsm_copy(UnionP),fsm_copy(Notvar))))));
- if (sigma_find("@VARX@", Result->sigma) != -1) {
- Result = fsm_complement(fsm_substitute_symbol(Result, "@VARX@","@_EPSILON_SYMBOL_@"));
- } else {
- Result = fsm_complement(Result);
+ Filter = fsm_minimize(fsm_kleene_star(fsm_union(One, fsm_union(Two, Three))));
+ return fsm_minimize(fsm_lower(fsm_compose(upper, Filter)));
+}
+
+struct fsm *rewrite_align(struct fsm *upper, struct fsm *lower) {
+ struct fsm *first, *second, *third, *align, *align2;
+ /* `[[`[[Tape1of2(upper "@0@"*) & Tape2of2(lower "@0@"*) & ~[[? ?]* "@0@" "@0@" [? ?]*]], %@%_IDENTITY%_SYMBOL%_%@,%@UNK%@] .o. [? ?|"@UNK@" "@UNK@":"@ID@"]*].l, %@UNK%@,%@%_IDENTITY%_SYMBOL%_%@] */
+ first = fsm_minimize(rewrite_tape_m_to_n_of_k(fsm_concat(upper, fsm_kleene_star(fsm_symbol("@0@"))), 1, 1, 2));
+ second = fsm_minimize(rewrite_tape_m_to_n_of_k(fsm_concat(lower, fsm_kleene_star(fsm_symbol("@0@"))), 2, 2, 2));
+ third = fsm_minimize(fsm_parse_regex("~[[? ?]* \"@0@\" \"@0@\" [? ?]*]", NULL, NULL));
+
+ align = fsm_minimize(fsm_intersect(third, fsm_intersect(first, second)));
+ align = fsm_minimize(fsm_substitute_symbol(align, "@_IDENTITY_SYMBOL_@", "@UNK@"));
+ align2 = fsm_minimize(fsm_lower(fsm_compose(align, fsm_parse_regex("[? ? | \"@UNK@\" \"@UNK@\":\"@ID@\" ]*", NULL, NULL))));
+ align2 = fsm_minimize(fsm_substitute_symbol(align2, "@UNK@", "@_IDENTITY_SYMBOL_@"));
+ return align2;
+}
+
+struct fsm *rewrite_align_markup(struct fsm *upper, struct fsm *lower1, struct fsm *lower2) {
+ struct fsm *first, *second, *third, *fourth, *fifth, *sixth, *align, *align2;
+ /* [Tape1of2("@0@"*) & Tape2of2(lower1)] [Tape1of2(upper) & Tape2of2("@ID@"*)] [ Tape1of2(lower1) & Tape2of2("@0@"*)] */
+ /* + make sure IDENTITY and UNKNOWN are taken care of */
+ first = fsm_minimize(rewrite_tape_m_to_n_of_k(fsm_kleene_star(fsm_symbol("@0@")), 1, 1, 2));
+ second = fsm_minimize(rewrite_tape_m_to_n_of_k(lower1, 2, 2, 2));
+ third = fsm_minimize(rewrite_tape_m_to_n_of_k(upper, 1, 1, 2));
+ fourth = fsm_minimize(rewrite_tape_m_to_n_of_k(fsm_kleene_star(fsm_symbol("@ID@")), 2, 2, 2));
+ fifth = fsm_minimize(rewrite_tape_m_to_n_of_k(fsm_kleene_star(fsm_symbol("@0@")), 1, 1, 2));
+ sixth = fsm_minimize(rewrite_tape_m_to_n_of_k(lower2, 2, 2, 2));
+ align = fsm_minimize(fsm_concat(fsm_intersect(first, second), fsm_concat(fsm_intersect(third, fourth), fsm_intersect(fifth, sixth))));
+ align = fsm_minimize(fsm_substitute_symbol(align, "@_IDENTITY_SYMBOL_@", "@UNK@"));
+ align2 = fsm_minimize(fsm_lower(fsm_compose(align, fsm_parse_regex("[? ? | \"@UNK@\" \"@UNK@\":\"@ID@\" ]*", NULL, NULL))));
+ align2 = fsm_minimize(fsm_substitute_symbol(align2, "@UNK@", "@_IDENTITY_SYMBOL_@"));
+ return align2;
+}
+
+struct fsm *rewrite_itape(struct rewrite_batch *rb) {
+ if (rb->ITape == NULL) {
+ rb->ITape = fsm_parse_regex("[\"@I[]@\" ? ? ? | \"@I[@\" ? ? ? [\"@I@\" ? ? ?]* \"@I]@\" ? [?-\"@0@\"] ? ] [\"@I]@\" ? \"@0@\" ?]* | 0" , NULL, NULL);
}
+ return fsm_copy(rb->ITape);
+}
+
+struct fsm *rewrite_cp_markup(struct rewrite_batch *rb, struct fsm *upper, struct fsm *lower1, struct fsm *lower2, int rule_number) {
+ /* Same as rewrite_cp, could be consolidated */
+ struct fsm *Aligned, *threetape, *rulenumtape;
+ /* define CP(X,Y) Tape23of3(Align2(X,Y)) & [ "@I[@" ? ? ["@I@" ? ?]* "@I]@" ? ? | "@I[]@" ? ? | 0 ] */
+ Aligned = rewrite_align_markup(upper, lower1, lower2);
+ Aligned = rewrite_tape_m_to_n_of_k(Aligned, 3, 4, 4);
+ threetape = fsm_minimize(fsm_intersect(Aligned, rewrite_itape(rb)));
+ rulenumtape = rewrite_tape_m_to_n_of_k(fsm_minimize(fsm_kleene_star(fsm_symbol(rb->namestrings[rule_number-1]))), 2, 2, 4);
+ return fsm_minimize(fsm_intersect(threetape, rulenumtape));
+}
+
+struct fsm *rewrite_cp_transducer(struct rewrite_batch *rb, struct fsm *t, int rule_number) {
+ struct fsm *Aligned, *threetape, *rulenumtape;
+ Aligned = fsm_flatten(t, fsm_symbol("@0@"));
+ Aligned = rewrite_tape_m_to_n_of_k(Aligned, 3, 4, 4);
+ threetape = fsm_minimize(fsm_intersect(Aligned, rewrite_itape(rb)));
+ rulenumtape = rewrite_tape_m_to_n_of_k(fsm_minimize(fsm_kleene_star(fsm_symbol(rb->namestrings[rule_number-1]))), 2, 2, 4);
+ return fsm_minimize(fsm_intersect(threetape, rulenumtape));
+}
+
+struct fsm *rewrite_cp(struct rewrite_batch *rb, struct fsm *upper, struct fsm *lower, int rule_number) {
+ struct fsm *Aligned, *threetape, *rulenumtape;
+ /* define CP(X,Y) Tape23of3(Align2(X,Y)) & [ "@I[@" ? ? ["@I@" ? ?]* "@I]@" ? ? | "@I[]@" ? ? | 0 ] */
+ Aligned = rewrite_align(upper, lower);
+ Aligned = rewrite_tape_m_to_n_of_k(Aligned, 3, 4, 4);
+ threetape = fsm_minimize(fsm_intersect(Aligned, rewrite_itape(rb)));
+ rulenumtape = rewrite_tape_m_to_n_of_k(fsm_minimize(fsm_kleene_star(fsm_symbol(rb->namestrings[rule_number-1]))), 2, 2, 4);
+ return fsm_minimize(fsm_intersect(threetape, rulenumtape));
+}
+
+void rewrite_add_special_syms(struct rewrite_batch *rb, struct fsm *net) {
+ int i;
+ if (net == NULL)
+ return;
+ sigma_substitute(".#.", "@#@", net->sigma); /* We convert boundaries to our interal rep. */
+ /* This is because sigma merging (fsm_merge_sigma()) is handled */
+ /* in a special way for .#., which we don't want here. */
- if (sigma_find("@#@", Result->sigma) != -1) {
- Word = fsm_minimize(fsm_concat(fsm_symbol("@#@"),fsm_concat(fsm_kleene_star(fsm_term_negation(fsm_symbol("@#@"))),fsm_symbol("@#@"))));
- Result = fsm_intersect(Word, Result);
- Result = fsm_substitute_symbol(Result, "@#@", "@_EPSILON_SYMBOL_@");
+ for (i = 0; specialsymbols[i] != NULL; i++) {
+ if (sigma_find(specialsymbols[i], net->sigma) == -1)
+ sigma_add(specialsymbols[i], net->sigma);
}
- fsm_destroy(UnionP);
- fsm_destroy(Var);
- fsm_destroy(Notvar);
- fsm_destroy(X);
- fsm_clear_contexts(pairs);
- return(Result);
+ for (i = 1; i <= rb->num_rules; i++) {
+ sigma_add(rb->namestrings[i-1], net->sigma);
+ }
+ sigma_sort(net);
}
+
void fsm_clear_contexts(struct fsmcontexts *contexts) {
struct fsmcontexts *c, *cp;
for (c = contexts; c != NULL; c = cp) {
@@ -628,208 +532,79 @@ void fsm_clear_contexts(struct fsmcontexts *contexts) {
}
}
-struct fsm *rewrite_pad(struct fsm *net) {
- struct fsm *pad;
- /* [? 0:"@>@" ? 0:"@<@"]* [? 0:"@>@" ?] */
- pad = fsm_minimize(fsm_parse_regex("[? 0:\"@>@\" ? 0:\"@<@\"]* [? 0:\"@>@\" ?]", NULL, NULL));
- return(fsm_minimize(fsm_lower(fsm_compose(net,pad))));
-}
-
-/** Takes two languages and produces the cross-product on one tape with marker symbols */
-/** e.g. CP(a b c, d e) -> a@>@d@<@b@>@e@<@c@>@@Z@, where @Z@ is a "hard zero" symbol. */
-struct fsm *rewrite_cp(struct fsm *U, struct fsm *L) {
+struct fsm *rewr_context_restrict(struct rewrite_batch *rb, struct fsm *X, struct fsmcontexts *LR) {
- struct fsm *PadU, *PadL, *One, *Two, *Three, *Result;
-
- PadU = fsm_create("");
- PadL = fsm_create("");
-
- PadU->states = xxmalloc(sizeof(struct fsm_state)*5);
- PadL->states = xxmalloc(sizeof(struct fsm_state)*5);
-
- sigma_add("@<@",PadU->sigma); /* 3 */
- sigma_add("@>@",PadU->sigma); /* 4 */
- sigma_add("@<@",PadL->sigma); /* 3 */
- sigma_add("@>@",PadL->sigma); /* 4 */
-
- sigma_add_special(IDENTITY,PadL->sigma);
- sigma_add_special(EPSILON,PadL->sigma);
- sigma_add_special(UNKNOWN,PadL->sigma);
-
- sigma_add_special(IDENTITY,PadU->sigma);
- sigma_add_special(EPSILON,PadU->sigma);
- sigma_add_special(UNKNOWN,PadU->sigma);
-
- /* define PadU [NN 0:%> 0:NN 0:%<]* [NN 0:%> 0:NN]; */
- add_fsm_arc(PadU->states,0, 0,IDENTITY,IDENTITY,1,1,1);
- add_fsm_arc(PadU->states,1, 1,EPSILON,4,2,0,0);
- add_fsm_arc(PadU->states,2, 2,EPSILON,UNKNOWN,3,0,0);
- add_fsm_arc(PadU->states,3, 3,EPSILON,3,0,1,0);
- add_fsm_arc(PadU->states,4, -1,-1,-1,-1,-1,-1);
-
- /* define PadL [0:NN 0:%> NN 0:%<]* [0:NN 0:%> NN] */
- add_fsm_arc(PadL->states,0, 0,EPSILON,UNKNOWN,1,1,1);
- add_fsm_arc(PadL->states,1, 1,EPSILON,4,2,0,0);
- add_fsm_arc(PadL->states,2, 2,IDENTITY,IDENTITY,3,0,0);
- add_fsm_arc(PadL->states,3, 3,EPSILON,3,0,1,0);
- add_fsm_arc(PadL->states,4, -1,-1,-1,-1,-1,-1);
-
- sigma_sort(PadL);
- sigma_sort(PadU);
-
- PadL->statecount = PadU->statecount = 4;
- PadL->pathcount = PadU->pathcount = -1;
- PadL->linecount = PadU->linecount = 5;
-
- fsm_update_flags(PadU, YES, YES, YES, YES, NO, NO);
- fsm_update_flags(PadL, YES, YES, YES, YES, NO, NO);
-
- /* Result = [U "@Z@"* .o. PadU].l & [L "@Z@"* .o. PadL].l & ~$["@Z@" "@>@" "@Z@"] */;
- One = fsm_lower(fsm_compose(fsm_concat(fsm_copy(U),fsm_kleene_star(fsm_symbol("@Z@"))),PadU));
- Two = fsm_lower(fsm_compose(fsm_concat(fsm_copy(L),fsm_kleene_star(fsm_symbol("@Z@"))),PadL));
-
- Three = fsm_complement(fsm_contains(fsm_concat(fsm_symbol("@Z@"),fsm_concat(fsm_symbol("@>@"),fsm_symbol("@Z@")))));
+ struct fsm *Var, *Notvar, *UnionL, *UnionP, *Result, *NewX, *Left, *Right;
+ struct fsmcontexts *pairs;
- Result = fsm_minimize(fsm_intersect(fsm_intersect(One,Two),Three));
- Result = fsm_intersect(Result, fsm_complement(fsm_concat(fsm_universal(),fsm_symbol("@<@"))));
- fsm_destroy(U);
- fsm_destroy(L);
- return(Result);
-}
+ Var = fsm_symbol("@VARX@");
+ //Notvar = fsm_minimize(fsm_kleene_star(fsm_term_negation(fsm_symbol("@VARX@"))));
+ Notvar = fsm_minus(rewrite_any_4tape(rb), fsm_contains(fsm_symbol("@VARX@")));
+ /* We add the variable symbol to all alphabets to avoid ? matching it */
+ /* which would cause extra nondeterminism */
-/* Take a simple FSM and convert it to a transducer where any */
-/* symbol followed by the lower_symbol is part of a a:b arc */
-/* e.g. a>ba>c0>d -> a:b a:c 0:d , where 0 in the FSM is the hard zero */
-
-struct fsm *rewrite_cp_to_fst(struct fsm *net, char *lower_symbol, char *zero_symbol) {
- struct state_link {
- int in;
- int out;
- struct fsm_state *outptr; /* This points to where the target state is */
- _Bool crowded;
- } *state_link;
-
- int i, j, t, lower, zero, *targets, linecount, in, out, has_unk;
- _Bool *useless_states;
- struct fsm_state *fsm, *fsm2, *tempfsm;
-
- has_unk = 0;
-
- if ((lower = sigma_find(lower_symbol, net->sigma)) == -1)
- return(net);
- if ((zero = sigma_find(zero_symbol, net->sigma)) == -1)
- zero = 0;
- state_link = xxmalloc(sizeof(struct state_link) * net->statecount);
- useless_states = xxcalloc(net->statecount, sizeof(_Bool));
- targets = xxmalloc(net->statecount * sizeof(int));
- fsm = net->states;
- for (i=0; i < net->statecount ; i++) {
- (state_link+i)->in = -1;
- (state_link+i)->out = -1;
- (state_link+i)->crowded = 0;
- (state_link+i)->outptr = NULL;
- *(targets+i) = -1;
- }
-
- /* Mark states that have > symbol outgoing (U states) */
- for (i=0, linecount = 0; (fsm+i)->state_no != -1; i++) {
- linecount++;
- if ((fsm+i)->in == lower) {
- (state_link+(fsm+i)->state_no)->out = (fsm+i)->target;
- /* Mark both source and target useless (to be deleted) */
- *(targets+(fsm+i)->target) = (fsm+i)->state_no;
- *(useless_states+(fsm+i)->state_no) = 1;
- *(useless_states+(fsm+i)->target) = 1;
- }
- }
- /* Mark the states that target the U states */
- for (i=0; (fsm+i)->state_no != -1; i++) {
- if ((fsm+i)->target != -1 && (state_link+(fsm+i)->target)->out != -1) {
- (state_link+(fsm+i)->state_no)->in = (fsm+i)->state_no;
- }
+ NewX = fsm_copy(X);
+ if (sigma_find("@VARX@", NewX->sigma) == -1) {
+ sigma_add("@VARX@", NewX->sigma);
+ sigma_sort(NewX);
}
- /* Store a ptr to the first line of the target states in the lookup table */
- for (i=0; (fsm+i)->state_no != -1; i++) {
- if ((state_link+(fsm+i)->state_no)->out != -1 && (fsm+i)->in != lower) {
- printf("**Warning, mediator state is crowded!\n");
- (state_link+(fsm+i)->state_no)->crowded = 1;
- *(useless_states+(fsm+i)->state_no) = 0;
- }
- /* Is (fsm+i)->state_no a target? */
- if ((t = *(targets+(fsm+i)->state_no)) != -1) {
- /* Yes, whose target */
+ UnionP = fsm_empty_set();
- if ((state_link+t)->outptr == NULL) {
- (state_link+t)->outptr = (fsm+i);
- }
- }
+ for (pairs = LR; pairs != NULL ; pairs = pairs->next) {
+ if (pairs->left == NULL) {
+ Left = fsm_empty_string();
+ } else {
+ Left = fsm_copy(pairs->cpleft);
+ sigma_add("@VARX@", Left->sigma);
+ sigma_sort(Left);
+ }
+ if (pairs->right == NULL) {
+ Right = fsm_empty_string();
+ } else {
+ Right = fsm_copy(pairs->cpright);
+ sigma_add("@VARX@", Right->sigma);
+ sigma_sort(Right);
+ }
+ UnionP = fsm_union(fsm_concat(Left, fsm_concat(fsm_copy(Var), fsm_concat(fsm_copy(Notvar), fsm_concat(fsm_copy(Var), Right)))), UnionP);
}
+ UnionL = fsm_concat(fsm_copy(Notvar), fsm_concat(fsm_copy(Var), fsm_concat(fsm_copy(NewX), fsm_concat(fsm_copy(Var), fsm_copy(Notvar)))));
+ Result = fsm_minus(UnionL, fsm_concat(fsm_copy(Notvar), fsm_concat(fsm_copy(UnionP), fsm_copy(Notvar))));
- /* If there is an arc to a U state, do the cross-product */
- /* in-out pairings for all arcs outgoing from the associated out state */
-
- fsm2 = xxmalloc(sizeof(struct fsm)*linecount);
-
- for (i=0,j=0; (fsm+i)->state_no != -1; i++) {
- if ((fsm+i)->target != -1 && (state_link+(fsm+i)->target)->out != -1) {
- for (tempfsm=(state_link+(fsm+i)->target)->outptr,t=tempfsm->state_no;tempfsm->state_no == t ;tempfsm++) {
- in = (fsm+i)->in;
- out = tempfsm->out;
- if (in == IDENTITY || out == IDENTITY) {
- has_unk = 1;
- }
-
- if (in == IDENTITY && out == IDENTITY) {
- add_fsm_arc(fsm2,j, (fsm+i)->state_no, in, out, tempfsm->target, (fsm+i)->final_state, (fsm+i)->start_state);
- j++;
- }
- in = in == IDENTITY ? UNKNOWN : in;
- out = out == IDENTITY ? UNKNOWN : out;
- add_fsm_arc(fsm2,j, (fsm+i)->state_no, in, out, tempfsm->target, (fsm+i)->final_state, (fsm+i)->start_state);
- j++;
- }
- if ((state_link+(fsm+i)->target)->crowded) {
- add_fsm_arc(fsm2,j,(fsm+i)->state_no,(fsm+i)->in,(fsm+i)->out,(fsm+i)->target,(fsm+i)->final_state,(fsm+i)->start_state);
- j++;
- }
- } else if (!*(useless_states+(fsm+i)->state_no) && (fsm+i)->in != lower) {
- add_fsm_arc(fsm2,j,(fsm+i)->state_no,(fsm+i)->in,(fsm+i)->out,(fsm+i)->target,(fsm+i)->final_state,(fsm+i)->start_state);
- j++;
- }
+ if (sigma_find("@VARX@", Result->sigma) != -1) {
+ Result = fsm_complement(fsm_substitute_symbol(Result, "@VARX@","@_EPSILON_SYMBOL_@"));
+ } else {
+ Result = fsm_complement(Result);
}
+ fsm_destroy(UnionP);
+ fsm_destroy(Var);
+ fsm_destroy(Notvar);
+ fsm_destroy(NewX);
+ return(Result);
+}
- add_fsm_arc(fsm2, j, -1, -1, -1, -1, -1, -1);
+struct fsm *rewrite_epextend(struct rewrite_batch *rb) {
- /* Renumber states, store new number in *(targets+x) */
- /* Also, replace epsilons */
+ struct fsm *one, *two, *allzeroupper, *threea, *threeb, *threec, *three;
- for (i=0, j=0; i<net->statecount;i++) {
- *(targets+i) = j;
- if (*(useless_states+i) != 1) {
- j++;
- }
- }
- for (i=0; (fsm2+i)->state_no != -1 ; i++) {
- (fsm2+i)->state_no = *(targets+(fsm2+i)->state_no);
- if ((fsm2+i)->target != -1) {
- (fsm2+i)->target = *(targets+(fsm2+i)->target);
- }
- if ((fsm2+i)->in == zero)
- (fsm2+i)->in = EPSILON;
- if ((fsm2+i)->out == zero)
- (fsm2+i)->out = EPSILON;
-
- }
+ /* 1. @O@ @0@ [ANY|@#@] @ID@ */
+ /* 2. @I[]@ @#Rule@ [ANY] [@ID@|@0@|ANY] */
+ /* 3a. @I[@ @#Rule@ [ANY] [@ID@|@0@|ANY] */
+ /* 3b. @I@ @#Rule@ [ANY] [@ID@|@0@|ANY] */
+ /* 3c. @I]@ @#Rule@ [ANY] [@ID@|@0@|ANY] */
+ /* 3. [3a|3b|3c] & ~[[? ? "@0@" ?]*] */
+
+ /* TODO lower version as well */
- if (has_unk == 1 && sigma_find_number(UNKNOWN, net->sigma) == -1) {
- sigma_add_special(UNKNOWN, net->sigma);
+ if (rb->Epextend == NULL) {
+ one = fsm_minimize(fsm_concat(fsm_symbol("@O@"), fsm_concat(fsm_symbol("@0@"), fsm_concat(fsm_union(fsm_copy(rb->ANY), fsm_symbol("@#@")), fsm_symbol("@ID@")))));
+ two = fsm_minimize(fsm_concat(fsm_symbol("@I[]@"), fsm_concat(fsm_copy(rb->Rulenames), fsm_concat(fsm_copy(rb->ANY), fsm_union(fsm_symbol("@0@"), fsm_union(fsm_symbol("@ID@"), fsm_copy(rb->ANY)))))));
+ allzeroupper = fsm_parse_regex("~[[? ? \"@0@\" ?]*]", NULL, NULL);
+ threea = fsm_minimize(fsm_concat(fsm_symbol("@I[@"), fsm_concat(fsm_copy(rb->Rulenames), fsm_concat(fsm_union(fsm_copy(rb->ANY), fsm_symbol("@0@")), fsm_union(fsm_symbol("@0@"), fsm_union(fsm_symbol("@ID@"), fsm_copy(rb->ANY)))))));
+ threeb = fsm_minimize(fsm_kleene_star(fsm_concat(fsm_symbol("@I@"), fsm_concat(fsm_copy(rb->Rulenames), fsm_concat(fsm_union(fsm_copy(rb->ANY), fsm_symbol("@0@")), fsm_union(fsm_symbol("@0@"), fsm_union(fsm_symbol("@ID@"), fsm_copy(rb->ANY))))))));
+ threec = fsm_minimize(fsm_concat(fsm_symbol("@I]@"), fsm_concat(fsm_copy(rb->Rulenames), fsm_concat(fsm_union(fsm_copy(rb->ANY), fsm_symbol("@0@")), fsm_union(fsm_symbol("@0@"), fsm_union(fsm_symbol("@ID@"), fsm_copy(rb->ANY)))))));
+ three = fsm_intersect(allzeroupper, fsm_concat(threea, fsm_concat(threeb, threec)));
+ rb->Epextend = fsm_minimize(fsm_union(fsm_union(one, two), three));
}
-
- net->states = fsm2;
- xxfree(fsm);
- xxfree(useless_states);
- xxfree(targets);
- xxfree(state_link);
- return(net);
+ return fsm_copy(rb->Epextend);
}
diff --git a/back-ends/foma/sigma.c b/back-ends/foma/sigma.c
index 3d58608..bbaed3e 100644
--- a/back-ends/foma/sigma.c
+++ b/back-ends/foma/sigma.c
@@ -82,7 +82,7 @@ int sigma_add_special (int symbol, struct sigma *sigma) {
(sigma_previous)->next = sigma_splice;
sigma_splice->number = symbol;
sigma_splice->symbol = str;
- (sigma_splice)->next = sigma;
+ (sigma_splice)->next = sigma;
return(symbol);
} else {
sigma_splice->symbol = sigma->symbol;
@@ -130,7 +130,7 @@ int sigma_add (char *symbol, struct sigma *sigma) {
}
sigma = sigma->next;
}
- sigma->next = NULL;
+ sigma->next = NULL;
sigma->symbol = xxstrdup(symbol);
return(sigma->number);
} else {
@@ -146,7 +146,7 @@ int sigma_add (char *symbol, struct sigma *sigma) {
sigma_splice->number = assert;
sigma_splice->symbol = xxmalloc(sizeof(char)*(strlen(symbol)+1));
strcpy(sigma_splice->symbol, symbol);
- (sigma_splice)->next = sigma;
+ (sigma_splice)->next = sigma;
return(assert);
} else {
sigma_splice->symbol = sigma->symbol;
@@ -200,7 +200,7 @@ void sigma_cleanup (struct fsm *net, int force) {
j++;
}
}
- for (i=0; (fsm+i)->state_no != -1; i++) {
+ for (i=0; (fsm+i)->state_no != -1; i++) {
if ((fsm+i)->in > 2)
(fsm+i)->in = *(attested+(fsm+i)->in);
if ((fsm+i)->out > 2)
@@ -232,6 +232,8 @@ void sigma_cleanup (struct fsm *net, int force) {
int sigma_max(struct sigma *sigma) {
int i;
+ if (sigma == NULL)
+ return -1;
for (i=-1; sigma != NULL; sigma = sigma->next)
i = sigma->number > i ? sigma->number : i;
return(i);
@@ -311,7 +313,7 @@ int sigma_substitute(char *symbol, char *sub, struct sigma *sigma) {
for (; sigma != NULL && sigma->number != -1 ; sigma = sigma->next) {
if (strcmp(sigma->symbol, symbol) == 0) {
xxfree(sigma->symbol);
- sigma->symbol = strdup(sub);
+ sigma->symbol = xxstrdup(sub);
return(sigma->number);
}
}
@@ -320,7 +322,7 @@ int sigma_substitute(char *symbol, char *sub, struct sigma *sigma) {
int sigma_find(char *symbol, struct sigma *sigma) {
- if (sigma->number == -1) {
+ if (sigma == NULL || sigma->number == -1) {
return -1;
}
for (; sigma != NULL && sigma->number != -1 ; sigma = sigma->next) {
diff --git a/back-ends/foma/spelling.c b/back-ends/foma/spelling.c
index 426eab4..9cddbf7 100644
--- a/back-ends/foma/spelling.c
+++ b/back-ends/foma/spelling.c
@@ -95,7 +95,7 @@ void print_match(struct apply_med_handle *medh, struct astarnode *node, struct s
medh->instring = xxrealloc(medh->instring, medh->instring_length*sizeof(char));
}
for (i = 0; !(int_stack_isempty()); ) {
- sym = int_stack_pop();
+ sym = int_stack_pop();
if (sym > 2) {
printptr += sprintf(medh->instring+printptr,"%s", print_sym(sym, sigma));
i += utf8skip(word+i)+1;
@@ -115,7 +115,7 @@ void print_match(struct apply_med_handle *medh, struct astarnode *node, struct s
}
}
}
- medh->cost = node->g;
+ medh->cost = node->g;
// printf("Cost[f]: %i\n\n", node->g);
}
@@ -149,7 +149,7 @@ struct apply_med_handle *apply_med_init(struct fsm *net) {
struct apply_med_handle *medh;
struct sigma *sigma;
- medh = xxcalloc(1,sizeof(struct apply_med_handle));
+ medh = xxcalloc(1,sizeof(struct apply_med_handle));
medh->net = net;
medh->agenda = xxmalloc(sizeof(struct astarnode)*INITIAL_AGENDA_SIZE);
medh->agenda->f = -1;
@@ -260,7 +260,7 @@ char *apply_med(struct apply_med_handle *medh, char *word) {
thisskip = utf8skip(word+i)+1;
strncpy(temputf, word+i, thisskip);
temputf[thisskip] = '\0';
- if (sh_find_string(medh->sigmahash, temputf) != NULL) {
+ if (sh_find_string(medh->sigmahash, temputf) != NULL) {
*(medh->intword+j) = sh_get_value(medh->sigmahash);
} else {
*(medh->intword+j) = IDENTITY;
@@ -353,7 +353,7 @@ char *apply_med(struct apply_med_handle *medh, char *word) {
if (g+h <= medh->med_cutoff) {
if (!node_insert(medh, medh->curr_pos, target, g, h, in, out, medh->curr_agenda_offset)) {
goto out;
- }
+ }
}
if (medh->curr_pos == medh->utf8len)
goto skip;
@@ -451,16 +451,16 @@ struct astarnode *node_delete_min(struct apply_med_handle *medh) {
child = i<<1;
/* If right child is smaller (higher priority) than left child */
- if (child != medh->heapcount &&
- ((medh->agenda+medh->heap[child+1])->f < (medh->agenda+medh->heap[child])->f ||
- ((medh->agenda+medh->heap[child+1])->f <= (medh->agenda+medh->heap[child])->f &&
+ if (child != medh->heapcount &&
+ ((medh->agenda+medh->heap[child+1])->f < (medh->agenda+medh->heap[child])->f ||
+ ((medh->agenda+medh->heap[child+1])->f <= (medh->agenda+medh->heap[child])->f &&
(medh->agenda+medh->heap[child+1])->wordpos > (medh->agenda+medh->heap[child])->wordpos))) {
child++;
}
/* If child has lower priority than last element */
- if ((medh->agenda+medh->heap[child])->f < lastptr->f ||
- ((medh->agenda+medh->heap[child])->f <= lastptr->f &&
+ if ((medh->agenda+medh->heap[child])->f < lastptr->f ||
+ ((medh->agenda+medh->heap[child])->f <= lastptr->f &&
(medh->agenda+medh->heap[child])->wordpos > lastptr->wordpos)) {
medh->heap[i] = medh->heap[child];
@@ -686,7 +686,7 @@ void fsm_create_letter_lookup(struct apply_med_handle *medh, struct fsm *net) {
if (BITTEST(medh->letterbits+(i*medh->bytes_per_letter_array),j)) {
//printf("[%i]",j);
}
- }
+ }
//printf("\n");
}
int_stack_clear();
@@ -735,7 +735,7 @@ void cmatrix_print_att(struct fsm *net, FILE *outfile) {
cm = net->medlookup->confusion_matrix;
- for (i = 0; i < maxsigma ; i++) {
+ for (i = 0; i < maxsigma ; i++) {
for (j = 0; j < maxsigma ; j++) {
if ((i != 0 && i < 3) || (j != 0 && j < 3)) { continue; }
if (i == 0 && j != 0) {
@@ -776,7 +776,7 @@ void cmatrix_print(struct fsm *net) {
printf("\n");
- for (i = 0; i < maxsigma ; i++) {
+ for (i = 0; i < maxsigma ; i++) {
for (j = 0; j < maxsigma ; j++) {
if (j == 0) {
if (i == 0) {
@@ -818,7 +818,7 @@ void cmatrix_init(struct fsm *net) {
else
*(cm+i*maxsigma+j) = 1;
}
- }
+ }
}
void cmatrix_default_substitute(struct fsm *net, int cost) {
@@ -828,12 +828,12 @@ void cmatrix_default_substitute(struct fsm *net, int cost) {
for (i = 1; i < maxsigma; i++) {
for (j = 1; j < maxsigma; j++) {
if (i == j) {
- *(cm+i*maxsigma+j) = 0;
+ *(cm+i*maxsigma+j) = 0;
} else {
*(cm+i*maxsigma+j) = cost;
}
}
- }
+ }
}
void cmatrix_default_insert(struct fsm *net, int cost) {
diff --git a/back-ends/foma/stack.c b/back-ends/foma/stack.c
index 37b2ced..cf0397d 100644
--- a/back-ends/foma/stack.c
+++ b/back-ends/foma/stack.c
@@ -47,7 +47,7 @@ int stack_add(struct fsm *fsm) {
stack_ptr_previous = NULL;
fsm_count(fsm);
- if (strcmp(fsm->name,"") == 0)
+ if (strcmp(fsm->name,"") == 0)
sprintf(fsm->name, "%X",rand());
for (i=0, stack_ptr = main_stack; stack_ptr->number != -1; i++) {
stack_ptr_previous = stack_ptr;
@@ -116,7 +116,7 @@ struct fsm *stack_pop(void) {
stack_ptr->amedh = NULL;
}
stack_ptr->fsm = NULL;
- xxfree(stack_ptr);
+ xxfree(stack_ptr);
return(fsm);
}
@@ -171,7 +171,7 @@ struct stack_entry *stack_find_bottom () {
struct stack_entry *stack_find_second () {
struct stack_entry *stack_ptr;
- /*
+ /*
if (main_stack->number == -1) {
return NULL;
}
diff --git a/back-ends/foma/structures.c b/back-ends/foma/structures.c
index 2a17ec9..8de27ae 100644
--- a/back-ends/foma/structures.c
+++ b/back-ends/foma/structures.c
@@ -72,7 +72,7 @@ void fsm_sort_arcs(struct fsm *net, int direction) {
if (direction == 1)
qsort(fsm+lasthead, numlines, sizeof(struct fsm_state), scin);
else
- qsort(fsm+lasthead, numlines, sizeof(struct fsm_state), scout);
+ qsort(fsm+lasthead, numlines, sizeof(struct fsm_state), scout);
}
numlines = 0;
lasthead = i + 1;
@@ -92,7 +92,7 @@ void fsm_sort_arcs(struct fsm *net, int direction) {
if (direction == 2) {
net->arcs_sorted_out = 1;
net->arcs_sorted_in = 0;
- }
+ }
}
struct state_array *map_firstlines(struct fsm *net) {
@@ -311,7 +311,7 @@ int fsm_isuniversal(struct fsm *net) {
net = fsm_minimize(net);
fsm_compact(net);
fsm = net->states;
- if ((fsm->target == 0 && fsm->final_state == 1 && (fsm+1)->state_no == 0) &&
+ if ((fsm->target == 0 && fsm->final_state == 1 && (fsm+1)->state_no == 0) &&
(fsm->in == IDENTITY && fsm->out == IDENTITY) &&
((fsm+1)->state_no == -1) &&
(sigma_max(net->sigma)<3) ) {
@@ -327,7 +327,7 @@ int fsm_isempty(struct fsm *net) {
fsm = net->states;
if (fsm->target == -1 && fsm->final_state == 0 && (fsm+1)->state_no == -1)
return 1;
- else
+ else
return 0;
}
@@ -427,7 +427,7 @@ int fsm_isidentity(struct fsm *net) {
struct discrepancy {
short int *string;
short int length;
- _Bool visited;
+ Boolean visited;
};
struct state_array *state_array;
@@ -585,7 +585,7 @@ struct fsm *fsm_lowerdet(struct fsm *net) {
}
if (maxarc > (maxsigma-2)) {
for (i=maxarc; i > (maxsigma-2); i--) {
- sprintf(repstr,"%012X",newsym++);
+ sprintf(repstr,"%012X",newsym++);
sigma_add(repstr, net->sigma);
}
sigma_sort(net);
@@ -624,7 +624,7 @@ struct fsm *fsm_lowerdeteps(struct fsm *net) {
}
if (maxarc > (maxsigma-2)) {
for (i=maxarc; i > (maxsigma-2); i--) {
- sprintf(repstr,"%012X",newsym++);
+ sprintf(repstr,"%012X",newsym++);
sigma_add(repstr, net->sigma);
}
sigma_sort(net);
@@ -649,7 +649,7 @@ struct fsm *fsm_extract_nonidentity(struct fsm *net) {
struct discrepancy {
short int *string;
short int length;
- _Bool visited;
+ Boolean visited;
};
struct state_array *state_array;
@@ -768,11 +768,11 @@ struct fsm *fsm_extract_nonidentity(struct fsm *net) {
goto nopop;
}
continue;
- fail:
+ fail:
curr_ptr->out = killnum;
if (curr_ptr->state_no == (curr_ptr+1)->state_no) {
ptr_stack_push(curr_ptr+1);
- }
+ }
}
ptr_stack_clear();
sigma_sort(net);
@@ -794,7 +794,7 @@ struct fsm *fsm_copy (struct fsm *net) {
fsm_count(net);
net_copy->sigma = sigma_copy(net->sigma);
- net_copy->states = fsm_state_copy(net->states, net->linecount);
+ net_copy->states = fsm_state_copy(net->states, net->linecount);
return(net_copy);
}
@@ -832,7 +832,7 @@ void add_quantifier (char *string) {
if (quantifiers == NULL) {
q = xxmalloc(sizeof(struct defined_quantifiers));
quantifiers = q;
- } else {
+ } else {
for (q = quantifiers; q->next != NULL; q = q->next) {
}
@@ -882,7 +882,7 @@ char *find_quantifier (char *string) {
}
void purge_quantifier (char *string) {
- struct defined_quantifiers *q, *q_prev;
+ struct defined_quantifiers *q, *q_prev;
for (q = quantifiers, q_prev = NULL; q != NULL; q_prev = q, q = q->next) {
if (strcmp(string, q->name) == 0) {
if (q_prev != NULL) {
diff --git a/back-ends/foma/topsort.c b/back-ends/foma/topsort.c
index 3ed77b5..3c39285 100644
--- a/back-ends/foma/topsort.c
+++ b/back-ends/foma/topsort.c
@@ -60,7 +60,7 @@ struct fsm *fsm_topsort (struct fsm *net) {
*(pathcount+i) = 0;
}
- for (i=0, lc=0; (fsm+i)->state_no != -1; i++) {
+ for (i=0, lc=0; (fsm+i)->state_no != -1; i++) {
lc++;
if ((fsm+i)->target != -1) {
(*(invcount+(fsm+i)->target))++;
@@ -117,7 +117,7 @@ struct fsm *fsm_topsort (struct fsm *net) {
}
}
curr_fsm++;
- }
+ }
}
/* Case (2) */
@@ -136,7 +136,7 @@ struct fsm *fsm_topsort (struct fsm *net) {
if (curr_fsm->final_state == 1 && !overflow) {
grand_pathcount += *(pathcount + curr_state);
if (grand_pathcount < 0)
- overflow = 1;
+ overflow = 1;
}
for (; curr_fsm->state_no == curr_state; curr_fsm++) {
diff --git a/back-ends/foma/trie.c b/back-ends/foma/trie.c
index 77d3066..8c53c46 100644
--- a/back-ends/foma/trie.c
+++ b/back-ends/foma/trie.c
@@ -48,7 +48,7 @@ struct fsm *fsm_trie_done(struct fsm_trie_handle *th) {
fsm_construct_add_arc(newh, thash->sourcestate, thash->targetstate, thash->insym, thash->outsym);
} else {
break;
- }
+ }
}
}
for (i = 0; i <= th->used_states; i++) {
diff --git a/back-ends/foma/utf8.c b/back-ends/foma/utf8.c
index b8c1999..4ac1ebc 100644
--- a/back-ends/foma/utf8.c
+++ b/back-ends/foma/utf8.c
@@ -78,8 +78,8 @@ char *escape_string(char *string, char chr) {
newstring[j] = chr;
} else {
newstring[j] = string[i];
- }
- }
+ }
+ }
return(newstring);
} else {
return(string);
@@ -94,7 +94,7 @@ void strip_newline(char *s) {
for (i = 0; i < len; i++ ) {
if (s[i] == '\n' ) {
s[i] = '\0';
- return;
+ return;
}
}
}
@@ -109,7 +109,7 @@ void dequote_string(char *s) {
}
*(s+j) = '\0';
decode_quoted(s);
- }
+ }
}
/* Decode quoted strings. This includes: */
@@ -255,16 +255,16 @@ int hexstrtoint(char *str) {
int hex;
if (*str > 0x60) {
- hex = (*str - 0x57) << 4;
+ hex = (*str - 0x57) << 4;
} else if (*str > 0x40) {
- hex = (*str - 0x37) << 4;
+ hex = (*str - 0x37) << 4;
} else {
hex = (*str - 0x30) << 4;
}
if (*(str+1) > 0x60) {
- hex += (*(str+1) - 0x57);
+ hex += (*(str+1) - 0x57);
} else if (*(str+1) > 0x40) {
- hex += (*(str+1) - 0x37);
+ hex += (*(str+1) - 0x37);
} else {
hex += (*(str+1) - 0x30);
}
diff --git a/back-ends/openfst/NEWS b/back-ends/openfst/NEWS
index 0152c55..00f292f 100644
--- a/back-ends/openfst/NEWS
+++ b/back-ends/openfst/NEWS
@@ -12,7 +12,7 @@ OpenFst - Release 1.2
* Added SignedLogWeight and SignedLogArc (1.2.4)
* Added ExpectationWeight and ExpectationArc (1.2.4)
* Added AStarQueue, PruneQueue and NaturalPruneQueue disciplines (1.2.6)
- * Added Log64Weight and Log64Arc to FST library throughout, including
+ * Added Log64Weight and Log64Arc to FST library throughout, including
support throughout scripts/bins/dsos (1.2.8)
* Added delayed RandGenFst that outputs tree of paths weighted
by count (1.2.8)
@@ -24,13 +24,13 @@ OpenFst - Release 1.2
* Made "fstmap" handle semiring conversion by adding "to_std", "to_log"
and "to_log64" as supported 'map_type' arguments (1.2.8).
* Made the destructive implementation of RmEpsilon skip over states
- admitting no non-epsilon incoming transition (1.2.8).
+ admitting no non-epsilon incoming transition (1.2.8).
* Fixed numerous bugs (1.2 through 1.2.9) including:
- improper types of some approximation deltas
- sub-optimal hashing functions
- issues in internal reuse of shortest distance
- - hashing bug in FloatWeight
- - bug in shortest path queue
+ - hashing bug in FloatWeight
+ - bug in shortest path queue
- symbol table checksumming issues
- various C++ standards issues
- Visit() behavior when visitation aborted
@@ -38,7 +38,7 @@ OpenFst - Release 1.2
- EditFst::Copy(bool) method when the boolean parameter is true (1.2.7)
- SymbolTable memory leak in Invert() (1.2.8)
- Added escaping of " and \ in labels in fstdraw, needed for dot to
- function properly (1.2.8)
+ function properly (1.2.8)
- Fixed handling of final weight of start state in fstpush (1.2.8)
- Added FST_LL_FORMAT to fix 64-bit integer printf issues (1.2.9)
- Fixed missing <functional> includes (1.2.9)
@@ -53,10 +53,10 @@ OpenFst - Release 1.2
- pushdown transducers (improved in 1.2.1 through 1.2.7).
* Added StateMap/StateMapFst; renamed Map/MapFst to ArcMap/ArcMapFst;
map/MapFst retained (but deprecated) (1.2.9)
- * Deleted ArcSum() and ArcMerge; use StateMap w/ ArcSumMapper and
+ * Deleted ArcSum() and ArcMerge; use StateMap w/ ArcSumMapper and
ArcUniqueMapper (1.2.9).
* Incremented version of ConstFst/CompactFsts to stop memory alignment
- that fails on pipes. Made old version raises errors when read on
+ that fails on pipes. Made old version raises errors when read on
pipes (1.2.9).
* Improved determinize hash (1.2.9)
* Removed stdio uses (1.2.10)
@@ -64,7 +64,7 @@ OpenFst - Release 1.2
OpenFst - Release 1.1
* Added compat.h to src/include/fst to fix missing defines
- * Fixed bug in acyclic minimization that led to non-minimal
+ * Fixed bug in acyclic minimization that led to non-minimal
(but equiv) results
* Fixed missing FST typedef in various matchers in matcher.h
so that they can be cascaded
@@ -78,14 +78,14 @@ OpenFst - Release 1.0
special matchers for sigma (any), rho ('rest'), and phi ('fail')
labels. (see matcher.h)
* Composition generalized with arbitrary filters, matchers, and state tables.
- Sequence and matching composition filters provided. (see compose.h,
+ Sequence and matching composition filters provided. (see compose.h,
compose-filter.h, matcher.h, state-table.h)
* Unique n-best (see shortest-path.h)
- * Pruning in determinization and epsilon removal (see determinize.h,
+ * Pruning in determinization and epsilon removal (see determinize.h,
rmepsilon.h)
- * New Fst classes:
+ * New Fst classes:
* Compact Fsts for space-efficient representation (see compact-fst.h)
- * New Weight classes:
+ * New Weight classes:
* MinMax
* Lexicographic
* Miscellaneous bug fixes
diff --git a/back-ends/openfst/README b/back-ends/openfst/README
index d491e6c..8ca5607 100644
--- a/back-ends/openfst/README
+++ b/back-ends/openfst/README
@@ -4,15 +4,15 @@ OpenFst is a library for constructing, combining, optimizing, and
searching weighted finite-state transducers (FSTs).
REQUIREMENTS:
- This version is known to work under Linux, MacOS, and Solaris using
- g++ (>= 4.1). Expected to work wherever adequate POSIX (dlopen, ssize_t,
- basename), c99 (snprintf, strtoll, <stdint.h>), and tr1 (<tr1/ordered_set>,
+ This version is known to work under Linux, MacOS, and Solaris using
+ g++ (>= 4.1). Expected to work wherever adequate POSIX (dlopen, ssize_t,
+ basename), c99 (snprintf, strtoll, <stdint.h>), and tr1 (<tr1/ordered_set>,
<tr1/ordered_map>) support are available.
INSTALLATION:
Follow the generic GNU build system instructions in ./INSTALL. We
recommend configuring with =--enable-static=no= for faster
- compiles.
+ compiles.
Optional features:
--enable-bin Build fst::script and command-line binaries (def: yes)
@@ -32,13 +32,13 @@ INSTALLATION:
USAGE:
Assuming you've installed under the default /usr/local, the FST
binaries are found on /usr/local/bin. To use in your own program,
- include <fst/fstlib.h> and compile with '-I /usr/local/include'.
+ include <fst/fstlib.h> and compile with '-I /usr/local/include'.
Link against /usr/local/lib/libfst.so and -ldl. Set your
LD_LIBRARY_PATH (or equivalent) to contain /usr/local/lib. The
linking is, by default, dynamic so that the Fst and Arc type DSO
extensions can be used correctly if desired. Any extensions will
be found under /usr/local/include/fst/extensions and /usr/local/lib/fst.
-DOCUMENTATION:
+DOCUMENTATION:
See www.openfst.org for general documentation.
See ./NEWS for updates since the last release.
diff --git a/back-ends/openfst/src/include/fst/config.h b/back-ends/openfst/src/include/fst/config.h
index fa230ce..c049029 100644
--- a/back-ends/openfst/src/include/fst/config.h
+++ b/back-ends/openfst/src/include/fst/config.h
@@ -1,5 +1,5 @@
/* src/include/fst/config.h. Generated from config.h.in by configure. */
-// OpenFst config file
+// OpenFst config file
/* Define to 1 if you have the ICU library. */
/* #undef HAVE_ICU */
diff --git a/back-ends/openfst/src/include/fst/config.h.in b/back-ends/openfst/src/include/fst/config.h.in
index 7815dfc..50d111d 100644
--- a/back-ends/openfst/src/include/fst/config.h.in
+++ b/back-ends/openfst/src/include/fst/config.h.in
@@ -1,4 +1,4 @@
-// OpenFst config file
+// OpenFst config file
/* Define to 1 if you have the ICU library. */
#undef HAVE_ICU
diff --git a/back-ends/openfst/src/include/fst/encode.h b/back-ends/openfst/src/include/fst/encode.h
index 0f77535..ae65616 100644
--- a/back-ends/openfst/src/include/fst/encode.h
+++ b/back-ends/openfst/src/include/fst/encode.h
@@ -178,7 +178,7 @@ template <class A> class EncodeTable {
static EncodeTable<A> *Read(istream &strm, const string &source);
- const uint32 flags() const { return flags_ & kEncodeFlags; }
+ /*const*/ uint32 flags() const { return flags_ & kEncodeFlags; }
int RefCount() const { return ref_count_.count(); }
int IncrRefCount() { return ref_count_.Incr(); }
@@ -359,8 +359,8 @@ template <class A> class EncodeMapper {
return props & mask;
}
- const uint32 flags() const { return flags_; }
- const EncodeType type() const { return type_; }
+ /*const*/ uint32 flags() const { return flags_; }
+ /*const*/ EncodeType type() const { return type_; }
const EncodeTable<A> &table() const { return *table_; }
bool Write(ostream &strm, const string& source) {
diff --git a/back-ends/openfst/src/include/fst/generic-register.h b/back-ends/openfst/src/include/fst/generic-register.h
index 4f8b512..a3642cc 100644
--- a/back-ends/openfst/src/include/fst/generic-register.h
+++ b/back-ends/openfst/src/include/fst/generic-register.h
@@ -72,6 +72,7 @@ class GenericRegister {
// Override this if you want to be able to load missing definitions from
// shared object files.
virtual EntryType LoadEntryFromSharedObject(const KeyType &key) const {
+#if ENABLE_LOAD_SO_ENTRIES
string so_filename = ConvertKeyToSoFilename(key);
void *handle = dlopen(so_filename.c_str(), RTLD_LAZY);
@@ -90,6 +91,9 @@ class GenericRegister {
return EntryType();
}
return *entry;
+#else
+ return EntryType();
+#endif // ENABLE_LOAD_SO_ENTRIES
}
// Override this to define how to turn a key into an SO filename.
diff --git a/back-ends/openfst/src/include/fst/interval-set.h b/back-ends/openfst/src/include/fst/interval-set.h
index ab64020..017c792 100644
--- a/back-ends/openfst/src/include/fst/interval-set.h
+++ b/back-ends/openfst/src/include/fst/interval-set.h
@@ -81,7 +81,7 @@ class IntervalSet {
const vector<Interval> *Intervals() const { return &intervals_; }
- const bool Empty() const { return intervals_.empty(); }
+ /*const*/ bool Empty() const { return intervals_.empty(); }
const T Size() const { return intervals_.size(); }
diff --git a/back-ends/openfst/src/include/fst/minimize.h b/back-ends/openfst/src/include/fst/minimize.h
index 0400f99..877fcab 100644
--- a/back-ends/openfst/src/include/fst/minimize.h
+++ b/back-ends/openfst/src/include/fst/minimize.h
@@ -378,7 +378,7 @@ class AcyclicMinimizer {
const vector<StateId>& height() const { return height_; }
- const size_t num_states() const { return num_states_; }
+ /*const*/ size_t num_states() const { return num_states_; }
private:
vector<StateId> height_;
diff --git a/back-ends/openfst/src/include/fst/partition.h b/back-ends/openfst/src/include/fst/partition.h
index dcee67b..c893811 100644
--- a/back-ends/openfst/src/include/fst/partition.h
+++ b/back-ends/openfst/src/include/fst/partition.h
@@ -217,7 +217,7 @@ class Partition {
return class_size_;
}
- const size_t class_size(T class_id) const {
+ /*const*/ size_t class_size(T class_id) const {
return class_size_[class_id];
}
diff --git a/back-ends/openfst/src/lib/compat.cc b/back-ends/openfst/src/lib/compat.cc
index d365cfb..c9151ad 100644
--- a/back-ends/openfst/src/lib/compat.cc
+++ b/back-ends/openfst/src/lib/compat.cc
@@ -34,7 +34,7 @@ void SplitToVector(char* full, const char* delim, vector<char*>* vec,
bool omit_empty_strings) {
char *p = full;
while (p) {
- if (p = strpbrk(full, delim))
+ if ((p = strpbrk(full, delim)))
p[0] = '\0';
if (!omit_empty_strings || full[0] != '\0')
vec->push_back(full);
diff --git a/back-ends/openfstwin/README b/back-ends/openfstwin/README
index 79d54d9..e80add4 100644
--- a/back-ends/openfstwin/README
+++ b/back-ends/openfstwin/README
@@ -1,2 +1,2 @@
-This directory contains files from OpenFst library
+This directory contains files from OpenFst library
(version 1.3.2 for Windows).
diff --git a/back-ends/openfstwin/src/include/fst/compat.h b/back-ends/openfstwin/src/include/fst/compat.h
index 946cc92..9cd82d7 100644
--- a/back-ends/openfstwin/src/include/fst/compat.h
+++ b/back-ends/openfstwin/src/include/fst/compat.h
@@ -23,7 +23,9 @@
#ifdef _MSC_VER //AddedPD
#include <BaseTsd.h>
typedef SSIZE_T ssize_t;
+#if _MSC_VER < 1900
#define snprintf _snprintf
+#endif
#define strtoll _strtoi64
#ifndef OPENFSTEXPORT
@@ -39,7 +41,7 @@ typedef SSIZE_T ssize_t;
#pragma comment (lib, "openfst64.lib")
#else
#pragma comment (lib, "openfst.lib")
- #endif
+ #endif
#endif
#endif
#endif
diff --git a/back-ends/openfstwin/src/include/fst/config.h b/back-ends/openfstwin/src/include/fst/config.h
index d894e13..9cdbc77 100644
--- a/back-ends/openfstwin/src/include/fst/config.h
+++ b/back-ends/openfstwin/src/include/fst/config.h
@@ -1,4 +1,4 @@
-#ifdef _MSC_VER
+#ifdef _MSC_VER
#pragma warning(disable:4018 4244 4996 4099 4503 4800 4305 4355 4396 4715 4180 4244 4224 4267)
#define HAVE_STD__TR1__HASH_LONG_LONG_UNSIGNED_ 1
#define WIN32_LEAN_AND_MEAN
diff --git a/back-ends/openfstwin/src/include/fst/flags.h b/back-ends/openfstwin/src/include/fst/flags.h
index 0d413df..4416402 100644
--- a/back-ends/openfstwin/src/include/fst/flags.h
+++ b/back-ends/openfstwin/src/include/fst/flags.h
@@ -39,16 +39,16 @@ using std::string;
//#define IMPORT __declspec(dllexport)
//#endif
#ifdef OPENFSTEXPORT
- #define OPENFSTDLL __declspec(dllexport)
+ #define OPENFSTDLL __declspec(dllexport)
#else
#ifdef OPENFSTIMPORT
#define OPENFSTDLL __declspec(dllimport)
#else
- #define OPENFSTDLL
+ #define OPENFSTDLL
#endif
#endif
#else
- #define OPENFSTDLL
+ #define OPENFSTDLL
#endif
//
// FLAGS USAGE:
diff --git a/back-ends/openfstwin/src/include/fst/replace.h b/back-ends/openfstwin/src/include/fst/replace.h
index fe4a373..4a8302d 100644
--- a/back-ends/openfstwin/src/include/fst/replace.h
+++ b/back-ends/openfstwin/src/include/fst/replace.h
@@ -1073,7 +1073,7 @@ class ArcIterator< ReplaceFst<A, T> > {
#else
(fst_.GetImpl())->template CacheImpl<A>::InitArcIterator(state_,
&cache_data_);
- #endif
+ #endif
num_arcs_ = cache_data_.narcs;
arcs_ = cache_data_.arcs; // 'arcs_' is a ptr to the cached arcs.
data_flags_ = kArcValueFlags; // All the arc member values are valid.
diff --git a/back-ends/openfstwin/src/lib/compat.cc b/back-ends/openfstwin/src/lib/compat.cc
index 4c433fe..44b6689 100644
--- a/back-ends/openfstwin/src/lib/compat.cc
+++ b/back-ends/openfstwin/src/lib/compat.cc
@@ -43,7 +43,7 @@ void SplitToVector(char* full, const char* delim, vector<char*>* vec,
}
//AddedPD
-void UTF8ToUTF16(const string& utf8, wstring* utf16) {
+void UTF8ToUTF16(const string& utf8, wstring* utf16) {
utf16->clear();
utf16->reserve(utf8.size());
for ( size_t i = 0; i < utf8.size(); ++i ) {
@@ -53,13 +53,13 @@ void UTF8ToUTF16(const string& utf8, wstring* utf16) {
} else {
if ((ch0 & 0xe0) == 0xc0) {
unsigned char ch1 = utf8[++i];
- utf16->push_back(((ch0 & 0x3f) << 6)|((ch1 & 0x3f)));
+ utf16->push_back(((ch0 & 0x3f) << 6)|((ch1 & 0x3f)));
} else {
unsigned char ch1 = utf8[++i];
unsigned char ch2 = utf8[++i];
utf16->push_back(((ch0 & 0x0f)<<12)|((ch1 & 0x3f)<<6)|((ch2 & 0x3f)));
}
}
- }
+ }
}
} // namespace fst
diff --git a/back-ends/sfst/alphabet.cc b/back-ends/sfst/alphabet.cc
index c31cedd..7203815 100644
--- a/back-ends/sfst/alphabet.cc
+++ b/back-ends/sfst/alphabet.cc
@@ -54,7 +54,7 @@ namespace SFST {
Alphabet::Alphabet()
- {
+ {
utf8 = false;
add(EpsilonString, Label::epsilon);
}
@@ -251,7 +251,7 @@ namespace SFST {
/* */
/*******************************************************************/
- void Alphabet::write_char( Character c, char *buffer, int *pos,
+ void Alphabet::write_char( Character c, char *buffer, int *pos,
bool with_brackets) const
{
const char *s = code2symbol(c);
@@ -299,7 +299,7 @@ namespace SFST {
/* */
/*******************************************************************/
- void Alphabet::write_label( Label l, char *buffer, int *pos,
+ void Alphabet::write_label( Label l, char *buffer, int *pos,
bool with_brackets ) const
{
Character lc=l.lower_char();
@@ -650,7 +650,7 @@ namespace SFST {
char buffer[BUFFER_SIZE];
Character c;
read_num(&c, sizeof(c), file);
- if (!read_string(buffer, BUFFER_SIZE, file) ||
+ if (!read_string(buffer, BUFFER_SIZE, file) ||
feof(file) || ferror(file))
throw "Error1 occurred while reading alphabet!\n";
@@ -745,8 +745,8 @@ namespace SFST {
// Is it a participle
if (c != Label::epsilon) {
sym = write_char(c);
- if (strcmp(sym,"<OLDORTH>") == 0 ||
- strcmp(sym,"<NEWORTH>") == 0 ||
+ if (strcmp(sym,"<OLDORTH>") == 0 ||
+ strcmp(sym,"<NEWORTH>") == 0 ||
strcmp(sym,"<SUFF>") == 0) {
for( k++; k<ana.size(); k++ )
if ((c = ana[k].lower_char()) != Label::epsilon)
diff --git a/back-ends/sfst/alphabet.h b/back-ends/sfst/alphabet.h
index bb27275..0bdbdf5 100644
--- a/back-ends/sfst/alphabet.h
+++ b/back-ends/sfst/alphabet.h
@@ -92,7 +92,7 @@ namespace SFST {
{ return !(l == *this); };
// comparison operator needed for sorting labels in compact.C
- int operator<( Label l ) const {
+ int operator<( Label l ) const {
if (upper_char() < l.upper_char())
return true;
if (upper_char() > l.upper_char())
@@ -101,7 +101,7 @@ namespace SFST {
return true;
return false;
};
- int operator>( Label l ) const {
+ int operator>( Label l ) const {
if (upper_char() > l.upper_char())
return true;
if (upper_char() < l.upper_char())
@@ -127,7 +127,7 @@ namespace SFST {
// hash function needed to store labels in a hash table
struct label_hash {
size_t operator() ( const Label l ) const {
- return (size_t)l.lower_char() ^
+ return (size_t)l.lower_char() ^
((size_t)l.upper_char() << 16) ^
((size_t)l.upper_char() >> 16);
}
@@ -137,7 +137,7 @@ namespace SFST {
struct label_cmp {
bool operator() ( const Label l1, const Label l2 ) const {
return (l1.lower_char() < l2.lower_char() ||
- (l1.lower_char() == l2.lower_char() &&
+ (l1.lower_char() == l2.lower_char() &&
l1.upper_char() < l2.upper_char()));
}
};
@@ -234,7 +234,7 @@ namespace SFST {
void complement( vector<Character> &sym );
// return the code of the argument symbol
- int symbol2code( const char *s ) const {
+ int symbol2code( const char *s ) const {
SymbolMap::const_iterator p = sm.find(s);
if (p != sm.end()) return p->second;
return EOF;
diff --git a/back-ends/sfst/compact.cc b/back-ends/sfst/compact.cc
index 6576709..c1c1949 100644
--- a/back-ends/sfst/compact.cc
+++ b/back-ends/sfst/compact.cc
@@ -54,7 +54,7 @@ namespace SFST {
/*******************************************************************/
void CompactTransducer::analyze(unsigned int n, vector<Character> &input,
- size_t ipos, CAnalysis &ca,
+ size_t ipos, CAnalysis &ca,
vector<CAnalysis> &analyses )
{
// "n" is the number of the current transducer node/state
@@ -76,8 +76,8 @@ namespace SFST {
// first_arc[n+1]-1 is the number of the last outgoing transition of node n
// first_arc[n+1] is the number of the first outgoing transition of node n+1
unsigned int i;
- for( i=first_arc[n];
- i<first_arc[n+1] && label[i].upper_char() == Label::epsilon;
+ for( i=first_arc[n];
+ i<first_arc[n+1] && label[i].upper_char() == Label::epsilon;
i++)
{
ca.push_back(i);
@@ -90,8 +90,8 @@ namespace SFST {
// scan the next input symbol
if (ipos < input.size()) {
// find the set of arcs with matching upper character in the sorted list
- pair<Label*,Label*>range =
- equal_range(label+i, label+first_arc[n+1], Label(input[ipos]),
+ pair<Label*,Label*>range =
+ equal_range(label+i, label+first_arc[n+1], Label(input[ipos]),
label_less());
unsigned int to = (unsigned int)(range.second - label);
@@ -304,7 +304,8 @@ namespace SFST {
{
size_t n,m;
- fread(&n, sizeof(n), 1, file);
+ if (fread(&n, sizeof(n), 1, file) != 1) // HFST addition: checking return value of fread
+ throw "read_probs: fread failed";
if (fread(&m, sizeof(n), 1, file) != 1 ||
n != node_count() || m != arc_count())
{
@@ -313,7 +314,8 @@ namespace SFST {
}
final_logprob = new float[n];
arc_logprob = new float[m];
- fread(final_logprob, sizeof(float), n, file);
+ if (fread(final_logprob, sizeof(float), n, file) != n) // HFST addition: checking return value of fread
+ throw "read_probs: fread failed";
if (fread(arc_logprob, sizeof(float), n, file) != n) {
fprintf(stderr,"Error: in probability file!\n");
exit(1);
@@ -368,7 +370,7 @@ namespace SFST {
/* */
/*******************************************************************/
- void CompactTransducer::longest_match2(unsigned int n, char *string, int l,
+ void CompactTransducer::longest_match2(unsigned int n, char *string, int l,
CAnalysis &ca, int &bl, CAnalysis &ba)
{
// n: transducer state
@@ -386,8 +388,8 @@ namespace SFST {
// follow the epsilon transitions
unsigned int i;
- for( i=first_arc[n];
- i<first_arc[n+1] && label[i].upper_char() == Label::epsilon;
+ for( i=first_arc[n];
+ i<first_arc[n+1] && label[i].upper_char() == Label::epsilon;
i++)
{
ca.push_back(i);
@@ -401,8 +403,8 @@ namespace SFST {
l += (int)(end - string);
if (c != EOF) {
// find the set of arcs with matching upper character in the sort list
- pair<Label*,Label*>range =
- equal_range(label+i, label+first_arc[n+1], Label((Character)c),
+ pair<Label*,Label*>range =
+ equal_range(label+i, label+first_arc[n+1], Label((Character)c),
label_less());
unsigned int to = (unsigned int)(range.second - label);
for( i=(unsigned)(range.first-label); i<to; i++) {
@@ -490,7 +492,7 @@ namespace SFST {
/* */
/*******************************************************************/
- bool CompactTransducer::train2( char *s, vector<double> &arcfreq,
+ bool CompactTransducer::train2( char *s, vector<double> &arcfreq,
vector<double> &finalfreq )
{
vector<CAnalysis> analyses;
@@ -532,7 +534,7 @@ namespace SFST {
/* */
/*******************************************************************/
- bool CompactTransducer::train( char *s, vector<double> &arcfreq,
+ bool CompactTransducer::train( char *s, vector<double> &arcfreq,
vector<double> &finalfreq )
{
vector<CAnalysis> analyses;
@@ -572,7 +574,7 @@ namespace SFST {
/* */
/*******************************************************************/
- void CompactTransducer::estimate_probs( vector<double> &arcfreq,
+ void CompactTransducer::estimate_probs( vector<double> &arcfreq,
vector<double> &finalfreq )
{
// turn frequencies into probabilities
@@ -596,7 +598,7 @@ namespace SFST {
/* */
/*******************************************************************/
- void CompactTransducer::compute_probs( vector<CAnalysis> &analyses,
+ void CompactTransducer::compute_probs( vector<CAnalysis> &analyses,
vector<double> &prob )
{
prob.resize(analyses.size());
diff --git a/back-ends/sfst/determinise.cc b/back-ends/sfst/determinise.cc
index 25c1cf5..5348f22 100644
--- a/back-ends/sfst/determinise.cc
+++ b/back-ends/sfst/determinise.cc
@@ -31,7 +31,7 @@ namespace SFST {
typedef set<Node*>::iterator iterator;
NodeSet() {};
void add( Node* );
- bool insert(Node *node) {
+ bool insert(Node *node) {
pair<iterator, bool> result = ht.insert(node);
return result.second;
};
@@ -78,7 +78,7 @@ namespace SFST {
private:
struct hashf {
- size_t operator()(const NodeArray *na) const {
+ size_t operator()(const NodeArray *na) const {
size_t key=na->size() ^ na->is_final();
for( size_t i=0; i<na->size(); i++)
key = (key<<1) ^ (size_t)(*na)[i];
@@ -234,7 +234,7 @@ namespace SFST {
/* */
/*******************************************************************/
- static void determinise_node( NodeArray &na, Node *node, Transducer *a,
+ static void determinise_node( NodeArray &na, Node *node, Transducer *a,
NodeMapping &map )
{
node->set_final(na.is_final());
diff --git a/back-ends/sfst/fst.cc b/back-ends/sfst/fst.cc
index 3ccd859..fb8bce6 100644
--- a/back-ends/sfst/fst.cc
+++ b/back-ends/sfst/fst.cc
@@ -279,7 +279,7 @@ namespace SFST {
/* */
/*******************************************************************/
- Transducer::Transducer( istream &is, const Alphabet *a, bool verbose,
+ Transducer::Transducer( istream &is, const Alphabet *a, bool verbose,
bool lexcomments )
: root(), mem()
{
@@ -375,7 +375,7 @@ namespace SFST {
/* */
/*******************************************************************/
- void Transducer::store_symbols(Node *node, SymbolMap &symbol,
+ void Transducer::store_symbols(Node *node, SymbolMap &symbol,
LabelSet &labels)
{
if (!node->was_visited( vmark )) {
@@ -501,7 +501,7 @@ namespace SFST {
/* */
/*******************************************************************/
- void Transducer::enumerate_paths_node( Node *node, vector<Label> &path,
+ void Transducer::enumerate_paths_node( Node *node, vector<Label> &path,
NodeHashSet &previous,
vector<Transducer*> &result )
{
@@ -574,7 +574,7 @@ namespace SFST {
Arc *arc=i;
Label l=arc->label();
alphabet.write_label(l, buffer, &p, with_brackets);
- result |= print_strings_node(arc->target_node(), buffer, p,
+ result |= print_strings_node(arc->target_node(), buffer, p,
file, with_brackets );
}
node->set_forward(NULL);
@@ -666,7 +666,7 @@ namespace SFST {
if (!arc->label().is_epsilon())
alphabet.insert(arc->label());
complete(arc->target_node(), alphabet, vmark);
- }
+ }
}
@@ -797,7 +797,7 @@ namespace SFST {
/* */
/*******************************************************************/
- static void store_lowmem_node( FILE *file, Node *node,
+ static void store_lowmem_node( FILE *file, Node *node,
vector<unsigned int> &startpos)
{
store_node_info( file, node );
@@ -885,18 +885,23 @@ namespace SFST {
static void read_node( FILE *file, Node *node, Node **p, Transducer *a )
{
char c;
- fread(&c,sizeof(c),1,file);
+ if (fread(&c,sizeof(c),1,file) != 1) // HFST addition: checking return value of fread
+ throw "read_node: fread failed";
node->set_final(c);
unsigned short n;
- fread( &n, sizeof(n), 1, file);
+ if (fread( &n, sizeof(n), 1, file) != 1) // HFST addition: checking return value of fread
+ throw "read_node: fread failed";
for( int i=0; i<n; i++ ) {
Character lc,uc;
unsigned int t;
- fread(&lc,sizeof(lc),1,file);
- fread(&uc,sizeof(uc),1,file);
- fread(&t,sizeof(t),1,file);
+ if (fread(&lc,sizeof(lc),1,file) != 1) // HFST addition: checking return value of fread
+ throw "read_node: fread failed";
+ if (fread(&uc,sizeof(uc),1,file) != 1) // HFST addition: checking return value of fread
+ throw "read_node: fread failed";
+ if (fread(&t,sizeof(t),1,file) != 1) // HFST addition: checking return value of fread
+ throw "read_node: fread failed";
if (ferror(file))
throw "Error encountered while reading transducer from file";
if (p[t])
@@ -924,7 +929,8 @@ namespace SFST {
vmark = deterministic = 0;
unsigned int n;
- fread(&n,sizeof(n),1,file); // number of nodes
+ if (fread(&n,sizeof(n),1,file) != 1) // number of nodes // HFST addition: checking return value of fread
+ throw "read_transducer_binary: fread failed";
if (ferror(file))
throw "Error encountered while reading transducer from file";
@@ -952,7 +958,7 @@ namespace SFST {
{
static char message[1000];
- sprintf(message, "Error: in line %u of text transducer file",
+ sprintf(message, "Error: in line %u of text transducer file",
(unsigned int)line);
throw message;
}
@@ -1101,7 +1107,7 @@ namespace SFST {
}
else
return it->second;
- }
+ }
/*******************************************************************/
@@ -1127,7 +1133,7 @@ namespace SFST {
if (arc.label().is_epsilon()) {
// 'forward', which is originally NULL, is used as a flag
// for detecting epsilon transition loops
- if (search_node->forward() != copy_tr_start_node) {
+ if (search_node->forward() != copy_tr_start_node) {
search_node->set_forward(copy_tr_start_node); // set epsilon flag
if (arc.target_node()->is_final())
copy_tr_start_node->set_final(true);
@@ -1138,7 +1144,7 @@ namespace SFST {
else {
// target node in copy_tr
- Node *copy_tr_end_node =
+ Node *copy_tr_end_node =
node_in_copy_tr(arc.target_node(), copy_tr, mapper);
// add arc to copy_tr
copy_tr_start_node->add_arc( Label(arc.label().lower_char(),
@@ -1176,12 +1182,12 @@ namespace SFST {
// set copy_tr root node final, if needed
if (root_node()->is_final())
copy_tr->root_node()->set_final(true);
- // associate the root_nodes in this and copy_tr
+ // associate the root_nodes in this and copy_tr
// (node indexing for root_node is zero)
mapper[0] = copy_tr->root_node();
copy_nodes(root_node(), copy_tr, copy_tr->root_node(), mapper);
- incr_vmark();
+ incr_vmark();
return *copy_tr;
}
diff --git a/back-ends/sfst/fst.h b/back-ends/sfst/fst.h
index 6918e7a..773cae5 100644
--- a/back-ends/sfst/fst.h
+++ b/back-ends/sfst/fst.h
@@ -41,7 +41,7 @@ namespace SFST {
static const Index undef = (Index)(-1);
// data type of the generation counter for transducer traversal
- typedef unsigned short VType;
+ typedef unsigned short VType;
extern int Quiet;
@@ -100,7 +100,7 @@ namespace SFST {
return first_epsilon_arcp != NULL;
};
bool non_epsilon_transition_exists( void ) const {
- return first_arcp != NULL;
+ return first_arcp != NULL;
};
int size( void ) const;
@@ -196,7 +196,7 @@ namespace SFST {
private:
struct hashf {
- size_t operator()(const NodePair p) const {
+ size_t operator()(const NodePair p) const {
return (size_t)p.first ^ (size_t)p.second;
}
};
@@ -244,7 +244,7 @@ namespace SFST {
};
void reverse_node( Node *old_node, Transducer *new_node );
Label recode_label( Label, bool lswitch, bool recode, Alphabet& );
- Node *copy_nodes( Node *n, Transducer *a,
+ Node *copy_nodes( Node *n, Transducer *a,
bool lswitch=false, bool recode=false );
void rec_cat_nodes( Node*, Node* );
void negate_nodes( Node*, Node* );
@@ -259,7 +259,7 @@ namespace SFST {
void splice_nodes(Node*, Node*, Label sl, Transducer*, Transducer*);
void splice_arc( Node*, Node*, Node*, Transducer* );
- void enumerate_paths_node( Node*, vector<Label>&, NodeHashSet&,
+ void enumerate_paths_node( Node*, vector<Label>&, NodeHashSet&,
vector<Transducer*>& );
void replace_char2( Node*, Node*, Character, Character, Transducer* );
Node *create_node( vector<Node*>&, char*, size_t line );
@@ -280,9 +280,9 @@ namespace SFST {
Alphabet alphabet; // The set of all labels, i.e. character pairs
- Transducer( bool empty=false ) : root(), mem() {
- vmark = 0;
- deterministic = minimised = empty;
+ Transducer( bool empty=false ) : root(), mem() {
+ vmark = 0;
+ deterministic = minimised = empty;
indexed = false;
node_count = transition_count = 0;
};
@@ -292,7 +292,7 @@ namespace SFST {
// convertion of a string to an transducer
Transducer( char *s, const Alphabet *a=NULL, bool extended=false );
// reads a word list from a file and stores it in the transducer
- Transducer( istream&, const Alphabet *a=NULL, bool verbose=false,
+ Transducer( istream&, const Alphabet *a=NULL, bool verbose=false,
bool lexcomments=false );
// reads a transducer from a binary or text file
Transducer( FILE*, bool binary=true );
@@ -339,7 +339,7 @@ namespace SFST {
Transducer &upper_level( void ) // creates an transducer for the "upper" language
{ return level(upper); };
Transducer &determinise( bool copy_alphabet=true ); // creates a deterministic transducer
- Transducer &rev_det_minimise( bool verbose );
+ Transducer &rev_det_minimise( bool verbose );
Transducer &hopcroft_minimise( bool verbose );
Transducer &minimise( bool verbose=true ) {
if (hopcroft_minimisation)
diff --git a/back-ends/sfst/generate.cc b/back-ends/sfst/generate.cc
index 31bcbc8..54eae74 100644
--- a/back-ends/sfst/generate.cc
+++ b/back-ends/sfst/generate.cc
@@ -23,10 +23,10 @@ namespace SFST {
Index previous;
Label label;
- Gen( Node *n, Label l=Label::epsilon , Index p=undef )
+ Gen( Node *n, Label l=Label::epsilon , Index p=undef )
: node(n), previous(p), label(l) {}
- void print( vector<Gen> &paths, FILE *file, Alphabet &alphabet,
+ void print( vector<Gen> &paths, FILE *file, Alphabet &alphabet,
OutputType ot )
{
if (previous != undef) {
diff --git a/back-ends/sfst/hopcroft.cc b/back-ends/sfst/hopcroft.cc
index 502320a..549e49f 100644
--- a/back-ends/sfst/hopcroft.cc
+++ b/back-ends/sfst/hopcroft.cc
@@ -31,10 +31,10 @@ namespace SFST
Label label;
Transition( Index s, Label l, Index n ) {
- source = s;
- label = l;
- next_for_target = n;
- next_for_label = undef;
+ source = s;
+ label = l;
+ next_for_target = n;
+ next_for_label = undef;
}
};
@@ -50,9 +50,9 @@ namespace SFST
Index first_transition; // index of first transition with this
// state as target
- State() {
+ State() {
group = next_in_group = previous_in_group = undef;
- first_transition = undef;
+ first_transition = undef;
}
};
@@ -62,23 +62,23 @@ namespace SFST
class StateGroup {
public:
- Index next; // index of next source group
+ Index next; // index of next source group
Index next_in_agenda;
Index previous_in_agenda;
Index size; // number of states in this group
Index first_state; // pointer to first state
- Index new_size;
+ Index new_size;
Index first_new_state; // pointer to the set of intersection states
- void init( Index i ) {
+ void init( Index i ) {
next_in_agenda = i;
size = new_size = 0;
next = first_state = first_new_state = undef;
}
bool is_empty() {
- return first_state == undef;
+ return first_state == undef;
}
};
@@ -96,7 +96,7 @@ namespace SFST
public:
Agenda( vector<StateGroup> &g ) : group(g) {
- // allocate some dummy groups for the agenda
+ // allocate some dummy groups for the agenda
g.resize(bucket_count);
for( Index i=0; i<bucket_count; i++ )
group[i].next_in_agenda = group[i].previous_in_agenda = i;
@@ -152,7 +152,7 @@ namespace SFST
Transducer &transducer; // pointer to original transducer
size_t number_of_nodes; // node count in original t.
size_t number_of_transitions; // transition count in original t.
- vector<Node*> nodearray; // maps indices to original transducer nodes
+ vector<Node*> nodearray; // maps indices to original transducer nodes
// CAVEAT: Do not use references to elements of the group vector
// because they become invalid when the group vector is resized.
@@ -493,7 +493,7 @@ namespace SFST
do {
State &S = state[s];
// for all transitions T into S
- for( Index t=S.first_transition; t!=undef;
+ for( Index t=S.first_transition; t!=undef;
t=transition[t].next_for_target )
{
Transition &T = transition[t];
@@ -528,7 +528,7 @@ namespace SFST
first_source_group = undef;
// for all incoming transitions with label l
- for( Index t = first_transition_for_label[l]; t != undef;
+ for( Index t = first_transition_for_label[l]; t != undef;
t = transition[t].next_for_label )
{
// get the transition, source state, and source state group
diff --git a/back-ends/sfst/interface.h b/back-ends/sfst/interface.h
index 38b1094..792609e 100644
--- a/back-ends/sfst/interface.h
+++ b/back-ends/sfst/interface.h
@@ -64,24 +64,24 @@ namespace SFST {
Transducer *pi_machine( Alphabet &alph );
Transducer *cp( Range *lower_range, Range *upper_range );
Transducer *anti_cp( Range *lower_range, Range *upper_range );
- Transducer *twol_right_rule( Transducer *lc, Range *lower_range,
+ Transducer *twol_right_rule( Transducer *lc, Range *lower_range,
Range *upper_range, Transducer *rc);
Transducer *twol_left_rule( Transducer *lc, Range *lower_range,
Range *upper_range, Transducer *rc );
Transducer *restriction_transducer( Transducer *l1, Transducer *l2,
Character marker );
- Transducer *marker_transducer( Transducer *t, Contexts *c,
+ Transducer *marker_transducer( Transducer *t, Contexts *c,
Character &marker );
- Transducer *center_transducer( Transducer *t, Transducer *pi,
+ Transducer *center_transducer( Transducer *t, Transducer *pi,
Transducer *mt );
Transducer *context_transducer( Transducer *t, Transducer *pi,
Transducer *mt, Contexts *c );
Transducer *constrain_boundary_transducer( Character leftm, Character rm );
- Transducer *extended_left_transducer( Transducer *t,
+ Transducer *extended_left_transducer( Transducer *t,
Character m1, Character m2 );
Transducer *left_context( Transducer *t, Character m1, Character m2 );
Transducer *make_optional( Transducer *t, Repl_Type type );
- Transducer *replace_transducer( Transducer *ct, Character lm,
+ Transducer *replace_transducer( Transducer *ct, Character lm,
Character rm, Repl_Type type );
Transducer *result_transducer( Transducer *l1, Transducer *l2,
@@ -141,7 +141,7 @@ namespace SFST {
Transducer *composition( Transducer *a1, Transducer *a2 );
Transducer *restriction( Transducer *a, Twol_Type type, Contexts *c, int );
Transducer *replace( Transducer *a, Repl_Type type, bool optional );
- Transducer *replace_in_context( Transducer *a, Repl_Type type,
+ Transducer *replace_in_context( Transducer *a, Repl_Type type,
Contexts *c, bool optional );
Transducer *negation( Transducer *a );
Transducer *upper_level( Transducer *a );
diff --git a/back-ends/sfst/operators.cc b/back-ends/sfst/operators.cc
index 0cf6146..6d02177 100644
--- a/back-ends/sfst/operators.cc
+++ b/back-ends/sfst/operators.cc
@@ -47,7 +47,7 @@ namespace SFST {
class CharNode2Trans {
struct hashf {
- size_t operator()(const NodeSym &ns) const {
+ size_t operator()(const NodeSym &ns) const {
return ns.nodeID ^ ns.symbol;
}
};
@@ -60,7 +60,7 @@ namespace SFST {
typedef hash_map<NodeSym, FromTo, hashf, equalf > NodeSym2Range;
- // data structure for storing an index from node + symbol to a list
+ // data structure for storing an index from node + symbol to a list
// of transitions with that symbol on the upper/lower layer
Transducer &transducer;
vector<Index> node_size;
@@ -76,7 +76,7 @@ namespace SFST {
CharNode2Trans &c2t;
Index current, end;
public:
- iterator( CharNode2Trans &table, Index nodeID, Character symbol )
+ iterator( CharNode2Trans &table, Index nodeID, Character symbol )
: c2t(table)
{
FromTo range=c2t.trange[NodeSym(nodeID, symbol)];
@@ -157,9 +157,9 @@ namespace SFST {
/* */
/*******************************************************************/
- static bool check_cyclicity( Node *node, NodeHashSet &visited,
+ static bool check_cyclicity( Node *node, NodeHashSet &visited,
const Alphabet &alphabet)
- {
+ {
if (!visited.insert(node).second)
return true; // node was visited before
@@ -233,7 +233,7 @@ namespace SFST {
// iterate over all outgoing arcs
for( ArcsIter p(node->arcs()); p; p++ ) {
Arc *arc=p;
- if (previous.find(arc->target_node()) != previous.end() ||
+ if (previous.find(arc->target_node()) != previous.end() ||
is_cyclic_node( arc->target_node(), previous ))
return true;
}
@@ -397,7 +397,7 @@ namespace SFST {
/* */
/*******************************************************************/
- Label Transducer::recode_label( Label l, bool lswitch, bool recode,
+ Label Transducer::recode_label( Label l, bool lswitch, bool recode,
Alphabet &al )
{
if (lswitch)
@@ -420,7 +420,7 @@ namespace SFST {
/* */
/*******************************************************************/
- Node *Transducer::copy_nodes( Node *node, Transducer *a,
+ Node *Transducer::copy_nodes( Node *node, Transducer *a,
bool lswitch, bool recode )
{
if (!node->was_visited(vmark)) {
@@ -665,7 +665,7 @@ namespace SFST {
/* */
/*******************************************************************/
- static void conjoin_nodes( Node *n1, Node *n2, Node *node,
+ static void conjoin_nodes( Node *n1, Node *n2, Node *node,
Transducer *a, PairMapping &map )
{
@@ -697,7 +697,7 @@ namespace SFST {
conjoin_nodes( t1, t2, target_node, a, map );
}
else {
- // add an arc to the already existing target node
+ // add an arc to the already existing target node
node->add_arc( l, it->second, a );
}
}
@@ -758,9 +758,9 @@ namespace SFST {
/* */
/*******************************************************************/
- static void add_transition( Label l, Node *n1, Node *n2, Node *node,
- Transducer *a, PairMapping &map,
- CharNode2Trans &cn2trans1,
+ static void add_transition( Label l, Node *n1, Node *n2, Node *node,
+ Transducer *a, PairMapping &map,
+ CharNode2Trans &cn2trans1,
CharNode2Trans &cn2trans2 )
{
@@ -771,7 +771,7 @@ namespace SFST {
PairMapping::iterator it=map.find(n1, n2);
if (it != map.end()) {
- // add an arc to the already existing target node
+ // add an arc to the already existing target node
node->add_arc( l, it->second, a );
return;
}
@@ -796,7 +796,7 @@ namespace SFST {
/* */
/*******************************************************************/
- static void compose_nodes( Node *n1, Node *n2, Node *node, Transducer *a,
+ static void compose_nodes( Node *n1, Node *n2, Node *node, Transducer *a,
PairMapping &map, CharNode2Trans &cn2trans1,
CharNode2Trans &cn2trans2 )
{
@@ -837,7 +837,7 @@ namespace SFST {
assert(uc1 == l2.lower_char());
Character uc2=l2.upper_char();
- add_transition( Label(lc1,uc2), t1, t2, node, a, map,
+ add_transition( Label(lc1,uc2), t1, t2, node, a, map,
cn2trans1, cn2trans2 );
}
}
@@ -855,7 +855,7 @@ namespace SFST {
}
}
- else { /* !hash2 */
+ else { /* !hash2 */
// iterate over all outgoing arcs of the second node
for( ArcsIter i(n2->arcs()); i; i++ ) {
Arc *arc2=i;
@@ -878,7 +878,7 @@ namespace SFST {
assert(l1.upper_char() == lc2);
Character lc1=l1.lower_char();
- add_transition( Label(lc1,uc2), t1, t2, node, a, map,
+ add_transition( Label(lc1,uc2), t1, t2, node, a, map,
cn2trans1, cn2trans2 );
}
}
@@ -918,7 +918,7 @@ namespace SFST {
// recursively compose the two automata
CharNode2Trans cn2trans1(*this);
CharNode2Trans cn2trans2(a);
- compose_nodes( root_node(), a.root_node(), na->root_node(),
+ compose_nodes( root_node(), a.root_node(), na->root_node(),
na, map, cn2trans1, cn2trans2 );
return *na;
@@ -1144,7 +1144,7 @@ namespace SFST {
/* */
/*******************************************************************/
- void Transducer::splice_nodes(Node *node, Node *node2, Label sl,
+ void Transducer::splice_nodes(Node *node, Node *node2, Label sl,
Transducer *sa, Transducer *a)
{
if (!node->was_visited(vmark)) {
@@ -1237,7 +1237,7 @@ namespace SFST {
/* */
/*******************************************************************/
- void Transducer::replace_char2(Node *node, Node *node2, Character c,
+ void Transducer::replace_char2(Node *node, Node *node2, Character c,
Character nc, Transducer *a)
{
if (!node->was_visited(vmark)) {
diff --git a/back-ends/sfst/sgi.h b/back-ends/sfst/sgi.h
index 89e5196..e647fc2 100644
--- a/back-ends/sfst/sgi.h
+++ b/back-ends/sfst/sgi.h
@@ -45,7 +45,7 @@
#endif
// Hfst addition
-namespace SFST
+namespace SFST
{
// from <http://gcc.gnu.org/onlinedocs/libstdc++/manual/backwards.html>
#ifdef __GNUC__
@@ -57,7 +57,7 @@ namespace SFST
using std::hash_map;
using std::hash_set;
using std::hash;
-# else
+# else
using __gnu_cxx::hash_map;
using __gnu_cxx::hash_set;
using __gnu_cxx::hash;
diff --git a/configure.ac b/configure.ac
index e2a4a38..2036c7a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -18,8 +18,8 @@
# Information on package
HFST_NAME=hfst
HFST_MAJOR=3
-HFST_MINOR=10
-HFST_EXTENSION=0
+HFST_MINOR=12
+HFST_EXTENSION=1
HFST_VERSION=$HFST_MAJOR.$HFST_MINOR.$HFST_EXTENSION
### When the VERSION is INCREMENTED, REMEMBER to increment the LONGVERSION too.
@@ -27,11 +27,11 @@ HFST_VERSION=$HFST_MAJOR.$HFST_MINOR.$HFST_EXTENSION
# for package-config pc file
LIBHFST_NAME=hfst
LIBHFST_MAJOR=3
-LIBHFST_MINOR=10
-LIBHFST_EXTENSION=0
+LIBHFST_MINOR=12
+LIBHFST_EXTENSION=1
LIBHFST_VERSION=$LIBHFST_MAJOR.$LIBHFST_MINOR.$LIBHFST_EXTENSION
-AC_INIT([hfst], [3.10.0], [hfst-bugs at helsinki.fi], [hfst])
+AC_INIT([hfst], [3.12.2], [hfst-bugs at helsinki.fi], [hfst])
AC_CONFIG_AUX_DIR([build-aux])
AM_INIT_AUTOMAKE([-Wall std-options foreign check-news])
@@ -43,9 +43,9 @@ AC_CONFIG_SRCDIR([libhfst/src/HfstTransducer.h])
AC_CONFIG_HEADERS([config.h libhfst/src/hfst.hpp])
AC_SUBST([LIBHFST_MAJOR], [3])
-AC_SUBST([LIBHFST_MINOR], [10])
-AC_SUBST([LIBHFST_EXTENSION], [0])
-AC_SUBST([LIBHFST_VERSION], [3.10.0])
+AC_SUBST([LIBHFST_MINOR], [12])
+AC_SUBST([LIBHFST_EXTENSION], [1])
+AC_SUBST([LIBHFST_VERSION], [3.12.1])
AC_SUBST([LIBHFST_NAME], [hfst])
# long version = version vector cast in base 10000, for automatic comparisons
@@ -56,7 +56,7 @@ AC_SUBST([LIBHFST_NAME], [hfst])
# $LIBHFST_MINOR * 10000 + $LIBHFST_EXTENSION + "L"
# NB! It turned out to be not portable, and can't be used!
-AC_DEFINE([HFST_LONGVERSION], [300100000L],
+AC_DEFINE([HFST_LONGVERSION], [300120001L],
[Define to hfst version vector as long in base 10000])
AC_DEFINE([HFST_REVISION], ["$Revision$"],
[Automatically substitute to configure.ac revision])
@@ -106,10 +106,10 @@ AS_IF([test "x$with_xfsm" != xno], [AC_DEFINE([HAVE_XFSM], [1],
[Define to compile xfsm support in HFST])])
AM_CONDITIONAL([WANT_XFSM], [test x$with_xfsm != xno])
-AS_IF([test "x$with_xfsm" != "xno"],
+AS_IF([test "x$with_xfsm" != "xno"],
[AC_CHECK_LIB([xfsm], [main], [], AC_MSG_ERROR([xfsm back-end requested but libxfsm not found]))])
-AS_IF([test "x$with_xfsm" != "xno"],
+AS_IF([test "x$with_xfsm" != "xno"],
[AC_CHECK_HEADER([xfsm/xfsm_api.h], [xfsmheader_exists=yes], AC_MSG_ERROR([xfsm back-end requested but libxfsm header not found]))])
### Add here your library ###
@@ -146,10 +146,19 @@ AC_ARG_WITH([readline],
[with_readline=$withval],
[with_readline=no])
AM_CONDITIONAL([WANT_READLINE], [test x$with_readline != xno])
+# make it possible to disable loading entries from shared object (openfst back-end feature),
+# since this adds a dependecy on dl library
+AC_ARG_ENABLE([load_so_entries],
+ [AS_HELP_STRING([--enable-load-so-entries],
+ [enable loading entries from shared objects (openfst feature) @<:@default=yes@:>@])],
+ [enable_load_so_entries=$withval],
+ [enable_load_so_entries=yes])
+AS_IF([test "x$enable_load_so_entries" != xno], [AC_DEFINE([ENABLE_LOAD_SO_ENTRIES], [1],
+ [Define to compile with so entry loading enabled])])
AC_ARG_ENABLE([all_tools],
[AS_HELP_STRING([--enable-all-tools],
- [build all tools @<:@default=no@:>@])],
+ [build all tools (except hfst-twolc-script and hfst-train-tagger which must be explicitly enabled with --enable-twolc-script and --enable-train-tagger) @<:@default=no@:>@])],
[enable_all_tools=$enableval],
[enable_all_tools=no])
@@ -158,7 +167,7 @@ AC_ARG_ENABLE([no_tools],
[build no tools @<:@default=no@:>@])],
[enable_no_tools=$enableval],
[enable_no_tools=no])
-
+AM_CONDITIONAL([ENABLE_NO_TOOLS], [test x$enable_no_tools != xno])
# allow programs to be built separately (sort alphabetically)
AC_ARG_ENABLE([affix_guessify],
@@ -169,7 +178,7 @@ AC_ARG_ENABLE([affix_guessify],
AM_CONDITIONAL([WANT_AFFIX_GUESSIFY], [test x$enable_affix_guessify != xno])
AC_ARG_ENABLE([calculate],
[AS_HELP_STRING([--enable-calculate],
- [build calculate tool @<:@default=no@:>@])],
+ [build calculate tool @<:@default=yes@:>@])],
[enable_calculate=$enableval],
[enable_calculate=$enable_all_tools])
AM_CONDITIONAL([WANT_CALCULATE], [test x$enable_calculate != xno])
@@ -305,12 +314,24 @@ AC_ARG_ENABLE([twolc],
[enable_twolc=$enableval],
[if test x$enable_no_tools != xno; then enable_twolc=no; else enable_twolc=yes; fi])
AM_CONDITIONAL([WANT_TWOLC], [test "x$enable_twolc" != xno])
+AC_ARG_ENABLE([twolc-script],
+ [AS_HELP_STRING([--enable-twolc-script],
+ [build script implementation of twolc @<:@default=no@:>@])],
+ [enable_twolc_script=$enableval],
+ [enable_twolc_script=no])
+AM_CONDITIONAL([WANT_TWOLC_SCRIPT], [test "x$enable_twolc_script" != xno])
AC_ARG_ENABLE([tagger],
[AS_HELP_STRING([--enable-tagger],
[build tagger utilities @<:@default=no@:>@])],
[enable_tagger=$enableval],
[enable_tagger=$enable_all_tools])
AM_CONDITIONAL([WANT_TAGGER], [test "x$enable_tagger" != xno])
+AC_ARG_ENABLE([train-tagger],
+ [AS_HELP_STRING([--enable-train-tagger],
+ [build tagger training utilities @<:@default=no@:>@])],
+ [enable_train_tagger=$enableval],
+ [enable_train_tagger=no])
+AM_CONDITIONAL([WANT_TRAIN_TAGGER], [test "x$enable_train_tagger" != xno])
AC_ARG_ENABLE([lexc],
[AS_HELP_STRING([--enable-lexc],
[build lexc cli @<:@default=yes@:>@])],
@@ -395,12 +416,6 @@ AC_ARG_ENABLE([regexp2fst],
[enable_regexp2fst=$enableval],
[if test x$enable_no_tools != xno; then enable_regexp2fst=no; else enable_regexp2fst=yes; fi])
AM_CONDITIONAL([WANT_REGEXP2FST], [test x$enable_regexp2fst != xno])
-AC_ARG_ENABLE([xfst],
- [AS_HELP_STRING([--enable-xfst],
- [build xfst tool @<:@default=no@:>@])],
- [enable_xfst=$enableval],
- [enable_xfst=$enable_all_tools])
-AM_CONDITIONAL([WANT_XFST], [test x$enable_xfst != xno])
AC_ARG_ENABLE([pmatch2fst],
[AS_HELP_STRING([--enable-pmatch2fst],
[build pmatch2fst tool @<:@default=yes@:>@])],
@@ -505,14 +520,14 @@ AC_ARG_ENABLE([foma_wrapper],
AM_CONDITIONAL([WANT_FOMA_WRAPPER], [test x$enable_foma_wrapper != xno])
AC_ARG_ENABLE([xfst],
[AS_HELP_STRING([--enable-xfst],
- [build xfst tool @<:@default=no@:>@])],
+ [build xfst tool @<:@default=yes@:>@])],
[enable_xfst=$enableval],
[enable_xfst=$enable_all_tools])
AM_CONDITIONAL([WANT_XFST], [test x$enable_xfst != xno])
# Check if zlib is available
AC_CHECK_LIB([z], [main], [zlib_exists=yes], [zlib_exists=no])
-# Check if foma and lexc wrappers are wanted and possible to generate
+# Check if foma and lexc wrappers are wanted and possible to generate
AM_CONDITIONAL([GENERATE_FOMA_WRAPPER], [test "x$enable_foma_wrapper" != "xno" -a "x$with_foma" != "xno" -a "x$zlib_exists" != "xno"])
AM_CONDITIONAL([GENERATE_LEXC_WRAPPER], [test "x$enable_lexc_wrapper" != "xno" -a "x$with_foma" != "xno" -a "x$zlib_exists" != "xno"])
# The same as variables..
@@ -571,7 +586,7 @@ AC_DEFINE_UNQUOTED([AUTOMAKE_VERSION], [$automake_version], [Defines automake ve
if test "$automake_version" \< "1.12" ; then
#Use *.h extension for parser header file
yacc_use_parser_h_extension=true
- echo "automake version < 1.12, using .h extension for yacc/bison generated header files"
+ # issue a warning at the end of configure
AC_DEFINE([YACC_USE_PARSER_H_EXTENSION], [1], [Use *.h extension for parser header file])
else
echo "automake version >= 1.12, using .hh extension for yacc/bison generated header files"
@@ -598,10 +613,10 @@ AM_CONDITIONAL([CAN_DOXYGEN], [test x$DOXYGEN != xno])
# Checks for libraries
AC_LANG_PUSH([C++])
-AS_IF([test "x$with_openfst" != "xno" -a "x$enable_mingw" == "xno"],
+AS_IF([test "x$with_openfst" != "xno" -a "x$enable_mingw" == "xno" -a "x$enable_load_so_entries" != "xno"],
[AC_CHECK_LIB([dl], [main])])
-AS_IF([test "x$with_openfst" != "xno"],
+AS_IF([test "x$with_openfst" != "xno"],
[AC_CHECK_LIB([pthread], [main])]
[AC_CHECK_LIB([m], [main])])
@@ -629,12 +644,19 @@ AC_LANG_POP
AS_IF([test "x$with_unicode_handler" = "xglib"],
[AC_CHECK_LIB([glib-2.0], [main])
+if test x$ac_cv_lib_glib_2_0_main != xyes ; then
+ AC_MSG_FAILURE([--with-unicode-handler=glib requested but glib-2.0 not found])
+fi
AC_DEFINE([USE_GLIB_UNICODE], 1, [if using glib for unicode string handling])])
AS_IF([test "x$with_unicode_handler" = "xicu"],
[AC_MSG_FAILURE([ICU not yet implemented (--with-unicode-handler=hfst to disable)])])
+AS_IF([test "x$with_unicode_handler" = "xyes"],
+ [AC_MSG_FAILURE([--with-unicode-handler=yes; unicode handler 'yes' not recognized])])
+AS_IF([test "x$with_unicode_handler" = "xno"],
+ [AC_MSG_FAILURE([--with-unicode-handler=no; unicode handler 'no' not recognized])])
AS_IF([test "x$with_readline" = "xyes"],
- [AC_SEARCH_LIBS([readline], [readline editline],
- [AC_DEFINE([HAVE_READLINE], 1, [if readline can be used])],
+ [AC_SEARCH_LIBS([readline], [readline editline],
+ [AC_DEFINE([HAVE_READLINE], 1, [if readline can be used])],
[AC_MSG_FAILURE([readline test failed (--without-readline to disable)])])])
### Add here your library ###
@@ -648,7 +670,11 @@ AC_CHECK_HEADERS([limits.h stdlib.h string.h error.h glob.h locale.h langinfo.h]
AC_LANG_PUSH([C++])
# Always use c++11 (flag 'c++0x' is understood also by older compilers)
-AS_IF([echo "$CXXFLAGS" | grep '\-std' | grep -E '((gnu\+\+)|(c\+\+))((11)|(0x)|(14)|(1y)|(17)|(1z))' > /dev/null 2> /dev/null], [], [CXXFLAGS="$CXXFLAGS -std=c++0x"])
+# Checks for highest supported C++ standard
+# But, clang++ does not accept invalid OpenFST code, so limit to C++11 for now
+AX_CHECK_COMPILE_FLAG([-std=c++11], [CXXFLAGS="$CXXFLAGS -std=c++11"], [
+ AS_IF([echo "$CXXFLAGS" | grep '\-std' | grep -E '((gnu\+\+)|(c\+\+))((11)|(0x)|(14)|(1y)|(17)|(1z))' > /dev/null 2> /dev/null], [], [CXXFLAGS="$CXXFLAGS -std=c++0x"])
+])
AC_CHECK_HEADERS([unordered_map], [], [],
[#ifdef HAVE_UNORDERED_MAP
@@ -685,7 +711,7 @@ AS_IF([test "x$with_unicode_handler" = "xglib"],
[AC_SUBST([GLIB_CPPFLAGS], [`$PKG_CONFIG glib-2.0 --cflags`])
AC_SUBST([GLIB_LIBS], [`$PKG_CONFIG glib-2.0 --libs`])
],
- [AC_CHECK_HEADERS([glib.h], [],
+ [AC_CHECK_HEADERS([glib.h], [],
[AC_MSG_ERROR([glib headers not found; set CPPFLAGS to -I glib install directiories or use other unicode handler instead])])])])
# remove if not needed
AS_IF([test "x$with_unicode_handler" != "xglib"],
@@ -710,8 +736,8 @@ AC_CONFIG_FILES([Makefile doc/Makefile test/Makefile
test/tools/fsmbook-tests/Makefile
libhfst/Makefile libhfst/src/Makefile
libhfst/src/implementations/Makefile
- libhfst/src/parsers/Makefile
- libhfst/hfst.pc
+ libhfst/src/parsers/Makefile
+ libhfst/hfst.pc
tools/Makefile tools/src/Makefile tools/src/hfst-proc/Makefile
tools/src/hfst-twolc/Makefile
tools/src/hfst-twolc/src/Makefile
@@ -797,7 +823,9 @@ cat <<EOF
* tail: $enable_tail
* test: $enable_test
* twolc: $enable_twolc
+ * twolc-script: $enable_twolc_script
* tagger: $enable_tagger
+ * train-tagger: $enable_train_tagger
* txt2fst: $enable_txt2fst
* xfst: $enable_xfst
* unicode parsed in corpus tools with: $with_unicode_handler
@@ -805,13 +833,15 @@ EOF
dnl stick important warnings to bottom
dnl important licensing information
dnl according to GNU GPLv2 strict (not plus) is incompatible with Apache
-dnl hope end users never disable ofst
+dnl hope end users never disable ofst
AS_IF([test "x$with_openfst" == "xno"],
[AC_MSG_WARN([Disabling openfst backend will severely cripple libhfst])])
dnl HFST's case mapping in corpus tools is mainly MES-2 set with other characters
dnl added as we go
AS_IF([test "x$with_unicode_handler" = "xhfst"],
[AC_MSG_WARN([HFST only supports basic unicode handling with limited case mapping tables etc.; for better support consider using glib or ICU --with-unicode-handler])])
+AS_IF([test "x$with_readline" == "xno"],
+ [AC_MSG_WARN([HFST tools will be compiled without readline; editing user input on command line is not supported; for better support consider using --with-readline])])
dnl warn about missing "important" tools
AS_IF([test "x$enable_lexc" == "xno"],
[AC_MSG_WARN([hfst-lexc is not enabled; enable with --enable-lexc])])
@@ -819,9 +849,11 @@ AS_IF([test "x$enable_twolc" == "xno"],
[AC_MSG_WARN([hfst-twolc is not enabled; you will not be able to compile xerox twolc grammars; enable using --enable-twolc])])
AS_IF([test "x$enable_tagger" == "xno"],
- [AC_MSG_WARN([hfst-train-tagger and hfst-tag are not enabled; you will not be able to train and use taggers; enable using --enable-tagger])])
+ [AC_MSG_WARN([hfst-tag is not enabled; you will not be able to use taggers; enable using --enable-tagger])])
AS_IF([test "x$enable_calculate" == "xno"],
[AC_MSG_WARN([hfst-calculate is not enabled; you will not be able to compile SFST-PL scripts; enable using --enable-calculate])])
AS_IF([test "x$enable_xfst" == "xno"],
[AC_MSG_WARN([hfst-xfst is not enabled; you will not be able to compile XFST scripts; enable using --enable-xfst])])
AC_MSG_WARN([Python bindings for HFST are not under autotools; see python/README for instructions about how to build and install them])
+AS_IF([test "x$yacc_use_parser_h_extension" == "xtrue"],
+ [AC_MSG_WARN([automake version < 1.12; using .h extension for yacc/bison generated header files; if you are building with pre-generated files, modifying them will make building fail, because they use .hh extension])])
diff --git a/doc/api_documentation.dox b/doc/api_documentation.dox
index 3f20582..7a90a2f 100644
--- a/doc/api_documentation.dox
+++ b/doc/api_documentation.dox
@@ -3,15 +3,15 @@
A binary HfstTransducer consists of an HFST header (more on HFST headers on the
<a href="https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstTransducerHeader">
-wiki pages</a>) and the transducer of the backend implementation.
-If you want to write backend transducers as such, you can specify it with
+wiki pages</a>) and the transducer of the backend implementation.
+If you want to write backend transducers as such, you can specify it with
the <i>hfst_format</i> argument of HfstOutputStream constructor:
\verbatim
HfstOutputStream(ImplementationType type, bool hfst_format=true);
\endverbatim
-The following piece of code will write an OpenFst transducer
+The following piece of code will write an OpenFst transducer
with tropical weights to standard output:
\verbatim
@@ -23,7 +23,7 @@ with tropical weights to standard output:
An HfstInputStream can also read backend transducers that do not have an
HFST header. If the standard input contains an SFST transducer, the following
piece of code will read it successfully and convert it into an HFST transducer
-of type SFST_TYPE and write it to standard output (with the HFST header
+of type SFST_TYPE and write it to standard output (with the HFST header
included).
\verbatim
@@ -44,11 +44,11 @@ wiki pages</a>.
Foma writes its binary transducers in gzipped format using the gz tools.
However, we experienced problems when trying to write to standard output or
read from standard in with gz tools (foma tools do not write to or read
-from standard streams). So we choose to write, and accordingly read,
-foma transducers unzipped when writing or reading binary HfstTransducers
+from standard streams). So we choose to write, and accordingly read,
+foma transducers unzipped when writing or reading binary HfstTransducers
of FOMA_TYPE. As a result, when we write an HfstTransducer of FOMA_TYPE
-in its plain backend format, the user must zip it themselves before it
-can be used by foma tools. Similarily, a foma transducer must be unzipped
+in its plain backend format, the user must zip it themselves before it
+can be used by foma tools. Similarily, a foma transducer must be unzipped
before it can be read by HFST tools.
Suppose we have written a FOMA_TYPE HfstTransducer and want to use it with
@@ -61,7 +61,7 @@ with the following piece of code:
out << ab;
\endverbatim
-The command
+The command
\verbatim
gzip ab.foma
@@ -79,7 +79,7 @@ The same with command line tools:
An example of the opposite case follows. Suppose we have a foma transducer
'transducer.foma' and want to read it inside an HFST program.
-The name of the file must be appended a .gz extension so that the program
+The name of the file must be appended a .gz extension so that the program
'gunzip' knows it is a zipped file. The commands
\verbatim
@@ -107,7 +107,7 @@ The same with command line tools:
/** \page VersionDifferences.html Differences between versions 2 and 3
- If you need documentation on HFST version 2, you find it
+ If you need documentation on HFST version 2, you find it
<a href="http://hfst.sourceforge.net/hfst2/index.html">here</a>.
The page is no longer updated and it is recommended to update
to version 3.
@@ -119,36 +119,36 @@ The same with command line tools:
<th>HFST version 2</th>
<th>HFST version 3</th>
</tr>
- <tr>
+ <tr>
<td>Namespaces HFST and HWFST</td>
- <td>Namespace #hfst and an
+ <td>Namespace #hfst and an
\link hfst::ImplementationType ImplementationType\endlink argument</td>
</tr>
<tr>
<td>KeyLayer and SymbolLayer</td>
- <td>Functions take a \link hfst::String String\endlink argument,
+ <td>Functions take a \link hfst::String String\endlink argument,
string-to-number mapping
is done internally and automatically</td>
</tr>
<tr>
- <td>TransducerLayer, Rule Layer, CoreExtensionLayer</td>
- <td>\link hfst::HfstTransducer HfstTransducer\endlink,
- namespace \link hfst::rules rules\endlink,
+ <td>TransducerLayer, Rule Layer, CoreExtensionLayer</td>
+ <td>\link hfst::HfstTransducer HfstTransducer\endlink,
+ namespace \link hfst::rules rules\endlink,
class \link hfst::implementations::HfstTransitionGraph HfstTransitionGraph\endlink</td>
</tr>
<tr>
- <td>Functions delete their transducer arguments</td>
- <td>Functions modify their calling object and do not change
+ <td>Functions delete their transducer arguments</td>
+ <td>Functions modify their calling object and do not change
their arguments (unless otherwise said)</td>
</tr>
<tr>
<td>No unknown or identity symbols</td>
- <td>Unknown and identity \link hfst::String symbols\endlink are supported
+ <td>Unknown and identity \link hfst::String symbols\endlink are supported
in all implementation formats</td>
</tr>
- </table>
+ </table>
- <BR>
+ <BR>
*/
@@ -173,7 +173,7 @@ autoreconf -i && ./configure && make && make install
\endverbatim
The command 'make' will compile all the API and command line tools and 'install'
- will install them to /usr/local. If you would rather install in eg. your home directory,
+ will install them to /usr/local. If you would rather install in eg. your home directory,
you can tell ./configure:
\verbatim
@@ -190,7 +190,7 @@ autoreconf -i && ./configure && make && make install
The HFST API is currently implemented with three finite-state libraries,
SFST, OpenFst and foma. The API is designed so that it is relatively easy to
- add a new implementation to it.
+ add a new implementation to it.
There are some places in the code where you must make modifications but they are all
clearly marked in the files inside comments. Most of these modifications
just make HFST aware that there is a new implementation available and are
@@ -201,13 +201,13 @@ autoreconf -i && ./configure && make && make install
This interface is used for cooperation between the different finite-state libraries
and the HFST API that is visible to the end-user.
- We first tell what functions and datatypes your implementation must
+ We first tell what functions and datatypes your implementation must
must offer so that it can be connected as a part of the HFST API.
We then go through the modifications that you must do to the code when adding
your implementation to the HFST API. All these modifications are also indicated in the
code inside comments, so we do not handle each change here but advise
- you to see the files instead. Finally we tell what changes must be done
- in the configuration file.
+ you to see the files instead. Finally we tell what changes must be done
+ in the configuration file.
<BR>
@@ -215,14 +215,14 @@ autoreconf -i && ./configure && make && make install
The directory libhfst/src/implementations contains two files per each
library that is added under HFST. For instance for SFST there are files
- SfstTransducer.h and SfstTransducer.cc.
- If a library offers more than one transducer format, there are separate files
+ SfstTransducer.h and SfstTransducer.cc.
+ If a library offers more than one transducer format, there are separate files
for each format. For example for OpenFst there are files TropicalWeightTransducer.cc and .h and
LogWeightTransducer.cc and .h. Each pair of files contains
three static classes that act as an interface between HFST and the
finite-state library in question. For instance the files FomaTransducer.h and FomaTransducer.cc
contain classes FomaTransducer, FomaInputStream and FomaOutputStream that
- take care of interoperation between the foma library and HFST.
+ take care of interoperation between the foma library and HFST.
The directory contains skeleton files
MyTransducerLibraryTransducer.h and MyTransducerLibraryTransducer.cc.
@@ -233,32 +233,32 @@ autoreconf -i && ./configure && make && make install
These classes act as an interface between HFST and your library.
You should write your implementations to these files.
The assumption is that most of the functionalities are found already in the
- finite-state library that you are using and you just have to modify them a little
- so that thay can be accessed via a standardized interface that works similarily
+ finite-state library that you are using and you just have to modify them a little
+ so that thay can be accessed via a standardized interface that works similarily
for all implementations.
All functions in the skeleton files return a FunctionNotImplementedException
as they have no implementation. When starting to write your own implementation,
- you can also return the same exception for all functions that you have not
+ you can also return the same exception for all functions that you have not
yet implemented.
In the same directory there are files
ConvertTransducerFormat.h ConvertTransducerFormat.cc
that contain functions that convert between HFST's own transducer format,
- hfst::implementations::HfstBasicTransducer, and the transducer formats
- of different implementations. Add here functions that convert between
- HfstBasicTransducer and your transducer class (change MyFst to the name of
+ hfst::implementations::HfstBasicTransducer, and the transducer formats
+ of different implementations. Add here functions that convert between
+ HfstBasicTransducer and your transducer class (change MyFst to the name of
your transducer class, my_namespace to the namespace where it is written and
- "my_transducer_library" to the name of your transducer library
+ "my_transducer_library" to the name of your transducer library
or some other descriptive name):
\verbatim
#if HAVE_MY_TRANSDUCER_LIBRARY
- static HfstBasicTransducer *
+ static HfstBasicTransducer *
my_transducer_library_transducer_to_hfst_basic_transducer
(my_namespace::MyFst * t);
- static my_namespace::MyFst *
+ static my_namespace::MyFst *
hfst_basic_transducer_to_my_transducer_library_transducer
(const HfstBasicTransducer * t);
#endif // HAVE_MY_TRANSDUCER_LIBRARY
@@ -292,10 +292,10 @@ BRIDGE_SRCS=$(MAYBE_SFST) $(MAYBE_OPENFST) $(MAYBE_FOMA) $(MAYBE_HFSTOL) # $(MAY
It needs a new enumerator:
\verbatim
- MY_TRANSDUCER_LIBRARY_TYPE,
+ MY_TRANSDUCER_LIBRARY_TYPE,
\endverbatim
- In file HfstTransducer.h you must include the header file
+ In file HfstTransducer.h you must include the header file
MyTransducerLibraryTransducer.h.
\verbatim
@@ -335,11 +335,11 @@ to TransducerImplementation, the union of possible transducer backend implementa
\endverbatim
In file HfstTransducer.cc, you must define the interface between
- HFST and your transducer library:
+ HFST and your transducer library:
\verbatim
#if HAVE_MY_TRANSDUCER_LIBRARY
- hfst::implementations::MyTransducerLibraryTransducer
+ hfst::implementations::MyTransducerLibraryTransducer
HfstTransducer::my_transducer_library_interface;
#endif
\endverbatim
@@ -348,7 +348,7 @@ to TransducerImplementation, the union of possible transducer backend implementa
\verbatim
#if HAVE_MY_TRANSDUCER_LIBRARY
- hfst::implementations::MyTransducerLibraryInputStream *
+ hfst::implementations::MyTransducerLibraryInputStream *
my_transducer_library;
#endif
\endverbatim
@@ -363,7 +363,7 @@ to TransducerImplementation, the union of possible transducer backend implementa
\verbatim
#if HAVE_MY_TRANSDUCER_LIBRARY
- hfst::implementations::MyTransducerLibraryOutputStream *
+ hfst::implementations::MyTransducerLibraryOutputStream *
my_transducer_library;
#endif
\endverbatim
@@ -394,7 +394,7 @@ to TransducerImplementation, the union of possible transducer backend implementa
You shoud go through carefully files HfstTransducer.cc, HfstInputStream.cc,
HfstOutputStream.cc and HfstApply.cc and add for each functionality
- add a case that calls the implementation of the new finite-state library
+ add a case that calls the implementation of the new finite-state library
if the implementation type requires it. We give here some examples of
the pattern that it used in HFST to handle different cases and choose
the right implementation.
@@ -417,12 +417,12 @@ to TransducerImplementation, the union of possible transducer backend implementa
#endif
#if HAVE_OPENFST
case TROPICAL_OFST_TYPE:
- implementation.tropical_ofst =
+ implementation.tropical_ofst =
tropical_ofst_interface.create_empty_transducer();
this->type = TROPICAL_OFST_TYPE;
break;
case LOG_OFST_TYPE:
- implementation.log_ofst =
+ implementation.log_ofst =
log_ofst_interface.create_empty_transducer();
break;
#endif
@@ -434,7 +434,7 @@ to TransducerImplementation, the union of possible transducer backend implementa
/* Add here your implementation. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//case MY_TRANSDUCER_LIBRARY_TYPE:
- //implementation.my_transducer_library
+ //implementation.my_transducer_library
// = my_transducer_library_interface.create_empty_transducer();
//break;
//#endif
@@ -497,7 +497,7 @@ HfstTransducer &apply(
//#endif
bool dummy /* makes sure there is always a parameter after the function pointer parameters,
* so commas between parameters are easier to handle */
-);
+);
\endverbatim
Then the function 'apply' chooses the right function pointer to use
@@ -506,7 +506,7 @@ HfstTransducer &apply(
\verbatim
HfstTransducer &HfstTransducer::apply(
#if HAVE_SFST
- SFST::Transducer * (*sfst_funct)(SFST::Transducer *),
+ SFST::Transducer * (*sfst_funct)(SFST::Transducer *),
#endif
#if HAVE_OPENFST
fst::StdVectorFst * (*tropical_ofst_funct)(fst::StdVectorFst *),
@@ -527,7 +527,7 @@ HfstTransducer &apply(
#if HAVE_SFST
case SFST_TYPE:
{
- SFST::Transducer * sfst_temp =
+ SFST::Transducer * sfst_temp =
sfst_funct(implementation.sfst);
delete implementation.sfst;
implementation.sfst = sfst_temp;
@@ -645,7 +645,7 @@ namespace hfst {
using namespace hfst;
-int main()
+int main()
{
HfstTransducer tr1("foo", "bar", SFST_TYPE);
HfstTransducer tr2("bar", "baz", SFST_TYPE);
@@ -700,14 +700,14 @@ using namespace hfst;
using implementations::HfstBasicTransducer;
using implementations::HfstBasicTransition;
-/* Create a HFST basic transducer [a:b] with transition weight 0.3 and
+/* Create a HFST basic transducer [a:b] with transition weight 0.3 and
final weight 0.5. */
HfstBasicTransducer t;
t.add_state(1);
t.add_transition(0, HfstBasicTransition(1, "a", "b", 0.3));
t.set_final_weight(1, 0.5);
-/* Convert to tropical OpenFst format and push weights
+/* Convert to tropical OpenFst format and push weights
toward final state. */
HfstTransducer T(t, TROPICAL_OFST_TYPE);
T.push_weights(TO_FINAL_STATE);
@@ -715,7 +715,7 @@ T.push_weights(TO_FINAL_STATE);
/* Convert back to HFST basic transducer. */
HfstBasicTransducer tc(T);
try {
- /* Rounding might affect the precision. */
+ /* Rounding might affect the precision. */
if (0.79 < tc.get_final_weight(1) &&
tc.get_final_weight(1) < 0.81) {
fprintf(stderr, "TEST OK\n");
@@ -723,7 +723,7 @@ try {
}
fprintf(stderr, "TEST FAILED\n");
exit(1);
-}
+}
/* If the state does not exist or is not final */
catch (HfstException e) {
fprintf(stderr, "TEST FAILED: An exception thrown.\n");
@@ -740,7 +740,7 @@ and printing the string pairs recognized by the resulting transducer.
ImplementationType type=FOMA_TYPE;
- /* Create a simple lexicon transducer
+ /* Create a simple lexicon transducer
[[foo bar foo] | [foo bar baz]]. */
HfstTokenizer tok;
@@ -753,7 +753,7 @@ and printing the string pairs recognized by the resulting transducer.
words.disjunct(t);
- /* Create a rule transducer that optionally replaces
+ /* Create a rule transducer that optionally replaces
"bar" with "baz" between "foo" and "foo". */
HfstTransducerPair context
@@ -772,7 +772,7 @@ and printing the string pairs recognized by the resulting transducer.
(context, mapping, optional, alphabet);
- /* Apply the rule transducer to the lexicon. */
+ /* Apply the rule transducer to the lexicon. */
words.compose(rule).minimize();
@@ -783,7 +783,7 @@ and printing the string pairs recognized by the resulting transducer.
try {
words.extract_paths(results);
- }
+ }
catch (TransducerIsCyclicException e)
{
/* This should not happen because transducer is not cyclic. */
@@ -791,7 +791,7 @@ and printing the string pairs recognized by the resulting transducer.
exit(1);
}
- /* Go through all paths. */
+ /* Go through all paths. */
for (HfstTwoLevelPaths::const_iterator it = results.begin();
it != results.end(); it++)
{
@@ -806,8 +806,8 @@ and printing the string pairs recognized by the resulting transducer.
istring.append(IT->first);
ostring.append(IT->second);
}
- fprintf(stdout, "%s : %s\n",
- istring,
+ fprintf(stdout, "%s : %s\n",
+ istring,
ostring);
}
\endverbatim
@@ -822,7 +822,7 @@ and in AT&T format to file "testfile.att":
FILE * file = fopen("testfile.att", "wb");
bool first_transducer=true;
- while (not in.is_eof())
+ while (not in.is_eof())
{
if (not first_transducer)
fprintf(file, "--\n"); /* AT&T format separator. */
@@ -843,20 +843,20 @@ and in AT&T format to file "testfile.att":
}
-/** \mainpage
+/** \mainpage
-HFST - The Helsinki Finite-State Transducer technology is intended for creating and
+HFST - The Helsinki Finite-State Transducer technology is intended for creating and
manipulating weighted or unweighted synchronic transducers implementing regular relations.
UTF-8 is chosen as the character encoding used in the HFST software.
Currently, HFST has been implemented using the
-<a href="http://www.ims.uni-stuttgart.de/projekte/gramotron/SOFTWARE/SFST.html">SFST</a>,
-<a href="http://www.openfst.org">OpenFst</a> and
+<a href="http://www.ims.uni-stuttgart.de/projekte/gramotron/SOFTWARE/SFST.html">SFST</a>,
+<a href="http://www.openfst.org">OpenFst</a> and
<a href="http://code.google.com/p/foma/">foma</a> software libraries.
-Other versions may be added in some future release.
+Other versions may be added in some future release.
SFST and foma implementations are unweighted and OpenFst implementation is weighted.
-Some of the HFST interface functionalities have also been implemented for
+Some of the HFST interface functionalities have also been implemented for
HFST's own two transducer formats, HfstTransitionGraph and optimized lookup format.
The previous is useful for converting between transducer formats and storing transducers
in an implementation-independent format. The latter is used for fast lookup
@@ -864,13 +864,13 @@ of strings in a transducer.
All implementations work according to the same interface, so it is possible
to compile the same piece of code using different backend libraries.
-It is also possible to <a href="AddingYourImplementation.html">add</a>
+It is also possible to <a href="AddingYourImplementation.html">add</a>
your own implementation under the HFST interface.
-For a quick start to the HFST interface with examples, see
+For a quick start to the HFST interface with examples, see
<a href="QuickStart.html">here</a>.
-The examples given in this documentation use
+The examples given in this documentation use
<a href="http://www.fsmbook.com/">Xerox transducer notation</a>.
@@ -911,7 +911,7 @@ The examples given in this documentation use
\section download_hfst Download
- - <a href="http://sourceforge.net/projects/hfst/files/hfst">Download</a> and
+ - <a href="http://sourceforge.net/projects/hfst/files/hfst">Download</a> and
<a href="InstallHfst.html">install</a> the HFST interface and command line tools
(NOTE: we have officially moved to <a href="https://hfst.github.io/index.html">Gihub</a>,
but for the time being the latest release is still available on Sourceforge.)
@@ -927,23 +927,23 @@ The examples given in this documentation use
*/
-/**
- @dir libhfst/src
- @brief A directory for the HFST interface.
+/**
+ @dir libhfst/src
+ @brief A directory for the HFST interface.
The files in this directory contain the public interface
of HFST and internal code that is not implementation-specific.
*/
-/**
- @dir libhfst/src/implementations
+/**
+ @dir libhfst/src/implementations
@brief A directory for the different backend implementations.
The files in this directory act as a bridge between the HFST
interface and the backend libraries.
Basicly this directory contains two files per implementation,
- for example the files SfstTransducer.h and SfstTransducer.cc
- define the SFST implementation.
+ for example the files SfstTransducer.h and SfstTransducer.cc
+ define the SFST implementation.
The implementation of HFST's own transducer format, HfstTransitionGraph
is also placed here.
diff --git a/doc/hfst-lexc.rst b/doc/hfst-lexc.rst
index 83f3a1c..f5a12b9 100644
--- a/doc/hfst-lexc.rst
+++ b/doc/hfst-lexc.rst
@@ -29,7 +29,7 @@ Compilation requirements:
- compiler *must* be able to find hfst headers (e.g. ``hfst2/hfst.h``)
-- linker *must* be able to find library named hfst (e.g. ``libhfst.so``,
+- linker *must* be able to find library named hfst (e.g. ``libhfst.so``,
``libhfst.dylib``, ``libhfst.dll``, or ``libhfst.a``)
- linker *should* be able to find library named readline, if readline interface
@@ -89,7 +89,7 @@ Usage
The executable is typically called ``hfst-lexc``. It can read
lexc files and produce HFST binary transducers. For basic operation you may use
-either ``hfst-lexc -o output.hfst input.hlexc`` or
+either ``hfst-lexc -o output.hfst input.hlexc`` or
``hfst-lexc -q < input.hlexc > output.hfsta``. If readline support was compiled
in, you may also use Xerox lexc lookalike user interface by simply calling
command ``hfst-lexc`` without any arguments. For more options say
@@ -101,7 +101,7 @@ command ``hfst-lexc`` without any arguments. For more options say
-h, --help Print this help message
-V, --version Print version info
--v, --verbose Print verbosely while compiling
+-v, --verbose Print verbosely while compiling
-q, --quiet Do not print verbosely while compiling
-o, --output=OUTFILE Write output to OUTFILE
-w, --weighted Use weighted transducer format
@@ -160,7 +160,7 @@ bug tracking. Following is sorted according in order of error codenames:
* *unexpected ULSTRING, expecting LEXICON_NAME [near: `Nouns']*. This
often means that there is a missing semicolon at the end of lexicon line.
- * *unexpected ';', expecting LEXICON_NAME or ULSTRING or
+ * *unexpected ';', expecting LEXICON_NAME or ULSTRING or
XEROX_REGEXP or ':' [near: `;']*. Usually from extraneous
semicolon at end of lexicon declaration.
diff --git a/doc/hfst3-metadata-header-registry.rst b/doc/hfst3-metadata-header-registry.rst
index d02f814..b50394e 100644
--- a/doc/hfst3-metadata-header-registry.rst
+++ b/doc/hfst3-metadata-header-registry.rst
@@ -33,7 +33,7 @@ metadata header in all files *should* be the version header, so incompatible
parsers can quit parsing as soon as possible without breaking.
name
- The user-readable name for the automata used in user interfaces
+ The user-readable name for the automata used in user interfaces
instead of the file name, if available. Example: *The Finnish
hyphenator of Omorfi dictionary*. A missing name header means
*filename[index]* is used to represent the automaton.
@@ -67,7 +67,7 @@ commandline-definition
The possible reconstruction of commands used to create current transducer.
This can be only as accurate as command-line reconstructions can be, i.e.
things that are parsed by shell but won't be passed to the program, such as
- pipes and redirections, can only be guessed. Example:
+ pipes and redirections, can only be guessed. Example:
*hfst-lexc lexicon.lexc -o lexicon.hfst && hfst-twolc rules.twolc -o
rules.hfst; hfst-compose-intersect lexicon.hfst rules.hfst*.
compression
@@ -75,7 +75,7 @@ compression
automata in the file can be compressed using any of
available algorithms. It should be noted that compressed
automata cannot be used as freely as uncompressed ones and
- their use requires extra libraries. Some compression
+ their use requires extra libraries. Some compression
schemes may not be usable in standard streams at all. This
is used by library to uncompress the data before passing
it to respective libraries for reading.
@@ -103,7 +103,7 @@ dcmi-text
DescriptionSet (
Description (
Statement (
- PropertyURI ( dcterms:title )
+ PropertyURI ( dcterms:title )
LiteralValueString ( "English morphological analyser" )
)
)
@@ -142,7 +142,7 @@ licence
copyrights as rendered by Berne convention, it is only
for informative purposes. Example: *GNU GPLv3*.
purpose
- The automaton's primary use. The corpora and writers'
+ The automaton's primary use. The corpora and writers'
tools use this to enumerate available functionalities.
Currently supported values are: *analysis*, *generation*,
*error model*, *hyphenation*, *phonological rules*,
@@ -154,7 +154,7 @@ revision
system. Example: *73624* (from SVN).
shasum
SHA check sum of the transducer data. This can be used
- to verify transducer integrity. Example:
+ to verify transducer integrity. Example:
*4a7f4849457500fe27585515f6d99924dbf99121*.
xfst-definition
Xerox regular expression defining the automata. Does not
diff --git a/hfst.m4 b/hfst.m4
index 9cfe6ba..44aff56 100644
--- a/hfst.m4
+++ b/hfst.m4
@@ -1,6 +1,6 @@
# hfst.m4 - Macros to locate and utilise hfst library and tools -*- Autoconf -*-
# serial 1 (hfst-3.1.1)
-#
+#
# Copyright © 2011 University of Helsinki <hfst-bugs at helsinki.fi>.
#
# This program is free software; you can redistribute it and/or modify
@@ -28,7 +28,7 @@ dnl HFST switches and variables
AC_CACHE_CHECK([whether pkg-config can find hfst tools], [hfst_cv_prog_hfst],
[hfst_cv_prog_hfst=no
_hfst_min_version=m4_default([$1], [ >= 3.1])
- PKG_CHECK_MODULES([HFST], [hfst$_hfst_min_version],
+ PKG_CHECK_MODULES([HFST], [hfst$_hfst_min_version],
[hfst_cv_prog_hfst=yes], [hfst_cv_prog_hfst=no])])
[]dnl
diff --git a/lgpl-release/COPYING b/lgpl-release/COPYING
new file mode 100644
index 0000000..65c5ca8
--- /dev/null
+++ b/lgpl-release/COPYING
@@ -0,0 +1,165 @@
+ GNU LESSER GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+ This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+ 0. Additional Definitions.
+
+ As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+ "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+ An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+ A "Combined Work" is a work produced by combining or linking an
+Application with the Library. The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+ The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+ The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+ 1. Exception to Section 3 of the GNU GPL.
+
+ You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+ 2. Conveying Modified Versions.
+
+ If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+ a) under this License, provided that you make a good faith effort to
+ ensure that, in the event an Application does not supply the
+ function or data, the facility still operates, and performs
+ whatever part of its purpose remains meaningful, or
+
+ b) under the GNU GPL, with none of the additional permissions of
+ this License applicable to that copy.
+
+ 3. Object Code Incorporating Material from Library Header Files.
+
+ The object code form of an Application may incorporate material from
+a header file that is part of the Library. You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+ a) Give prominent notice with each copy of the object code that the
+ Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the object code with a copy of the GNU GPL and this license
+ document.
+
+ 4. Combined Works.
+
+ You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+ a) Give prominent notice with each copy of the Combined Work that
+ the Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the Combined Work with a copy of the GNU GPL and this license
+ document.
+
+ c) For a Combined Work that displays copyright notices during
+ execution, include the copyright notice for the Library among
+ these notices, as well as a reference directing the user to the
+ copies of the GNU GPL and this license document.
+
+ d) Do one of the following:
+
+ 0) Convey the Minimal Corresponding Source under the terms of this
+ License, and the Corresponding Application Code in a form
+ suitable for, and under terms that permit, the user to
+ recombine or relink the Application with a modified version of
+ the Linked Version to produce a modified Combined Work, in the
+ manner specified by section 6 of the GNU GPL for conveying
+ Corresponding Source.
+
+ 1) Use a suitable shared library mechanism for linking with the
+ Library. A suitable mechanism is one that (a) uses at run time
+ a copy of the Library already present on the user's computer
+ system, and (b) will operate properly with a modified version
+ of the Library that is interface-compatible with the Linked
+ Version.
+
+ e) Provide Installation Information, but only if you would otherwise
+ be required to provide such information under section 6 of the
+ GNU GPL, and only to the extent that such information is
+ necessary to install and execute a modified version of the
+ Combined Work produced by recombining or relinking the
+ Application with a modified version of the Linked Version. (If
+ you use option 4d0, the Installation Information must accompany
+ the Minimal Corresponding Source and Corresponding Application
+ Code. If you use option 4d1, you must provide the Installation
+ Information in the manner specified by section 6 of the GNU GPL
+ for conveying Corresponding Source.)
+
+ 5. Combined Libraries.
+
+ You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+ a) Accompany the combined library with a copy of the same work based
+ on the Library, uncombined with any other library facilities,
+ conveyed under the terms of this License.
+
+ b) Give prominent notice with the combined library that part of it
+ is a work based on the Library, and explaining where to find the
+ accompanying uncombined form of the same work.
+
+ 6. Revised Versions of the GNU Lesser General Public License.
+
+ The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+ If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
diff --git a/lgpl-release/Makefile.am b/lgpl-release/Makefile.am
new file mode 100644
index 0000000..cc89e08
--- /dev/null
+++ b/lgpl-release/Makefile.am
@@ -0,0 +1,28 @@
+## Process this file with automake to produce Makefile.in
+
+# Copyright (c) 2016 University of Helsinki
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+# See the file COPYING included with this distribution for more
+# information.
+
+ACLOCAL_AMFLAGS = -I m4
+
+SUBDIRS = back-ends libhfst python
+
+# for external projects to use HFST from auto*
+m4dir=$(datadir)/aclocal
+m4_DATA=hfst.m4
+
+EXTRA_DIST=hfst.m4
+
+if CAN_DOXYGEN
+doxygen:
+ doxygen Doxyfile
+endif
+valgrind:
+ $(MAKE) -C libhfst/src/ valgrind
+ $(MAKE) -C test/tools/ valgrind
diff --git a/lgpl-release/README-DEV b/lgpl-release/README-DEV
new file mode 100644
index 0000000..ace9534
--- /dev/null
+++ b/lgpl-release/README-DEV
@@ -0,0 +1,13 @@
+
+This directory contains the following alternative files
+
+ ├── back-ends
+ │ └── Makefile.am
+ ├── configure.ac
+ ├── COPYING
+ └── Makefile.am
+
+which are used instead of the default ones when creating a minimal LGPL
+release of HFST. It contains only openfst back-end, HFST library code
+and python bindings. All command line tools and sfst and foma back-ends
+are disabled.
diff --git a/lgpl-release/back-ends/Makefile.am b/lgpl-release/back-ends/Makefile.am
new file mode 100644
index 0000000..0ebb195
--- /dev/null
+++ b/lgpl-release/back-ends/Makefile.am
@@ -0,0 +1,14 @@
+## Process this file with automake to produce Makefile.in
+
+# Copyright (c) 2016 University of Helsinki
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+# See the file COPYING included with this distribution for more
+# information.
+
+SUBDIRS = openfst
+
+AUTOMAKE_OPTIONS=subdir-objects
diff --git a/configure.ac b/lgpl-release/configure.ac
similarity index 91%
copy from configure.ac
copy to lgpl-release/configure.ac
index e2a4a38..0850829 100644
--- a/configure.ac
+++ b/lgpl-release/configure.ac
@@ -1,25 +1,19 @@
## Process this file with autoconf to produce configure script
-## Copyright (C) 2010 University of Helsinki
-
-## This program is free software: you can redistribute it and/or modify
-## it under the terms of the GNU General Public License as published by
-## the Free Software Foundation, either version 3 of the License, or
-## (at your option) any later version.
-
-## This program is distributed in the hope that it will be useful,
-## but WITHOUT ANY WARRANTY; without even the implied warranty of
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-## GNU General Public License for more details.
-
-## You should have received a copy of the GNU General Public License
-## along with this program. If not, see <http://www.gnu.org/licenses/>.
+# Copyright (c) 2016 University of Helsinki
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+# See the file COPYING included with this distribution for more
+# information.
# Information on package
HFST_NAME=hfst
HFST_MAJOR=3
-HFST_MINOR=10
-HFST_EXTENSION=0
+HFST_MINOR=12
+HFST_EXTENSION=1
HFST_VERSION=$HFST_MAJOR.$HFST_MINOR.$HFST_EXTENSION
### When the VERSION is INCREMENTED, REMEMBER to increment the LONGVERSION too.
@@ -27,11 +21,11 @@ HFST_VERSION=$HFST_MAJOR.$HFST_MINOR.$HFST_EXTENSION
# for package-config pc file
LIBHFST_NAME=hfst
LIBHFST_MAJOR=3
-LIBHFST_MINOR=10
-LIBHFST_EXTENSION=0
+LIBHFST_MINOR=12
+LIBHFST_EXTENSION=1
LIBHFST_VERSION=$LIBHFST_MAJOR.$LIBHFST_MINOR.$LIBHFST_EXTENSION
-AC_INIT([hfst], [3.10.0], [hfst-bugs at helsinki.fi], [hfst])
+AC_INIT([hfst], [3.12.1], [hfst-bugs at helsinki.fi], [hfst])
AC_CONFIG_AUX_DIR([build-aux])
AM_INIT_AUTOMAKE([-Wall std-options foreign check-news])
@@ -43,9 +37,9 @@ AC_CONFIG_SRCDIR([libhfst/src/HfstTransducer.h])
AC_CONFIG_HEADERS([config.h libhfst/src/hfst.hpp])
AC_SUBST([LIBHFST_MAJOR], [3])
-AC_SUBST([LIBHFST_MINOR], [10])
-AC_SUBST([LIBHFST_EXTENSION], [0])
-AC_SUBST([LIBHFST_VERSION], [3.10.0])
+AC_SUBST([LIBHFST_MINOR], [12])
+AC_SUBST([LIBHFST_EXTENSION], [1])
+AC_SUBST([LIBHFST_VERSION], [3.12.1])
AC_SUBST([LIBHFST_NAME], [hfst])
# long version = version vector cast in base 10000, for automatic comparisons
@@ -56,7 +50,7 @@ AC_SUBST([LIBHFST_NAME], [hfst])
# $LIBHFST_MINOR * 10000 + $LIBHFST_EXTENSION + "L"
# NB! It turned out to be not portable, and can't be used!
-AC_DEFINE([HFST_LONGVERSION], [300100000L],
+AC_DEFINE([HFST_LONGVERSION], [300120001L],
[Define to hfst version vector as long in base 10000])
AC_DEFINE([HFST_REVISION], ["$Revision$"],
[Automatically substitute to configure.ac revision])
@@ -66,9 +60,9 @@ AC_DEFINE_UNQUOTED([HFST_STRING], ["$PACKAGE_STRING"],
# allow backends to be removed
AC_ARG_WITH([sfst],
[AS_HELP_STRING([--with-sfst],
- [process unweighted fsts with SFST @<:@default=yes@:>@])],
+ [process unweighted fsts with SFST @<:@default=no@:>@])],
[],
- [with_sfst=yes])
+ [with_sfst=no])
AS_IF([test "x$with_sfst" != xno], [AC_DEFINE([HAVE_SFST], [1],
[Define to compile SFST support in HFST])])
AM_CONDITIONAL([WANT_SFST], [test x$with_sfst != xno])
@@ -90,9 +84,9 @@ AS_IF([test "x$with_openfst_log" != xno], [AC_DEFINE([HAVE_OPENFST_LOG], [1],
AM_CONDITIONAL([WANT_OPENFST_LOG], [test x$with_openfst_log != xno])
AC_ARG_WITH([foma],
[AS_HELP_STRING([--with-foma],
- [process unweighted fsts with foma @<:@default=yes@:>@])],
+ [process unweighted fsts with foma @<:@default=no@:>@])],
[],
- [with_foma=yes])
+ [with_foma=no])
AS_IF([test "x$with_foma" != xno], [AC_DEFINE([HAVE_FOMA], [1],
[Define to compile foma support in HFST])])
AM_CONDITIONAL([WANT_FOMA], [test x$with_foma != xno])
@@ -106,10 +100,10 @@ AS_IF([test "x$with_xfsm" != xno], [AC_DEFINE([HAVE_XFSM], [1],
[Define to compile xfsm support in HFST])])
AM_CONDITIONAL([WANT_XFSM], [test x$with_xfsm != xno])
-AS_IF([test "x$with_xfsm" != "xno"],
+AS_IF([test "x$with_xfsm" != "xno"],
[AC_CHECK_LIB([xfsm], [main], [], AC_MSG_ERROR([xfsm back-end requested but libxfsm not found]))])
-AS_IF([test "x$with_xfsm" != "xno"],
+AS_IF([test "x$with_xfsm" != "xno"],
[AC_CHECK_HEADER([xfsm/xfsm_api.h], [xfsmheader_exists=yes], AC_MSG_ERROR([xfsm back-end requested but libxfsm header not found]))])
### Add here your library ###
@@ -155,10 +149,10 @@ AC_ARG_ENABLE([all_tools],
AC_ARG_ENABLE([no_tools],
[AS_HELP_STRING([--enable-no-tools],
- [build no tools @<:@default=no@:>@])],
+ [build no tools @<:@default=yes@:>@])],
[enable_no_tools=$enableval],
- [enable_no_tools=no])
-
+ [enable_no_tools=yes])
+AM_CONDITIONAL([ENABLE_NO_TOOLS], [test x$enable_no_tools != xno])
# allow programs to be built separately (sort alphabetically)
AC_ARG_ENABLE([affix_guessify],
@@ -512,7 +506,7 @@ AM_CONDITIONAL([WANT_XFST], [test x$enable_xfst != xno])
# Check if zlib is available
AC_CHECK_LIB([z], [main], [zlib_exists=yes], [zlib_exists=no])
-# Check if foma and lexc wrappers are wanted and possible to generate
+# Check if foma and lexc wrappers are wanted and possible to generate
AM_CONDITIONAL([GENERATE_FOMA_WRAPPER], [test "x$enable_foma_wrapper" != "xno" -a "x$with_foma" != "xno" -a "x$zlib_exists" != "xno"])
AM_CONDITIONAL([GENERATE_LEXC_WRAPPER], [test "x$enable_lexc_wrapper" != "xno" -a "x$with_foma" != "xno" -a "x$zlib_exists" != "xno"])
# The same as variables..
@@ -571,7 +565,7 @@ AC_DEFINE_UNQUOTED([AUTOMAKE_VERSION], [$automake_version], [Defines automake ve
if test "$automake_version" \< "1.12" ; then
#Use *.h extension for parser header file
yacc_use_parser_h_extension=true
- echo "automake version < 1.12, using .h extension for yacc/bison generated header files"
+ # issue a warning at the end of configure
AC_DEFINE([YACC_USE_PARSER_H_EXTENSION], [1], [Use *.h extension for parser header file])
else
echo "automake version >= 1.12, using .hh extension for yacc/bison generated header files"
@@ -598,10 +592,10 @@ AM_CONDITIONAL([CAN_DOXYGEN], [test x$DOXYGEN != xno])
# Checks for libraries
AC_LANG_PUSH([C++])
-AS_IF([test "x$with_openfst" != "xno" -a "x$enable_mingw" == "xno"],
+AS_IF([test "x$with_openfst" != "xno" -a "x$enable_mingw" == "xno"],
[AC_CHECK_LIB([dl], [main])])
-AS_IF([test "x$with_openfst" != "xno"],
+AS_IF([test "x$with_openfst" != "xno"],
[AC_CHECK_LIB([pthread], [main])]
[AC_CHECK_LIB([m], [main])])
@@ -629,12 +623,15 @@ AC_LANG_POP
AS_IF([test "x$with_unicode_handler" = "xglib"],
[AC_CHECK_LIB([glib-2.0], [main])
+if test x$ac_cv_lib_glib_2_0_main != xyes ; then
+ AC_MSG_FAILURE([--with-unicode-handler=glib requested but glib-2.0 not found])
+fi
AC_DEFINE([USE_GLIB_UNICODE], 1, [if using glib for unicode string handling])])
AS_IF([test "x$with_unicode_handler" = "xicu"],
[AC_MSG_FAILURE([ICU not yet implemented (--with-unicode-handler=hfst to disable)])])
AS_IF([test "x$with_readline" = "xyes"],
- [AC_SEARCH_LIBS([readline], [readline editline],
- [AC_DEFINE([HAVE_READLINE], 1, [if readline can be used])],
+ [AC_SEARCH_LIBS([readline], [readline editline],
+ [AC_DEFINE([HAVE_READLINE], 1, [if readline can be used])],
[AC_MSG_FAILURE([readline test failed (--without-readline to disable)])])])
### Add here your library ###
@@ -648,7 +645,11 @@ AC_CHECK_HEADERS([limits.h stdlib.h string.h error.h glob.h locale.h langinfo.h]
AC_LANG_PUSH([C++])
# Always use c++11 (flag 'c++0x' is understood also by older compilers)
-AS_IF([echo "$CXXFLAGS" | grep '\-std' | grep -E '((gnu\+\+)|(c\+\+))((11)|(0x)|(14)|(1y)|(17)|(1z))' > /dev/null 2> /dev/null], [], [CXXFLAGS="$CXXFLAGS -std=c++0x"])
+# Checks for highest supported C++ standard
+# But, clang++ does not accept invalid OpenFST code, so limit to C++11 for now
+AX_CHECK_COMPILE_FLAG([-std=c++11], [CXXFLAGS="$CXXFLAGS -std=c++11"], [
+ AS_IF([echo "$CXXFLAGS" | grep '\-std' | grep -E '((gnu\+\+)|(c\+\+))((11)|(0x)|(14)|(1y)|(17)|(1z))' > /dev/null 2> /dev/null], [], [CXXFLAGS="$CXXFLAGS -std=c++0x"])
+])
AC_CHECK_HEADERS([unordered_map], [], [],
[#ifdef HAVE_UNORDERED_MAP
@@ -685,7 +686,7 @@ AS_IF([test "x$with_unicode_handler" = "xglib"],
[AC_SUBST([GLIB_CPPFLAGS], [`$PKG_CONFIG glib-2.0 --cflags`])
AC_SUBST([GLIB_LIBS], [`$PKG_CONFIG glib-2.0 --libs`])
],
- [AC_CHECK_HEADERS([glib.h], [],
+ [AC_CHECK_HEADERS([glib.h], [],
[AC_MSG_ERROR([glib headers not found; set CPPFLAGS to -I glib install directiories or use other unicode handler instead])])])])
# remove if not needed
AS_IF([test "x$with_unicode_handler" != "xglib"],
@@ -705,40 +706,17 @@ AM_CONDITIONAL([WINDOWS], [test x$version_type = xwindows])
# Checks for system services
# config files
-AC_CONFIG_FILES([Makefile doc/Makefile test/Makefile
- test/libhfst/Makefile test/tools/Makefile
- test/tools/fsmbook-tests/Makefile
+AC_CONFIG_FILES([Makefile
libhfst/Makefile libhfst/src/Makefile
libhfst/src/implementations/Makefile
- libhfst/src/parsers/Makefile
- libhfst/hfst.pc
- tools/Makefile tools/src/Makefile tools/src/hfst-proc/Makefile
- tools/src/hfst-twolc/Makefile
- tools/src/hfst-twolc/src/Makefile
- tools/src/hfst-twolc/test/Makefile
- tools/src/hfst-tagger/Makefile
- tools/src/hfst-tagger/src/Makefile
- tools/src/hfst-tagger/test/Makefile
- tools/src/parsers/Makefile
- tools/src/parsers/test/Makefile
+ libhfst/src/parsers/Makefile
+ libhfst/hfst.pc
back-ends/Makefile
- back-ends/dlfcn/Makefile
- back-ends/openfstwin/Makefile
- back-ends/openfstwin/src/Makefile
- back-ends/openfstwin/src/lib/Makefile
back-ends/openfst/Makefile
back-ends/openfst/src/Makefile
back-ends/openfst/src/lib/Makefile
- back-ends/foma/Makefile
- back-ends/sfst/Makefile
- man/Makefile
python/Makefile
- python/test/Makefile
- scripts/Makefile])
-AC_CONFIG_FILES([scripts/hfst-foma-wrapper.sh], [chmod +x scripts/hfst-foma-wrapper.sh])
-AC_CONFIG_FILES([scripts/hfst-foma], [chmod +x scripts/hfst-foma])
-
-
+ python/test/Makefile])
# output
AC_OUTPUT
@@ -805,7 +783,7 @@ EOF
dnl stick important warnings to bottom
dnl important licensing information
dnl according to GNU GPLv2 strict (not plus) is incompatible with Apache
-dnl hope end users never disable ofst
+dnl hope end users never disable ofst
AS_IF([test "x$with_openfst" == "xno"],
[AC_MSG_WARN([Disabling openfst backend will severely cripple libhfst])])
dnl HFST's case mapping in corpus tools is mainly MES-2 set with other characters
@@ -813,15 +791,6 @@ dnl added as we go
AS_IF([test "x$with_unicode_handler" = "xhfst"],
[AC_MSG_WARN([HFST only supports basic unicode handling with limited case mapping tables etc.; for better support consider using glib or ICU --with-unicode-handler])])
dnl warn about missing "important" tools
-AS_IF([test "x$enable_lexc" == "xno"],
- [AC_MSG_WARN([hfst-lexc is not enabled; enable with --enable-lexc])])
-AS_IF([test "x$enable_twolc" == "xno"],
- [AC_MSG_WARN([hfst-twolc is not enabled; you will not be able to compile xerox twolc grammars; enable using --enable-twolc])])
-
-AS_IF([test "x$enable_tagger" == "xno"],
- [AC_MSG_WARN([hfst-train-tagger and hfst-tag are not enabled; you will not be able to train and use taggers; enable using --enable-tagger])])
-AS_IF([test "x$enable_calculate" == "xno"],
- [AC_MSG_WARN([hfst-calculate is not enabled; you will not be able to compile SFST-PL scripts; enable using --enable-calculate])])
-AS_IF([test "x$enable_xfst" == "xno"],
- [AC_MSG_WARN([hfst-xfst is not enabled; you will not be able to compile XFST scripts; enable using --enable-xfst])])
AC_MSG_WARN([Python bindings for HFST are not under autotools; see python/README for instructions about how to build and install them])
+AS_IF([test "x$yacc_use_parser_h_extension" == "xtrue"],
+ [AC_MSG_WARN([automake version < 1.12; using .h extension for yacc/bison generated header files; if you are building with pre-generated files, modifying them will make building fail, because they use .hh extension])])
diff --git a/libhfst/Makefile.am b/libhfst/Makefile.am
index 5b688c5..e5e0af4 100644
--- a/libhfst/Makefile.am
+++ b/libhfst/Makefile.am
@@ -1,13 +1,13 @@
## Process this file with automake to produce Makefile.in
-# Copyright (c) 2016 University of Helsinki
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 3 of the License, or (at your option) any later version.
-# See the file COPYING included with this distribution for more
-# information.
+# Copyright (c) 2016 University of Helsinki
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+# See the file COPYING included with this distribution for more
+# information.
SUBDIRS = src
diff --git a/libhfst/hfst.pc.in b/libhfst/hfst.pc.in
index 5eed804..638aab7 100644
--- a/libhfst/hfst.pc.in
+++ b/libhfst/hfst.pc.in
@@ -6,5 +6,5 @@ includedir=@includedir@
Name: hfst
Description: Finite-state transducer library bridge for multiple FLOSS packages
Version: @LIBHFST_VERSION@
-Libs: -L${libdir} -l at LIBHFST_NAME@ @LIBS@
+Libs: -L${libdir} -l at LIBHFST_NAME@
Cflags: -I${includedir}/hfst/ @GLIB_CFLAGS@
diff --git a/libhfst/src/FormatSpecifiers.h b/libhfst/src/FormatSpecifiers.h
index f6a4898..da3582a 100644
--- a/libhfst/src/FormatSpecifiers.h
+++ b/libhfst/src/FormatSpecifiers.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _FORMAT_SPECIFIERS_H_
diff --git a/libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc b/libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc
index baff598..0f02e10 100644
--- a/libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc
+++ b/libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc
@@ -1,16 +1,15 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "HarmonizeUnknownAndIdentitySymbols.h"
#include "HfstTransducer.h"
-// --- HfstTransitionGraph.h is enough
-#include "implementations/HfstTransitionGraph.h"
+#include "implementations/HfstBasicTransducer.h"
#include "HfstFlagDiacritics.h"
#include "implementations/optimized-lookup/pmatch.h"
@@ -19,9 +18,9 @@
namespace hfst
{
-const char * HarmonizeUnknownAndIdentitySymbols::identity =
+const char * HarmonizeUnknownAndIdentitySymbols::identity =
"@_IDENTITY_SYMBOL_@"; // --- internal_identity.c_str();
-const char * HarmonizeUnknownAndIdentitySymbols::unknown =
+const char * HarmonizeUnknownAndIdentitySymbols::unknown =
"@_UNKNOWN_SYMBOL_@"; // --- internal_unknown.c_str();
// --- these functions could be useful elsewhere, too
@@ -141,17 +140,17 @@ HarmonizeUnknownAndIdentitySymbols::HarmonizeUnknownAndIdentitySymbols
if (t2_symbols_minus_t1_symbols.empty())
{ debug_harmonize_print("t1 includes no symbols not found in t2."); }
else
- {
+ {
HfstTransducer fst1(t1,TROPICAL_OPENFST_TYPE);
- std::cerr << fst1 << std::endl;
+ std::cerr << fst1 << std::endl;
}
if (t1_symbols_minus_t2_symbols.empty())
{ debug_harmonize_print("t2 includes no symbols not found in t1."); }
else
- {
+ {
HfstTransducer fst2(t2,TROPICAL_OPENFST_TYPE);
- std::cerr << fst2 << std::endl;
+ std::cerr << fst2 << std::endl;
}
}
}
@@ -161,7 +160,7 @@ void HarmonizeUnknownAndIdentitySymbols::populate_symbol_set
{
for (HfstBasicTransducer::const_iterator it = t.begin(); it != t.end(); ++it)
{
- for (HfstBasicTransducer::HfstTransitions::const_iterator jt =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator jt =
it->begin();
jt != it->end();
++jt)
@@ -196,9 +195,9 @@ void HarmonizeUnknownAndIdentitySymbols::harmonize_identity_symbols
for (HfstBasicTransducer::iterator it = t.begin(); it != t.end(); ++it)
{
- HfstBasicTransducer::HfstTransitions added_transitions;
+ hfst::implementations::HfstBasicTransitions added_transitions;
- for (HfstBasicTransducer::HfstTransitions::const_iterator jt =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator jt =
it->begin();
jt != it->end();
++jt)
@@ -219,7 +218,7 @@ void HarmonizeUnknownAndIdentitySymbols::harmonize_identity_symbols
}
it->insert(it->end(),
added_transitions.begin(),added_transitions.end());
- }
+ }
}
// --- documentation would make this function easier to follow
@@ -231,9 +230,9 @@ void HarmonizeUnknownAndIdentitySymbols::harmonize_unknown_symbols
for (HfstBasicTransducer::iterator it = t.begin(); it != t.end(); ++it)
{
- HfstBasicTransducer::HfstTransitions added_transitions;
+ hfst::implementations::HfstBasicTransitions added_transitions;
- for (HfstBasicTransducer::HfstTransitions::const_iterator jt =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator jt =
it->begin();
jt != it->end();
++jt)
@@ -247,11 +246,11 @@ void HarmonizeUnknownAndIdentitySymbols::harmonize_unknown_symbols
for (StringSet::const_iterator kt = missing_symbols.begin();
kt != missing_symbols.end();
++kt)
- {
+ {
added_transitions.push_back
(HfstBasicTransition(jt->get_target_state(),
*kt,jt->get_output_symbol(),
- jt->get_weight()));
+ jt->get_weight()));
}
}
if (jt->get_output_symbol() == unknown)
@@ -275,18 +274,18 @@ void HarmonizeUnknownAndIdentitySymbols::harmonize_unknown_symbols
for (StringSet::const_iterator kt = missing_symbols.begin();
kt != missing_symbols.end();
++kt)
- {
+ {
for (StringSet::const_iterator lt = missing_symbols.begin();
lt != missing_symbols.end();
++lt)
- {
+ {
if (kt == lt)
{ continue; }
added_transitions.push_back
(HfstBasicTransition(jt->get_target_state(),
*lt,*kt,
- jt->get_weight()));
+ jt->get_weight()));
}
}
}
diff --git a/libhfst/src/HarmonizeUnknownAndIdentitySymbols.h b/libhfst/src/HarmonizeUnknownAndIdentitySymbols.h
index 3aee34f..fa91939 100644
--- a/libhfst/src/HarmonizeUnknownAndIdentitySymbols.h
+++ b/libhfst/src/HarmonizeUnknownAndIdentitySymbols.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
// --- licensing stuff here?
@@ -13,17 +13,16 @@
#define HARMONIZE_UNKNOWN_AND_IDENTITY_SYMBOLS
// ---
-/* A more generalized class that takes an template <class C, class W>
- HfstTransitionGraph as its constructor arguments. The functions could
- take a HfstTransitionGraph::HfstTransitionGraphAlphabet as their
+/* A more generalized class that takes an template <class C, class W>
+ HfstTransitionGraph as its constructor arguments. The functions could
+ take a HfstTransitionGraph::HfstTransitionGraphAlphabet as their
arguments and use functions C::get_unknown() etc.
*/
#include <set>
#include <string>
-#include <iostream> // --- in the cc-file instead?
-#include <algorithm> // --- "" ---
-#include <cassert> // --- "" ---
+#include <iosfwd>
+#include <algorithm>
#include "HfstDataTypes.h"
#include "HfstSymbolDefs.h"
@@ -54,7 +53,7 @@ class HarmonizeUnknownAndIdentitySymbols
// Constructor whose side effect it is to harmonize the identity and unknown
// symbols of its arguments.
HFSTDLL HarmonizeUnknownAndIdentitySymbols
- (HfstBasicTransducer &,HfstBasicTransducer &);
+ (HfstBasicTransducer &,HfstBasicTransducer &);
protected:
HfstBasicTransducer &t1;
@@ -65,29 +64,29 @@ class HarmonizeUnknownAndIdentitySymbols
// Collect the symbols from the transitions of the argument transducer
// and store them in the argument set. This is needed since the alphabet
// of HfstBasicTransducers can sometimes unexplainedly be empty...
- // --- the alphabet can contain also symbols that are not found in
+ // --- the alphabet can contain also symbols that are not found in
// transitions...
HFSTDLL void populate_symbol_set(const HfstBasicTransducer &,StringSet &);
// Add all symbols in the StringSet to the alphabet of the transducer.
HFSTDLL void add_symbols_to_alphabet(HfstBasicTransducer &, const StringSet &);
- // For every x in the set, add x:x transitions for every identity:identity
- // transition in the argument transducer (the source and target states as
+ // For every x in the set, add x:x transitions for every identity:identity
+ // transition in the argument transducer (the source and target states as
// well as the weights are the same as in the original identity transition.
HFSTDLL void harmonize_identity_symbols
(HfstBasicTransducer &,const StringSet &);
- // For every x in the set
- // 1. add, x:c transitions for every unknown:c transition in the argument
- // transducer.
+ // For every x in the set
+ // 1. add, x:c transitions for every unknown:c transition in the argument
+ // transducer.
// 2. add, c:x transitions for every c:unknown transition in the argument
// transducer.
//
// For every x and y in the set with x != y, add x:y transitions for every
// unknown:unknown transition in the argument transducer.
//
- // (the source and target states as well as the weights are the same as in
+ // (the source and target states as well as the weights are the same as in
// the original identity transition)
HFSTDLL void harmonize_unknown_symbols
(HfstBasicTransducer &,const StringSet &);
diff --git a/libhfst/src/HfstApply.cc b/libhfst/src/HfstApply.cc
index 956f44d..9cb7b5d 100644
--- a/libhfst/src/HfstApply.cc
+++ b/libhfst/src/HfstApply.cc
@@ -1,15 +1,15 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
-/* \file HfstApply.cc
- \brief HFST transducer functions that take several parameters are
- handled here.
+/* \file HfstApply.cc
+ \brief HFST transducer functions that take several parameters are
+ handled here.
*/
#include "HfstTransducer.h"
@@ -44,7 +44,7 @@ namespace hfst
//#ifdef HAVE_MY_TRANSDUCER_LIBRARY
//if (original == MY_TRANSDUCER_LIBRARY_TYPE) {
// // From my transducer to weighted transducer
- // if ( converted == TROPICAL_OPENFST_TYPE ||
+ // if ( converted == TROPICAL_OPENFST_TYPE ||
// converted == LOG_OPENFST_TYPE) {
// return true; // if your library supports weights
// return false; // if your library does not support weights
@@ -61,17 +61,17 @@ namespace hfst
HfstTransducer &HfstTransducer::apply(
#if HAVE_SFST
- SFST::Transducer * (*sfst_funct)(SFST::Transducer *),
+ SFST::Transducer * (*sfst_funct)(SFST::Transducer *),
#endif
#if HAVE_OPENFST
fst::StdVectorFst * (*tropical_ofst_funct)(fst::StdVectorFst *),
#if HAVE_OPENFST_LOG
- hfst::implementations::LogFst *
+ hfst::implementations::LogFst *
(*log_ofst_funct)(hfst::implementations::LogFst *),
#endif
#endif
#if HAVE_FOMA
- fsm * (*foma_funct)(fsm *),
+ struct fsm * (*foma_funct)(struct fsm *),
#endif
#if HAVE_XFSM
NETptr (*xfsm_funct)(NETptr),
@@ -88,9 +88,9 @@ namespace hfst
#if HAVE_SFST
case SFST_TYPE:
{
- SFST::Transducer * sfst_temp =
+ SFST::Transducer * sfst_temp =
sfst_funct(implementation.sfst);
- delete implementation.sfst;
+ sfst_interface.delete_transducer(implementation.sfst);
implementation.sfst = sfst_temp;
break;
}
@@ -100,7 +100,7 @@ namespace hfst
{
fst::StdVectorFst * tropical_ofst_temp =
tropical_ofst_funct(implementation.tropical_ofst);
- delete implementation.tropical_ofst;
+ tropical_ofst_interface.delete_transducer(implementation.tropical_ofst);
implementation.tropical_ofst = tropical_ofst_temp;
break;
}
@@ -109,7 +109,7 @@ namespace hfst
{
hfst::implementations::LogFst * log_ofst_temp =
log_ofst_funct(implementation.log_ofst);
- delete implementation.log_ofst;
+ log_ofst_interface.delete_transducer(implementation.log_ofst);
implementation.log_ofst = log_ofst_temp;
break;
}
@@ -120,7 +120,7 @@ namespace hfst
{
fsm * foma_temp =
foma_funct(implementation.foma);
- this->foma_interface.delete_foma(implementation.foma);
+ this->foma_interface.delete_foma(implementation.foma);
implementation.foma = foma_temp;
break;
}
@@ -171,7 +171,7 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
#endif
/* Add your library here. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
-// my_namespace::MyFst *
+// my_namespace::MyFst *
// (*my_transducer_library_funct)(my_namespace::MyFst *,int n),
//#endif
unsigned int n )
@@ -181,9 +181,9 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
#if HAVE_SFST
case SFST_TYPE:
{
- SFST::Transducer * sfst_temp =
+ SFST::Transducer * sfst_temp =
sfst_funct(implementation.sfst,n);
- delete implementation.sfst;
+ sfst_interface.delete_transducer(implementation.sfst);
implementation.sfst = sfst_temp;
break;
}
@@ -193,7 +193,7 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
{
fst::StdVectorFst * tropical_ofst_temp =
tropical_ofst_funct(implementation.tropical_ofst,n);
- delete implementation.tropical_ofst;
+ tropical_ofst_interface.delete_transducer(implementation.tropical_ofst);
implementation.tropical_ofst = tropical_ofst_temp;
break;
}
@@ -202,7 +202,7 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
{
hfst::implementations::LogFst * log_ofst_temp =
log_ofst_funct(implementation.log_ofst,n);
- delete implementation.log_ofst;
+ log_ofst_interface.delete_transducer(implementation.log_ofst);
implementation.log_ofst = log_ofst_temp;
break;
}
@@ -211,9 +211,9 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
#if HAVE_FOMA
case FOMA_TYPE:
{
- fsm * foma_temp =
+ fsm * foma_temp =
foma_funct(implementation.foma,n);
- this->foma_interface.delete_foma(implementation.foma);
+ this->foma_interface.delete_foma(implementation.foma);
implementation.foma = foma_temp;
break;
}
@@ -221,7 +221,7 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
#if HAVE_XFSM
case XFSM_TYPE:
{
- NETptr xfsm_temp =
+ NETptr xfsm_temp =
xfsm_funct(implementation.xfsm,n);
delete implementation.xfsm;
implementation.xfsm = xfsm_temp;
@@ -232,7 +232,7 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
//#if HAVE_MY_TRANSDUCER_LIBRARY
//case MY_TRANSDUCER_LIBRARY_TYPE:
//{
- //my_namespace::MyFst * my_fst_temp =
+ //my_namespace::MyFst * my_fst_temp =
// my_transducer_library_funct(implementation.my_transducer_library,n);
//delete (implementation.my_transducer_library);
//implementation.my_transducer_library = my_fst_temp;
@@ -252,7 +252,7 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
SFST::Transducer * (*sfst_funct)(SFST::Transducer *, String, String),
#endif
#if HAVE_OPENFST
- fst::StdVectorFst * (*tropical_ofst_funct)(fst::StdVectorFst *,String,
+ fst::StdVectorFst * (*tropical_ofst_funct)(fst::StdVectorFst *,String,
String),
#if HAVE_OPENFST_LOG
hfst::implementations::LogFst * (*log_ofst_funct)
@@ -267,7 +267,7 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
#endif
/* Add your library here. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
- //my_namespace::MyFst *
+ //my_namespace::MyFst *
// (*my_transducer_library_funct)(my_namespace::MyFst *, String, String),
//#endif
String s1, String s2)
@@ -277,9 +277,9 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
#if HAVE_SFST
case SFST_TYPE:
{
- SFST::Transducer * sfst_temp =
+ SFST::Transducer * sfst_temp =
sfst_funct(implementation.sfst,s1,s2);
- delete implementation.sfst;
+ sfst_interface.delete_transducer(implementation.sfst);
implementation.sfst = sfst_temp;
break;
}
@@ -289,7 +289,7 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
{
fst::StdVectorFst * tropical_ofst_temp =
tropical_ofst_funct(implementation.tropical_ofst,s1,s2);
- delete implementation.tropical_ofst;
+ tropical_ofst_interface.delete_transducer(implementation.tropical_ofst);
implementation.tropical_ofst = tropical_ofst_temp;
break;
}
@@ -298,7 +298,7 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
{
hfst::implementations::LogFst * log_ofst_temp =
log_ofst_funct(implementation.log_ofst,s1,s2);
- delete implementation.log_ofst;
+ log_ofst_interface.delete_transducer(implementation.log_ofst);
implementation.log_ofst = log_ofst_temp;
break;
}
@@ -307,9 +307,9 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
#if HAVE_FOMA
case FOMA_TYPE:
{
- fsm * foma_temp =
+ fsm * foma_temp =
foma_funct(implementation.foma,s1,s2);
- this->foma_interface.delete_foma(implementation.foma);
+ this->foma_interface.delete_foma(implementation.foma);
implementation.foma = foma_temp;
break;
}
@@ -317,7 +317,7 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
#if HAVE_XFSM
case XFSM_TYPE:
{
- NETptr xfsm_temp =
+ NETptr xfsm_temp =
xfsm_funct(implementation.xfsm,s1,s2);
delete implementation.xfsm;
implementation.xfsm = xfsm_temp;
@@ -328,7 +328,7 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
//#if HAVE_MY_TRANSDUCER_LIBRARY
//case MY_TRANSDUCER_LIBRARY_TYPE:
//{
- //my_namespace::MyFst * my_fst_temp =
+ //my_namespace::MyFst * my_fst_temp =
// my_transducer_library_funct
// (implementation.my_transducer_library,s1,s2);
//delete(implementation.my_transducer_library);
@@ -387,7 +387,7 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
this->insert_missing_symbols_to_alphabet_from(another, true);
another.insert_missing_symbols_to_alphabet_from(*this, true);
HfstTransducer * another_ =
- this->harmonize_(another);
+ this->harmonize_(another);
if (another_ == NULL) // foma
{ another_ = new HfstTransducer(another); }
@@ -396,10 +396,10 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
#if HAVE_SFST
case SFST_TYPE:
{
- SFST::Transducer * sfst_temp =
+ SFST::Transducer * sfst_temp =
sfst_funct(implementation.sfst,
another_->implementation.sfst);
- delete implementation.sfst;
+ sfst_interface.delete_transducer(implementation.sfst);
implementation.sfst = sfst_temp;
break;
}
@@ -410,7 +410,7 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
fst::StdVectorFst * tropical_ofst_temp =
tropical_ofst_funct(this->implementation.tropical_ofst,
another_->implementation.tropical_ofst);
- delete implementation.tropical_ofst;
+ tropical_ofst_interface.delete_transducer(implementation.tropical_ofst);
implementation.tropical_ofst = tropical_ofst_temp;
break;
}
@@ -420,7 +420,7 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
hfst::implementations::LogFst * log_ofst_temp =
log_ofst_funct(implementation.log_ofst,
another_->implementation.log_ofst);
- delete implementation.log_ofst;
+ log_ofst_interface.delete_transducer(implementation.log_ofst);
implementation.log_ofst = log_ofst_temp;
break;
}
@@ -429,9 +429,9 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
#if HAVE_FOMA
case FOMA_TYPE:
{
- fsm * foma_temp =
+ fsm * foma_temp =
foma_funct(implementation.foma,another_->implementation.foma);
- delete implementation.foma;
+ this->foma_interface.delete_foma(implementation.foma);
implementation.foma = foma_temp;
break;
}
@@ -439,7 +439,7 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
#if HAVE_XFSM
case XFSM_TYPE:
{
- NETptr xfsm_temp =
+ NETptr xfsm_temp =
xfsm_funct(implementation.xfsm,another_->implementation.xfsm);
//delete implementation.xfsm;
implementation.xfsm = xfsm_temp;
@@ -450,7 +450,7 @@ NETptr (*xfsm_funct)(NETptr, unsigned int n),
//#if HAVE_MY_TRANSDUCER_LIBRARY
//case MY_TRANSDUCER_LIBRARY_TYPE:
//{
- // my_namespace::MyFst * my_fst_temp =
+ // my_namespace::MyFst * my_fst_temp =
// my_transducer_library_funct
// (implementation.my_transducer_library,
// another.implementation.my_transducer_library);
diff --git a/libhfst/src/HfstDataTypes.cc b/libhfst/src/HfstDataTypes.cc
index 9ab82a5..901a67a 100644
--- a/libhfst/src/HfstDataTypes.cc
+++ b/libhfst/src/HfstDataTypes.cc
@@ -1,15 +1,18 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "HfstDataTypes.h"
+#include <limits.h>
+#include <float.h>
+#include <stdexcept>
-namespace hfst
+namespace hfst
{
const char * implementation_type_to_string(ImplementationType type)
@@ -92,5 +95,55 @@ namespace hfst
}
}
+ int size_t_to_int(size_t value)
+ {
+ if (value > INT_MAX)
+ {
+ throw std::overflow_error("data is larger than INT_MAX");
+ }
+ return static_cast<int>(value);
+ }
+
+ unsigned int size_t_to_uint(size_t value)
+ {
+ if (value > UINT_MAX)
+ {
+ throw std::overflow_error("data is larger than UINT_MAX");
+ }
+ return static_cast<unsigned int>(value);
+ }
+
+ unsigned short size_t_to_ushort(size_t value)
+ {
+ if (value > USHRT_MAX)
+ {
+ throw std::overflow_error("data is larger than USHRT_MAX");
+ }
+ return static_cast<unsigned short>(value);
+ }
+
+ float double_to_float(double value)
+ {
+ if (value > FLT_MAX)
+ {
+ throw std::overflow_error("data is larger than FLT_MAX");
+ }
+ return static_cast<float>(value);
+ }
+
+ FILE * hfst_fopen(const char * filename, const char * mode)
+ {
+#ifdef _MSC_VER
+ FILE * f = NULL;
+ errno_t err = fopen_s(&f, filename, mode);
+ if (err != 0)
+ return NULL;
+ else
+ return f;
+#else
+ return fopen(filename, mode);
+#endif
+ }
+
}
diff --git a/libhfst/src/HfstDataTypes.h b/libhfst/src/HfstDataTypes.h
index 14ee911..3f7d593 100644
--- a/libhfst/src/HfstDataTypes.h
+++ b/libhfst/src/HfstDataTypes.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _HFST_DATA_TYPES_H_
@@ -15,13 +15,18 @@
#endif
#include <string>
-#include <iostream>
+#include <iosfwd>
#include <vector>
#include <map>
#include <set>
+#ifdef _MSC_VER
+ #define strdup _strdup
+ #define isatty _isatty
+ #define fileno _fileno
+#endif // _MSC_VER
-/** @file HfstDataTypes.h
+/** @file HfstDataTypes.h
\brief Datatypes that are needed when using the HFST API. */
namespace hfst
@@ -38,7 +43,7 @@ namespace hfst
// ENUMS AND TYPEDEFS...
/** \brief The type of an HfstTransducer. */
- enum ImplementationType
+ enum ImplementationType
{
SFST_TYPE, /**< An SFST transducer, unweighted. */
TROPICAL_OPENFST_TYPE, /**< An OpenFst transducer with tropical weights. */
@@ -46,25 +51,32 @@ namespace hfst
FOMA_TYPE, /**< A foma transducer, unweighted. */
XFSM_TYPE, /**< An xfsm transducer, unweighted (mostly for testing purposes). */
/* Add an enumerator for your transducer type here. */
- //MY_TRANSDUCER_LIBRARY_TYPE,
+ //MY_TRANSDUCER_LIBRARY_TYPE,
HFST_OL_TYPE, /**< An HFST optimized lookup transducer, unweighted */
HFST_OLW_TYPE, /**< An HFST optimized lookup transducer with weights */
HFST2_TYPE, /**< HFST2 header present, conversion required */
UNSPECIFIED_TYPE, /**< Format left open by e.g. default constructor */
- ERROR_TYPE /**< Type not recognised.
- This type might be returned by a function
- if an error occurs. */
+ ERROR_TYPE /**< Type not recognised.
+ This type might be returned by a function
+ if an error occurs. */
};
const char * implementation_type_to_string(ImplementationType type);
const char * implementation_type_to_format(ImplementationType type);
+ int size_t_to_int(size_t value);
+ unsigned int size_t_to_uint(size_t value);
+ unsigned short size_t_to_ushort(size_t value);
+ float double_to_float(double value);
+
+ FILE * hfst_fopen(const char * filename, const char * mode);
+
/** \brief The type of a push operation.
@see HfstTransducer::push_weights */
enum PushType
- {
+ {
TO_INITIAL_STATE /**< Push weights towards initial state. */,
- TO_FINAL_STATE /**< Push weights towards final state(s). */
+ TO_FINAL_STATE /**< Push weights towards final state(s). */
};
//! @brief A vector of transducers
@@ -81,6 +93,9 @@ namespace hfst
//! Used by functions in namespace \link hfst::rules rules\endlink
typedef std::vector <HfstTransducerPair> HfstTransducerPairVector;
+ typedef std::pair<std::string, std::string> StringPair;
+ typedef std::set<std::pair<std::string, std::string> > StringPairSet;
+
//! @brief A vector of strings
//!
//! Used by typedef HfstOneLevelPath.
@@ -111,18 +126,18 @@ namespace hfst
namespace implementations {
template <class C> class HfstTransitionGraph;
class HfstTropicalTransducerTransitionData;
- typedef HfstTransitionGraph<HfstTropicalTransducerTransitionData>
- HfstBasicTransducer;
- class HfstFastTransitionData;
- typedef HfstTransitionGraph<HfstFastTransitionData>
- HfstFastTransducer;
+ //typedef HfstTransitionGraph<HfstTropicalTransducerTransitionData>
+ class HfstBasicTransducer;
+ //class HfstFastTransitionData;
+ //typedef HfstTransitionGraph<HfstFastTransitionData>
+ //HfstFastTransducer;
typedef unsigned int HfstState;
template <class C> class HfstTransition;
- typedef HfstTransition<HfstTropicalTransducerTransitionData>
- HfstBasicTransition;
- typedef HfstTransition<HfstFastTransitionData>
- HfstFastTransition;
+ //typedef HfstTransition<HfstTropicalTransducerTransitionData>
+ class HfstBasicTransition;
+ //typedef HfstTransition<HfstFastTransitionData>
+ // HfstFastTransition;
}
@@ -130,5 +145,5 @@ namespace hfst
}
-// vim: set ft=cpp.doxygen:
+// vim: set ft=cpp.doxygen:
#endif
diff --git a/libhfst/src/HfstEpsilonHandler.cc b/libhfst/src/HfstEpsilonHandler.cc
index 76a168c..8aa620d 100644
--- a/libhfst/src/HfstEpsilonHandler.cc
+++ b/libhfst/src/HfstEpsilonHandler.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "HfstEpsilonHandler.h"
@@ -14,10 +14,10 @@ namespace hfst {
/* Create an epsilon handler which allows a maximum of \a cutoff
consecutive input epsilon cycles. The cycles do not need to be
the same. */
- HfstEpsilonHandler::HfstEpsilonHandler(size_t cutoff):
+ HfstEpsilonHandler::HfstEpsilonHandler(size_t cutoff):
epsilon_path(), max_cycles(cutoff), cycles(0) {};
- /* Called before calling lookup_fd recursively.
+ /* Called before calling lookup_fd recursively.
Appends state \a s to the epsilon path if not found at the end already. */
void HfstEpsilonHandler::push_back(hfst::implementations::HfstState s)
{
@@ -31,7 +31,7 @@ namespace hfst {
}
};
- /* Removes a state from the end of the epsilon path,
+ /* Removes a state from the end of the epsilon path,
unless the path it is empty. */
void HfstEpsilonHandler::pop_back()
{
@@ -39,7 +39,7 @@ namespace hfst {
epsilon_path.pop_back();
};
- /* This function is called at the beginning of lookup_fd.
+ /* This function is called at the beginning of lookup_fd.
It tells whether we can proceed in the current state (i.e. state \a s)
and updates the cycle counter and epsilon path of the HfstEpsilonHandler. */
bool HfstEpsilonHandler::can_continue(hfst::implementations::HfstState s)
@@ -50,7 +50,7 @@ namespace hfst {
if (*it == s) // a cycle detected
{
it++;
- // erase the cycle from the epsilon path
+ // erase the cycle from the epsilon path
// and check whether the number of cycles is exceeded
epsilon_path.erase(it, epsilon_path.end());
cycles++;
diff --git a/libhfst/src/HfstEpsilonHandler.h b/libhfst/src/HfstEpsilonHandler.h
index 8a7ed5c..7624afd 100644
--- a/libhfst/src/HfstEpsilonHandler.h
+++ b/libhfst/src/HfstEpsilonHandler.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _HFST_EPSILON_HANDLER_H_
@@ -20,7 +20,7 @@ namespace hfst {
protected:
typedef std::vector<hfst::implementations::HfstState> HfstStateVector;
// the path of consecutive input epsilon transitions
- HfstStateVector epsilon_path;
+ HfstStateVector epsilon_path;
size_t max_cycles; // maximum number of consecutive epsilon cycles allowed
size_t cycles; // number of cycles detected so far
public:
diff --git a/libhfst/src/HfstExceptionDefs.cc b/libhfst/src/HfstExceptionDefs.cc
index f5d7bfc..ddefee1 100644
--- a/libhfst/src/HfstExceptionDefs.cc
+++ b/libhfst/src/HfstExceptionDefs.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "HfstExceptionDefs.h"
@@ -29,7 +29,14 @@ HfstException::HfstException
HfstException::~HfstException() {}
std::string HfstException::operator() (void) const
-{
+{
+ std::ostringstream o;
+ o << "Exception: "<< name << " in file: " << file << " on line: " << line;
+ return o.str();
+}
+
+std::string HfstException::what() const
+{
std::ostringstream o;
o << "Exception: "<< name << " in file: " << file << " on line: " << line;
return o.str();
@@ -47,7 +54,18 @@ std::string HfstException::operator() (void) const
HFST_EXCEPTION_CHILD_DEFINITION(HfstTransducerTypeMismatchException);
-HFST_EXCEPTION_CHILD_DEFINITION(ImplementationTypeNotAvailableException);
+//HFST_EXCEPTION_CHILD_DEFINITION(ImplementationTypeNotAvailableException);
+
+ImplementationTypeNotAvailableException::ImplementationTypeNotAvailableException(const std::string &name,const std::string &file,size_t line, hfst::ImplementationType type):
+ HfstException(name, file, line),
+ type(type)
+{};
+
+hfst::ImplementationType ImplementationTypeNotAvailableException::get_type() const
+{
+ return type;
+}
+
HFST_EXCEPTION_CHILD_DEFINITION(FileIsInGZFormatException);
@@ -103,7 +121,7 @@ HFST_EXCEPTION_CHILD_DEFINITION(MetadataException);
HFST_EXCEPTION_CHILD_DEFINITION(FlagDiacriticsAreNotIdentitiesException);
-//HFST_EXCEPTION_CHILD_DEFINITION(SymbolRedefinedException);
+//HFST_EXCEPTION_CHILD_DEFINITION(SymbolRedefinedException);
//HFST_EXCEPTION_CHILD_DEFINITION(TransducerHasNoStartStateException);
//HFST_EXCEPTION_CHILD_DEFINITION(TransducerHasMoreThanOneStartStateException);
diff --git a/libhfst/src/HfstExceptionDefs.h b/libhfst/src/HfstExceptionDefs.h
index 34f4c78..42e7ab4 100644
--- a/libhfst/src/HfstExceptionDefs.h
+++ b/libhfst/src/HfstExceptionDefs.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _HFST_EXCEPTION_DEFS_H_
@@ -17,11 +17,12 @@ void hfst_set_exception(std::string name);
std::string hfst_get_exception();
#include "hfstdll.h"
+#include "HfstDataTypes.h"
//! @file HfstExceptionDefs.h
//! @brief A file for exceptions
-//! @brief Base class for HfstExceptions. Holds its own name and the file
+//! @brief Base class for HfstExceptions. Holds its own name and the file
//! and line number where it was thrown.
struct HfstException
{
@@ -33,6 +34,8 @@ struct HfstException
~HfstException();
//! @brief Get the error message.
HFSTDLL std::string operator() (void) const;
+ // An alias for python
+ HFSTDLL std::string what() const;
};
//! @brief Macro to throw an exception of type @a E.
@@ -53,7 +56,7 @@ struct HfstException
//! @brief Declare a subclass of @a HfstException of type @a CHILD.
#define HFST_EXCEPTION_CHILD_DECLARATION(CHILD) \
struct CHILD : public HfstException \
- { CHILD(const std::string &name,const std::string &file,size_t line); }
+ { CHILD(const std::string &name,const std::string &file,size_t line); }
//! Define a subclass of @a HfstException of type @a CHILD.
#define HFST_EXCEPTION_CHILD_DEFINITION(CHILD) \
@@ -77,8 +80,8 @@ struct HfstException
//! @brief Two or more HfstTransducers are not of the same type
HFST_EXCEPTION_CHILD_DECLARATION(HfstTransducerTypeMismatchException);
-/** \brief The library required by the implementation type requested
- is not linked to HFST.
+/** \brief The library required by the implementation type requested
+ is not linked to HFST.
An example:
\verbatim
@@ -86,15 +89,25 @@ try {
HfstTransducer("foo", "bar", type);
} catch (ImplementationTypeNotAvailableException e) {
fprintf(stderr, "ERROR: Type requested is not available.\n");
-}
+}
\endverbatim
*/
-HFST_EXCEPTION_CHILD_DECLARATION(ImplementationTypeNotAvailableException);
+//HFST_EXCEPTION_CHILD_DECLARATION(ImplementationTypeNotAvailableException);
+
+class ImplementationTypeNotAvailableException : public HfstException
+{
+ private:
+ hfst::ImplementationType type;
+ public:
+ ImplementationTypeNotAvailableException(const std::string &name,const std::string &file,size_t line, hfst::ImplementationType type);
+ hfst::ImplementationType get_type() const;
+};
+
/** \brief Function has not been implemented (yet). */
HFST_EXCEPTION_CHILD_DECLARATION(FunctionNotImplementedException);
-/** \brief Stream cannot be read.
+/** \brief Stream cannot be read.
Thrown by
hfst::HfstTransducer(const hfst::HfstInputStream&) and
@@ -106,14 +119,14 @@ try {
HfstInputStream in("foofile");
} catch (StreamNotReadableException e) {
fprintf(stderr, "ERROR: file cannot be read.\n");
-}
+}
\endverbatim
*/
HFST_EXCEPTION_CHILD_DECLARATION(StreamNotReadableException);
-/** \brief Stream cannot be written.
+/** \brief Stream cannot be written.
-Thrown by hfst::HfstOutputStream::operator<< and
+Thrown by hfst::HfstOutputStream::operator<< and
hfst::HfstTransducer::write_in_att_format
An example:
@@ -124,12 +137,12 @@ try {
out << tr;
} catch (StreamCannotBeWrittenException e) {
fprintf(stderr, "ERROR: file cannot be written.\n");
-}
+}
\endverbatim
*/
HFST_EXCEPTION_CHILD_DECLARATION(StreamCannotBeWrittenException);
-/** \brief Stream is closed.
+/** \brief Stream is closed.
Thrown by hfst::HfstTransducer::write_in_att_format
hfst::HfstTransducer(FILE*, ImplementationType, const std::string&)
@@ -146,7 +159,7 @@ try {
out << tr;
} catch (StreamIsClosedException e) {
fprintf(stderr, "ERROR: stream to file is closed.\n");
-}
+}
\endverbatim
*/
HFST_EXCEPTION_CHILD_DECLARATION(StreamIsClosedException);
@@ -155,12 +168,12 @@ HFST_EXCEPTION_CHILD_DECLARATION(StreamIsClosedException);
Thrown by
hfst::HfstTransducer(const hfst::HfstInputStream&)
- hfst::HfstInputStream()
+ hfst::HfstInputStream()
hfst::HfstInputStream(const std::string&)
*/
HFST_EXCEPTION_CHILD_DECLARATION(EndOfStreamException);
-/** \brief Transducer is cyclic.
+/** \brief Transducer is cyclic.
thrown by hfst::HfstTransducer::extract_paths and
hfst::HfstTransducer::extract_paths_fd. An example:
@@ -180,11 +193,11 @@ try {
HFST_EXCEPTION_CHILD_DECLARATION(TransducerIsCyclicException);
-/** \brief The stream does not contain transducers.
+/** \brief The stream does not contain transducers.
- Thrown by
+ Thrown by
hfst::HfstTransducer(const hfst::HfstInputStream&)
- hfst::HfstInputStream()
+ hfst::HfstInputStream()
hfst::HfstInputStream(const std::string&)
An example. The file "foofile" contains
@@ -201,14 +214,14 @@ try {
HfstInputStream in("foofile");
} catch (NotTransducerStreamException e) {
fprintf(stderr, "ERROR: file does not contain transducers.\n");
-}
+}
\endverbatim
*/
HFST_EXCEPTION_CHILD_DECLARATION(NotTransducerStreamException);
HFST_EXCEPTION_CHILD_DECLARATION(FileIsInGZFormatException);
-/** \brief The stream is not in valid AT&T format.
+/** \brief The stream is not in valid AT&T format.
An example. The file "testfile.att" contains
@@ -237,7 +250,7 @@ fprintf(stderr, "Read %i transducers in total.\n", (int)transducers.size());
\endverbatim
- thrown by
+ thrown by
hfst::HfstTransducer::HfstTransducer(FILE*,ImplementationType,const std::string&)
*/
HFST_EXCEPTION_CHILD_DECLARATION(NotValidAttFormatException);
@@ -246,7 +259,7 @@ HFST_EXCEPTION_CHILD_DECLARATION(NotValidPrologFormatException);
HFST_EXCEPTION_CHILD_DECLARATION(NotValidLexcFormatException);
-/** \brief State is not final (and cannot have a final weight).
+/** \brief State is not final (and cannot have a final weight).
An example:
@@ -276,7 +289,7 @@ HFST_EXCEPTION_CHILD_DECLARATION(StateIsNotFinalException);
\verbatim
ImplementationType type = SFST_TYPE;
-// The second context transducer is
+// The second context transducer is
HfstTransducerPair contexts(HfstTransducer("c", type),
HfstTransducer("c", "d", type));
HfstTransducer mapping("a", "b", type);
@@ -318,7 +331,7 @@ float w = tr.get_final_weight(2);
HFST_EXCEPTION_CHILD_DECLARATION(StateIndexOutOfBoundsException);
-/** \brief Transducer has a malformed HFST header.
+/** \brief Transducer has a malformed HFST header.
Thrown by hfst::HfstTransducer(HfstInputStream&)
hfst::HfstInputStream()
@@ -327,7 +340,7 @@ HFST_EXCEPTION_CHILD_DECLARATION(StateIndexOutOfBoundsException);
HFST_EXCEPTION_CHILD_DECLARATION(TransducerHeaderException);
-/** \brief An OpenFst transducer does not have an input symbol table.
+/** \brief An OpenFst transducer does not have an input symbol table.
When converting from OpenFst to tropical or log HFST, the OpenFst transducer
must have at least an input symbol table. If the output symbol table
@@ -355,21 +368,21 @@ An example:
\verbatim
HfstTransducer foo("foo", SFST_TYPE);
HfstTransducer bar("bar", FOMA_TYPE);
-foo.disjunct(bar); // an exception is thrown
+foo.disjunct(bar); // an exception is thrown
\endverbatim
*/
HFST_EXCEPTION_CHILD_DECLARATION(TransducerTypeMismatchException);
-/** \brief The set of transducer pairs is empty.
+/** \brief The set of transducer pairs is empty.
Thrown by rule functions in namespace #hfst::rules. An example:
\verbatim
HfstTransducerPairVector contexts; // contexts is empty
HfstTransducer rest = hfst::rules::restriction
- (contexts, mapping, alphabet, twol_type, direction);
+ (contexts, mapping, alphabet, twol_type, direction);
\endverbatim
*/
HFST_EXCEPTION_CHILD_DECLARATION(EmptySetOfContextsException);
@@ -392,16 +405,16 @@ HFST_EXCEPTION_CHILD_DECLARATION(SpecifiedTypeRequiredException);
HFST_EXCEPTION_CHILD_DECLARATION(HfstFatalException);
-/** \brief Transducer has wrong type.
+/** \brief Transducer has wrong type.
This exception suggests that an HfstTransducer has not been properly
initialized, probably due to a bug in the HFST library. Alternatively
- the default constructor of HfstTransducer has been called at some point.
+ the default constructor of HfstTransducer has been called at some point.
@see hfst::HfstTransducer() */
HFST_EXCEPTION_CHILD_DECLARATION(TransducerHasWrongTypeException);
-/** \brief String is not valid utf-8.
+/** \brief String is not valid utf-8.
This exception suggests that an input string is not valid utf8.
@@ -414,7 +427,7 @@ HFST_EXCEPTION_CHILD_DECLARATION(SymbolNotFoundException);
HFST_EXCEPTION_CHILD_DECLARATION(FlagDiacriticsAreNotIdentitiesException);
-//HFST_EXCEPTION_CHILD_DECLARATION(SymbolRedefinedException);
+//HFST_EXCEPTION_CHILD_DECLARATION(SymbolRedefinedException);
//HFST_EXCEPTION_CHILD_DECLARATION(TransducerHasNoStartStateException);
//HFST_EXCEPTION_CHILD_DECLARATION(TransducerHasMoreThanOneStartStateException);
diff --git a/libhfst/src/HfstExtractStrings.h b/libhfst/src/HfstExtractStrings.h
index 2aab044..5ebcd4d 100644
--- a/libhfst/src/HfstExtractStrings.h
+++ b/libhfst/src/HfstExtractStrings.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _EXTRACT_STRINGS_H_
@@ -12,7 +12,7 @@
#include "HfstSymbolDefs.h"
#include <string>
#include <vector>
-#include <iostream>
+#include <iosfwd>
#include <sstream>
#include <set>
@@ -31,7 +31,7 @@ namespace hfst {
/* \brief (Being replaced by HfstOneLevelPath and HfstTwoLevelPath)
A weighted string pair that represents a path in a transducer.
- @see WeightedPaths
+ @see WeightedPaths
@see HfstOneLevelPath
@see HfstTransducer::extract_paths
*/
@@ -45,13 +45,13 @@ namespace hfst {
std::string ostring;
/* \brief The weight of the path. */
W weight;
- /* \brief An optional StringPairVector representation of the path.
+ /* \brief An optional StringPairVector representation of the path.
This can be used when we are interested in the exact alignment of
symbols in a given path. If you are going to use this variable,
set the value of \a is_spv_in_use 'true'. */
StringPairVector spv;
- /* \brief Whether the StringPairVector representation is in use.
+ /* \brief Whether the StringPairVector representation is in use.
This variable tells whether we are using the string pair vector
representation. By default, it is 'false'. */
@@ -63,12 +63,12 @@ namespace hfst {
bool operator< (const WeightedPath &another) const
{ if (weight == another.weight)
{ if (istring == another.istring)
- { if (ostring == another.ostring)
+ { if (ostring == another.ostring)
{ /* Handle here spv. */
if (not is_spv_in_use)
return false; /* paths are equivalent */
- unsigned int common_length
- = (spv.size()<another.spv.size())?
+ unsigned int common_length
+ = (spv.size()<another.spv.size())?
spv.size() : another.spv.size();
/* Go through string pairs. */
for (unsigned int i=0; i<common_length; i++) {
@@ -87,7 +87,7 @@ namespace hfst {
std::string to_string(void) const
{ stringstream s_stream(ios::out);
- s_stream << istring << ":" << ostring << "\t" << weight;
+ s_stream << istring << ":" << ostring << "\t" << weight;
s_stream.flush();
return s_stream.str();
}
@@ -112,13 +112,13 @@ namespace hfst {
weight = weight + another.weight;
return *this;
}
- else
+ else
{
istring = istring + another.istring;
ostring = ostring + another.ostring;
weight = weight + another.weight;
return *this;
- }
+ }
}
void operator=(const WeightedPath &another)
{ if (this == &another) { return; }
@@ -128,8 +128,8 @@ namespace hfst {
};
- /* \brief A class for storing weighted string pairs that represent
- paths in a transducer.
+ /* \brief A class for storing weighted string pairs that represent
+ paths in a transducer.
Iterators to Vectors and Sets return paths in descending weight order
(the string with the biggest weight is returned first). (check this)
@@ -141,9 +141,9 @@ namespace hfst {
{ public:
/* \brief A vector of weighted string pairs. */
- typedef std::vector< WeightedPath<W> > Vector;
+ typedef std::vector< WeightedPath<W> > Vector;
/* \brief A set of weighted string pairs. */
- typedef std::set< WeightedPath<W> > Set;
+ typedef std::set< WeightedPath<W> > Set;
static void add(Vector &v,WeightedPath<W> &s)
{
@@ -192,8 +192,8 @@ namespace hfst {
* ends at a \a final state. The return value determines the future course
* of the extraction search.
*
- * @returns A data structure indicating whether the search
- * should continue, be broken off immediately,
+ * @returns A data structure indicating whether the search
+ * should continue, be broken off immediately,
* or whether the specific path should no longer be followed.
*/
//virtual RetVal operator()(WeightedPath<float>& path, bool final) = 0;
diff --git a/libhfst/src/HfstFlagDiacritics.cc b/libhfst/src/HfstFlagDiacritics.cc
index 384437e..d3b7344 100644
--- a/libhfst/src/HfstFlagDiacritics.cc
+++ b/libhfst/src/HfstFlagDiacritics.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "HfstFlagDiacritics.h"
@@ -13,6 +13,30 @@
namespace hfst {
+ FdOperation::FdOperation
+ (FdOperator op, FdFeature feat, FdValue val, const std::string& str):
+ op(op), feature(feat), value(val), name(str) {}
+
+ FdOperator FdOperation::Operator(void) const { return op; }
+ FdFeature FdOperation::Feature(void) const { return feature; }
+ FdValue FdOperation::Value(void) const { return value; }
+ std::string FdOperation::Name(void) const { return name; }
+
+ FdOperator FdOperation::char_to_operator(char c)
+ {
+ switch (c) {
+ case 'P': return Pop;
+ case 'N': return Nop;
+ case 'R': return Rop;
+ case 'D': return Dop;
+ case 'C': return Cop;
+ case 'U': return Uop;
+ default:
+ throw;
+ }
+ }
+
+
bool FdOperation::is_diacritic(const std::string& diacritic_string)
{
// All diacritics have form @[A-Z][.][A-Z]+([.][A-Z]+)?@
@@ -53,9 +77,9 @@ bool FdOperation::is_diacritic(const std::string& diacritic_string)
}
std::string FdOperation::get_operator(const std::string& diacritic)
- {
+ {
// The operator is the second char.
- return diacritic.substr(1,1);
+ return diacritic.substr(1,1);
}
std::string FdOperation::get_feature(const std::string& diacritic)
@@ -93,13 +117,13 @@ bool FdOperation::is_diacritic(const std::string& diacritic_string)
}
bool FdOperation::has_value(const std::string& flag_diacritic)
- {
- return flag_diacritic.find('.',flag_diacritic.find('.') + 1) !=
+ {
+ return flag_diacritic.find('.',flag_diacritic.find('.') + 1) !=
std::string::npos;
}
std::string::size_type FdOperation::find_diacritic
-(const std::string& diacritic_str,
+(const std::string& diacritic_str,
std::string::size_type& length)
{
std::string::size_type start = diacritic_str.find('@');
diff --git a/libhfst/src/HfstFlagDiacritics.h b/libhfst/src/HfstFlagDiacritics.h
index 46a4435..09908db 100644
--- a/libhfst/src/HfstFlagDiacritics.h
+++ b/libhfst/src/HfstFlagDiacritics.h
@@ -1,16 +1,16 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _FLAG_DIACRITICS_H_
#define _FLAG_DIACRITICS_H_
-#include <iostream>
+#include <iosfwd>
#include <string>
#include <map>
#include <vector>
@@ -18,6 +18,7 @@
#include <utility>
#include "hfstdll.h"
+#include "HfstDataTypes.h"
/** @file HfstFlagDiacritics.h
\brief Class declarations for flag diacritic handling. */
@@ -38,27 +39,14 @@ private:
std::string name;
public:
HFSTDLL FdOperation
- (FdOperator op, FdFeature feat, FdValue val, const std::string& str):
- op(op), feature(feat), value(val), name(str) {}
+ (FdOperator op, FdFeature feat, FdValue val, const std::string& str);
- HFSTDLL FdOperator Operator(void) const { return op; }
- HFSTDLL FdFeature Feature(void) const { return feature; }
- HFSTDLL FdValue Value(void) const { return value; }
- HFSTDLL std::string Name(void) const { return name; }
+ HFSTDLL FdOperator Operator(void) const;
+ HFSTDLL FdFeature Feature(void) const;
+ HFSTDLL FdValue Value(void) const;
+ HFSTDLL std::string Name(void) const;
- HFSTDLL static FdOperator char_to_operator(char c)
- {
- switch (c) {
- case 'P': return Pop;
- case 'N': return Nop;
- case 'R': return Rop;
- case 'D': return Dop;
- case 'C': return Cop;
- case 'U': return Uop;
- default:
- throw;
- }
- }
+ HFSTDLL static FdOperator char_to_operator(char c);
HFSTDLL static bool is_diacritic(const std::string& diacritic_str);
HFSTDLL static std::string::size_type find_diacritic
@@ -68,7 +56,7 @@ public:
HFSTDLL static std::string get_operator(const std::string& diacritic);
HFSTDLL static std::string get_feature(const std::string& diacritic);
HFSTDLL static std::string get_value(const std::string& diacritic);
- HFSTDLL static bool has_value(const std::string& diacritic);
+ HFSTDLL static bool has_value(const std::string& diacritic);
};
template<class T> class FdState;
@@ -107,52 +95,52 @@ public:
if(second_full_stop_pos == std::string::npos)
{
assert(op == Cop || op == Dop || op == Rop);
- feat = str.substr(first_full_stop_pos+1,
+ feat = str.substr(first_full_stop_pos+1,
last_char_pos-first_full_stop_pos-1);
}
else
{
- feat = str.substr(first_full_stop_pos+1,
+ feat = str.substr(first_full_stop_pos+1,
second_full_stop_pos-first_full_stop_pos-1);
- val = str.substr(second_full_stop_pos+1,
+ val = str.substr(second_full_stop_pos+1,
last_char_pos-second_full_stop_pos-1);
}
if(feature_map.count(feat) == 0)
{
- FdFeature next = feature_map.size();
+ FdFeature next = hfst::size_t_to_ushort(feature_map.size());
feature_map[feat] = next;
}
if(value_map.count(val) == 0)
{
- FdValue next = value_map.size()+1;
+ FdValue next = hfst::size_t_to_ushort(value_map.size()+1);
value_map[val] = next;
}
operations.insert
(std::pair<T,FdOperation>
- (symbol,
+ (symbol,
FdOperation(op, feature_map[feat], value_map[val], str)));
symbol_map.insert(std::pair<std::string,T>(str, symbol));
}
- FdFeature num_features() const { return feature_map.size(); }
+ FdFeature num_features() const { return (hfst::FdFeature)feature_map.size(); }
bool is_diacritic(T symbol) const
{ return operations.find(symbol) != operations.end(); }
const FdOperation* get_operation(T symbol) const
{
// for some reason this fails to compile???
- //std::map<T,FdOperation>::const_iterator i
+ //std::map<T,FdOperation>::const_iterator i
// = operations.find(symbol);
//return (i==operations.end()) ? NULL : &(i->second);
- return (operations.find(symbol)==operations.end()) ? NULL :
+ return (operations.find(symbol)==operations.end()) ? NULL :
&(operations.find(symbol)->second);
}
const FdOperation* get_operation(const std::string& symbol) const
{
- return (symbol_map.find(symbol)==symbol_map.end()) ? NULL :
+ return (symbol_map.find(symbol)==symbol_map.end()) ? NULL :
get_operation(symbol_map.find(symbol)->second);
}
@@ -176,7 +164,7 @@ public:
while(true)
{
- std::string::size_type next_diacritic_pos
+ std::string::size_type next_diacritic_pos
= FdOperation::find_diacritic(remaining, length);
if(next_diacritic_pos == std::string::npos)
break;
@@ -233,7 +221,7 @@ public:
if(op)
return apply_operation(*op);
return true; // if the symbol isn't a diacritic
- }
+ }
bool apply_operation(const FdOperation& op)
{
switch(op.Operator()) {
@@ -263,13 +251,13 @@ public:
case Uop: // unification
if(values[op.Feature()] == 0 || /* if the feature is unset or */
- values[op.Feature()] == op.Value() || /* the feature is at
- this value already
+ values[op.Feature()] == op.Value() || /* the feature is at
+ this value already
or */
(values[op.Feature()] < 0 &&
- (values[op.Feature()]*(-1) != op.Value())) /* the feature is
- negatively set
- to something
+ (values[op.Feature()]*(-1) != op.Value())) /* the feature is
+ negatively set
+ to something
else */
)
{
diff --git a/libhfst/src/HfstInputStream.cc b/libhfst/src/HfstInputStream.cc
index 1c1f64c..3aff8d5 100644
--- a/libhfst/src/HfstInputStream.cc
+++ b/libhfst/src/HfstInputStream.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
//! @FIXME: The structure of this class and its functions is disorganised.
@@ -119,7 +119,7 @@ namespace hfst
short &HfstInputStream::stream_get(short &i)
{
if (input_stream != NULL)
- {
+ {
input_stream->read((char*)&i,sizeof(i));
return i;
}
@@ -147,7 +147,7 @@ namespace hfst
#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
- return i =
+ return i =
this->implementation.my_transducer_library->stream_get_short();
break;
#endif
@@ -289,17 +289,17 @@ namespace hfst
bool HfstInputStream::set_implementation_specific_header_data
(StringPairVector& /* data */, unsigned int /* index*/)
{
+#if HAVE_SFST
switch (type)
{
-#if HAVE_SFST
case SFST_TYPE:
//return this->implementation.sfst->
//set_implementation_specific_header_data(data, index);
break;
-#endif
default:
break;
}
+#endif
return false;
}
@@ -337,17 +337,17 @@ namespace hfst
this->implementation.sfst->read_transducer();
/* If we were reading an SFST transducer with no HFST header,
- it is possible that epsilon is coded differently than
+ it is possible that epsilon is coded differently than
"@_EPSILON_SYMBOL_@" and/or that numbers 1 and 2 are reserved
- for other use than "@_UNKNOWN_SYMBOL_@" or
+ for other use than "@_UNKNOWN_SYMBOL_@" or
"@_IDENTITY_SYMBOL_@". */
if (not has_hfst_header)
{
- HfstBasicTransducer * net =
+ HfstBasicTransducer * net =
ConversionFunctions::
sfst_to_hfst_basic_transducer
(t.implementation.sfst);
- delete t.implementation.sfst;
+ hfst::implementations::SfstTransducer::delete_transducer(t.implementation.sfst);
t.implementation.sfst =
ConversionFunctions::
hfst_basic_transducer_to_sfst(net);
@@ -370,11 +370,11 @@ namespace hfst
"@_UNKNOWN_SYMBOL_@" or "@_IDENTITY_SYMBOL_@". */
if (! has_hfst_header)
{
- HfstBasicTransducer * net =
+ HfstBasicTransducer * net =
ConversionFunctions::
tropical_ofst_to_hfst_basic_transducer
(t.implementation.tropical_ofst, false);
- delete t.implementation.tropical_ofst;
+ t.tropical_ofst_interface.delete_transducer(t.implementation.tropical_ofst);
t.implementation.tropical_ofst =
ConversionFunctions::
hfst_basic_transducer_to_tropical_ofst(net);
@@ -393,10 +393,10 @@ namespace hfst
// special symbol-to-number mappings
- std::vector<std::pair<unsigned short, std::string> >
+ std::vector<std::pair<unsigned short, std::string> >
special_cases;
// normal symbol-to-number mappings
- std::vector<std::pair<unsigned short, std::string> >
+ std::vector<std::pair<unsigned short, std::string> >
symbol_mappings;
unsigned short max_number=0;
@@ -406,9 +406,9 @@ namespace hfst
max_number++;
unsigned short symbol_number=0;
- symbol_number = symbol_number +
+ symbol_number = symbol_number +
(unsigned short)stream_get() * 1;
- symbol_number = symbol_number +
+ symbol_number = symbol_number +
(unsigned short)stream_get() * 256;
std::string symbol_string("");
@@ -418,9 +418,9 @@ namespace hfst
c = stream_get();
}
- // epsilon, unknown and identity numbers must be handled
+ // epsilon, unknown and identity numbers must be handled
//separately
- if (symbol_number == 0 || symbol_number == 1 ||
+ if (symbol_number == 0 || symbol_number == 1 ||
symbol_number == 2) {
special_cases.push_back
(std::pair<unsigned short, std::string>
@@ -442,10 +442,10 @@ namespace hfst
else {
//special_cases[i].first, max_number+1);
- fst::StdVectorFst * tmp =
+ fst::StdVectorFst * tmp =
t.tropical_ofst_interface.substitute
- (t.implementation.tropical_ofst,
- special_cases[i].first,
+ (t.implementation.tropical_ofst,
+ special_cases[i].first,
(unsigned short)++max_number);
t.implementation.tropical_ofst = tmp;
@@ -486,11 +486,11 @@ namespace hfst
"@_UNKNOWN_SYMBOL_@" or "@_IDENTITY_SYMBOL_@". */
if (! has_hfst_header)
{
- HfstBasicTransducer * net =
+ HfstBasicTransducer * net =
ConversionFunctions::
log_ofst_to_hfst_basic_transducer
(t.implementation.log_ofst, false);
- delete t.implementation.log_ofst;
+ t.log_ofst_interface.delete_transducer(t.implementation.log_ofst);
t.implementation.log_ofst =
ConversionFunctions::
hfst_basic_transducer_to_log_ofst(net);
@@ -498,7 +498,7 @@ namespace hfst
}
if (hfst_version_2_weighted_transducer) // this should not happen
- {
+ {
//fprintf(stderr, "ERROR: not transducer stream\n");
//exit(1);
HFST_THROW_MESSAGE(HfstFatalException, "not transducer stream");
@@ -647,13 +647,13 @@ namespace hfst
return ERROR_TYPE_;
break;
}
+ case (char)0:
+ {
+ return XFSM_;
+ break;
+ }
default:
-#ifdef HAVE_XFSM
- return XFSM_;
-#else
return ERROR_TYPE_;
-#endif
-
}
return ERROR_TYPE_;
}
@@ -691,7 +691,7 @@ namespace hfst
strcmp("LOG_OFST", header_data[1].second.c_str()) == 0 )
type = LOG_OPENFST_TYPE;
#if HAVE_MY_TRANSDUCER_LIBRARY
- else if (strcmp("MY_TRANSDUCER_LIBRARY", header_data[1].second.c_str())
+ else if (strcmp("MY_TRANSDUCER_LIBRARY", header_data[1].second.c_str())
== 0 )
type = MY_TRANSDUCER_LIBRARY_TYPE;
#endif
@@ -720,10 +720,10 @@ namespace hfst
}
- /* Try to read a hfst header. If successful, return true and the number
- of bytes read. If not, return false and 0. Throw a
+ /* Try to read a hfst header. If successful, return true and the number
+ of bytes read. If not, return false and 0. Throw a
NotTransducerStreamException if the header cannot
- be parsed after a field "HFST3" or "HFST".
+ be parsed after a field "HFST3" or "HFST".
Throw a TransducerHeaderException if the header data cannot be parsed. */
bool HfstInputStream::read_hfst_header(int &bytes_read)
{
@@ -735,11 +735,11 @@ namespace hfst
}
int header_bytes=0;
// try to read an HFST version 3.0 header
- if (read_library_header(header_bytes))
+ if (read_library_header(header_bytes))
{
int size_bytes=0;
int header_size = get_header_size(size_bytes); // throws error
- StringPairVector header_info =
+ StringPairVector header_info =
get_header_data(header_size);
process_header_data(header_info, false); // throws error
@@ -748,7 +748,7 @@ namespace hfst
}
header_bytes=0;
// try to read a pre-release HFST version 3.0 header
- if (read_library_header_old(header_bytes))
+ if (read_library_header_old(header_bytes))
{
int type_bytes=0;
type = get_fst_type_old(type_bytes); // throws error
@@ -785,7 +785,7 @@ namespace hfst
return ERROR_TYPE;
}
- bool HfstInputStream::read_library_header_old(int &bytes_read)
+ bool HfstInputStream::read_library_header_old(int &bytes_read)
{
const char *id = "HFST3";
@@ -795,7 +795,7 @@ namespace hfst
stream_unget(c);
if(i > 0) {
for(int j=i-1; j>=0; j--) {
- stream_unget(id[j]);
+ stream_unget(id[j]);
}
}
bytes_read=0;
@@ -848,12 +848,12 @@ namespace hfst
StringPairVector retval;
int bytes_read=0;
- while(true)
+ while(true)
{
std::string str1 = stream_getstring();
std::string str2 = stream_getstring();
- bytes_read = bytes_read + (int)str1.length() + (int)str2.length() + 2;
+ bytes_read = bytes_read + (int)str1.length() + (int)str2.length() + 2;
if (bytes_read > header_size) {
debug_error("#7");
@@ -880,7 +880,7 @@ namespace hfst
/* The implementation type of the first transducer in the stream. */
ImplementationType HfstInputStream::stream_fst_type()
- {
+ {
int bytes_read=0;
// whether the stream contains an HFST version 3.0 transducer
@@ -937,7 +937,7 @@ namespace hfst
}
/* Open a transducer stream to stdout.
- The implementation type of the stream is defined by
+ The implementation type of the stream is defined by
the type of the first transducer in the stream. */
HfstInputStream::HfstInputStream(void):
bytes_to_skip(0), filename(std::string()), has_hfst_header(false),
@@ -949,7 +949,7 @@ namespace hfst
type = stream_fst_type();
if ( ! HfstTransducer::is_implementation_type_available(type)) {
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
}
switch (type)
@@ -961,12 +961,12 @@ namespace hfst
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
- implementation.tropical_ofst =
+ implementation.tropical_ofst =
new hfst::implementations::TropicalWeightInputStream;
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
- implementation.log_ofst =
+ implementation.log_ofst =
new hfst::implementations::LogWeightInputStream;
break;
#endif
@@ -984,7 +984,7 @@ namespace hfst
#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
- implementation.my_transducer_library
+ implementation.my_transducer_library
= new hfst::implementations::MyTransducerLibraryInputStream;
break;
#endif
@@ -993,7 +993,7 @@ namespace hfst
new hfst::implementations::HfstOlInputStream(false);
break;
case HFST_OLW_TYPE:
- implementation.hfst_ol =
+ implementation.hfst_ol =
new hfst::implementations::HfstOlInputStream(true);
break;
default:
@@ -1003,7 +1003,7 @@ namespace hfst
}
}
- // FIXME: HfstOutputStream takes a string parameter,
+ // FIXME: HfstOutputStream takes a string parameter,
// HfstInputStream a const char*
HfstInputStream::HfstInputStream(const std::string &filename):
bytes_to_skip(0), filename(std::string(filename)), has_hfst_header(false),
@@ -1012,7 +1012,7 @@ namespace hfst
if (strcmp("",filename.c_str()) != 0) {
std::ifstream ifs(filename.c_str());
if (ifs.fail())
- HFST_THROW_MESSAGE(NotTransducerStreamException,
+ HFST_THROW_MESSAGE(NotTransducerStreamException,
"file could not be opened");
input_stream = &ifs;
if (stream_eof())
@@ -1027,59 +1027,59 @@ namespace hfst
}
if ( ! HfstTransducer::is_implementation_type_available(type)) {
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
}
switch (type)
{
#if HAVE_SFST
case SFST_TYPE:
- implementation.sfst
+ implementation.sfst
= new hfst::implementations::SfstInputStream(filename);
break;
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
- if (strcmp(filename.c_str(),"") == 0) {
+ if (strcmp(filename.c_str(),"") == 0) {
// FIXME: this should be done in TropicalWeight layer
- implementation.tropical_ofst =
+ implementation.tropical_ofst =
new hfst::implementations::TropicalWeightInputStream();
}
else
- implementation.tropical_ofst =
+ implementation.tropical_ofst =
new hfst::implementations::TropicalWeightInputStream(filename);
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
- implementation.log_ofst =
+ implementation.log_ofst =
new hfst::implementations::LogWeightInputStream(filename);
break;
#endif
#endif
#if HAVE_FOMA
case FOMA_TYPE:
- implementation.foma
+ implementation.foma
= new hfst::implementations::FomaInputStream(filename);
break;
#endif
#if HAVE_XFSM
case XFSM_TYPE:
- implementation.xfsm
+ implementation.xfsm
= new hfst::implementations::XfsmInputStream(filename);
break;
#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
- implementation.my_transducer_library
+ implementation.my_transducer_library
= new hfst::implementations::MyTransducerLibraryInputStream(filename);
break;
#endif
case HFST_OL_TYPE:
- implementation.hfst_ol
+ implementation.hfst_ol
= new hfst::implementations::HfstOlInputStream(filename, false);
break;
case HFST_OLW_TYPE:
- implementation.hfst_ol
+ implementation.hfst_ol
= new hfst::implementations::HfstOlInputStream(filename, true);
break;
default:
@@ -1091,7 +1091,7 @@ namespace hfst
}
HfstInputStream::~HfstInputStream(void)
- {
+ {
switch (type)
{
#if HAVE_SFST
diff --git a/libhfst/src/HfstInputStream.h b/libhfst/src/HfstInputStream.h
index 8ce05be..96412f3 100644
--- a/libhfst/src/HfstInputStream.h
+++ b/libhfst/src/HfstInputStream.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _HFST_INPUTSTREAM_H_
@@ -18,7 +18,7 @@
#include "hfstdll.h"
-/** @file HfstInputStream.h
+/** @file HfstInputStream.h
\brief Declaration of class HfstInputStream.
*/
@@ -37,7 +37,7 @@ namespace hfst
#endif
#if HAVE_FOMA
class FomaInputStream;
-#endif
+#endif
#if HAVE_XFSM
class XfsmInputStream;
#endif
@@ -48,7 +48,7 @@ namespace hfst
}
- /** \brief A stream for reading HFST binary transducers.
+ /** \brief A stream for reading HFST binary transducers.
An example:
\verbatim
@@ -58,11 +58,11 @@ namespace hfst
HfstInputStream *in = NULL;
-try
+try
{
in = new HfstInputStream("testfile");
-}
-catch (StreamNotReadableException e)
+}
+catch (StreamNotReadableException e)
{
std::cerr << "ERROR: File does not exist." << std::endl;
exit(1);
@@ -70,20 +70,20 @@ catch (StreamNotReadableException e)
int transducers_read = 0;
-while (not in->is_eof())
+while (not in->is_eof())
{
- if (in->is_bad())
+ if (in->is_bad())
{
std::cerr << "ERROR: Stream cannot be read." << std::endl;
- exit(1);
+ exit(1);
}
HfstTransducer t(*in);
std::cerr << "One transducer successfully read." << std::endl;
transducers_read++;
}
-std::cerr << "Read "
- << transducers_read
+std::cerr << "Read "
+ << transducers_read
<< " transducers in total."
<< std::endl;
in->close();
@@ -117,7 +117,7 @@ For documentation on the HFST binary transducer format, see
#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
- hfst::implementations::MyTransducerLibraryInputStream *
+ hfst::implementations::MyTransducerLibraryInputStream *
my_transducer_library;
#endif
@@ -125,19 +125,19 @@ For documentation on the HFST binary transducer format, see
};
/* The backend implementation */
- StreamImplementation implementation;
+ StreamImplementation implementation;
/* Implementation type */
- ImplementationType type;
+ ImplementationType type;
/* Name of next transducer, given in the hfst header */
- std::string name;
+ std::string name;
std::map<std::string,std::string> props;
- /* How many bytes have been already read by the function
+ /* How many bytes have been already read by the function
when processing the hfst header */
- unsigned int bytes_to_skip;
+ unsigned int bytes_to_skip;
/* The name of the file, if stdin, name is "" */
- std::string filename;
+ std::string filename;
/* Whether the current transducer has an hfst header */
- bool has_hfst_header;
+ bool has_hfst_header;
/* A special case where an OpenFst transducer has no symbol tables but an
SFST alphabet is appended at the end. Should not occur very often, but
@@ -145,16 +145,16 @@ For documentation on the HFST binary transducer format, see
bool hfst_version_2_weighted_transducer;
/* The stream that the reading operations use when reading the first
- transducer. Then the type of the transducer is not known so there
+ transducer. Then the type of the transducer is not known so there
is no backend implementation whose reading functions could be used.
If input_stream==NULL, the backend implementation is used */
std::istream * input_stream;
- /* Basic stream operators, work on input_stream (if not NULL) or on
+ /* Basic stream operators, work on input_stream (if not NULL) or on
the stream implementation. */
/* Extract one character from the stream */
- char stream_get();
+ char stream_get();
/* Extract one character from the stream and store it in @a c. */
char &stream_get(char &c);
@@ -166,40 +166,40 @@ For documentation on the HFST binary transducer format, see
unsigned short &stream_get(unsigned short &i);
/* Return character c to the stream */
- void stream_unget(char c);
+ void stream_unget(char c);
/* Whether the stream is at end */
- bool stream_eof();
+ bool stream_eof();
/* Get a string from the stream */
- std::string stream_getstring();
+ std::string stream_getstring();
/* Return the next character in the stream without extracting it */
- char stream_peek();
+ char stream_peek();
/* The stream implementation ignores n bytes. */
void ignore(unsigned int n);
- /* The type of a transducer not supported directly by HFST version 3.0
+ /* The type of a transducer not supported directly by HFST version 3.0
but which can occur in conversion functions. */
- enum TransducerType {
+ enum TransducerType {
/* See the above variable. */
- HFST_VERSION_2_WEIGHTED,
+ HFST_VERSION_2_WEIGHTED,
/* An SFST transducer with no alphabet, not supported. */
HFST_VERSION_2_UNWEIGHTED_WITHOUT_ALPHABET,
- /* Old header + ordinary SFST transducer. */
+ /* Old header + ordinary SFST transducer. */
HFST_VERSION_2_UNWEIGHTED,
- /* An OpenFst transducer, can cause problems if it does not have
- symbol tables. */
+ /* An OpenFst transducer, can cause problems if it does not have
+ symbol tables. */
OPENFST_TROPICAL_,
OPENFST_LOG_,
- /* An SFST transducer. */
- SFST_,
- /* A foma transducer in unzipped format.
- A zipped file is handled by throwing a FileIsInGZFormatException. */
+ /* An SFST transducer. */
+ SFST_,
+ /* A foma transducer in unzipped format.
+ A zipped file is handled by throwing a FileIsInGZFormatException. */
FOMA_,
/* An xfsm transducer. */
XFSM_,
/* Your transducer type */
//MY_TRANSDUCER_LIBRARY_,
- /* Transducer type not recognized. */
- ERROR_TYPE_
+ /* Transducer type not recognized. */
+ ERROR_TYPE_
};
/* Read a transducer from the stream. */
@@ -211,36 +211,36 @@ For documentation on the HFST binary transducer format, see
TransducerType guess_fst_type(int &bytes_read);
bool read_hfst_header(int &bytes_read);
bool read_library_header(int &bytes_read);
- int get_header_size(int &bytes_read);
- StringPairVector get_header_data(int header_size);
+ int get_header_size(int &bytes_read);
+ StringPairVector get_header_data(int header_size);
void process_header_data
- (StringPairVector &header_data, bool warnings=false);
+ (StringPairVector &header_data, bool warnings=false);
bool set_implementation_specific_header_data
(StringPairVector &data, unsigned int index);
bool read_library_header_old(int &bytes_read);
- ImplementationType get_fst_type_old(int &bytes_read);
+ ImplementationType get_fst_type_old(int &bytes_read);
public:
- /** \brief Create a stream to standard input for reading binary
- transducers.
+ /** \brief Create a stream to standard input for reading binary
+ transducers.
- @throws StreamNotReadableException
+ @throws StreamNotReadableException
@throws NotTransducerStreamException
@throws EndOfStreamException
@throws TransducerHeaderException
*/
HFSTDLL HfstInputStream(void);
- /** \brief Open a stream to file \a filename for reading binary
- transducers.
+ /** \brief Open a stream to file \a filename for reading binary
+ transducers.
@pre The file exists. Otherwise, a StreamNotReadableException
is thrown.
- @throws StreamNotReadableException
+ @throws StreamNotReadableException
@throws NotTransducerStreamException
@throws EndOfStreamException
@throws TransducerHeaderException
@@ -262,7 +262,7 @@ For documentation on the HFST binary transducer format, see
/** \brief Whether the state of the stream is good for input operations. */
HFSTDLL bool is_good(void);
- /** \brief The type of the first transducer in the stream.
+ /** \brief The type of the first transducer in the stream.
By default, all transducers in a stream have the same type, else
a TransducerTypeMismatchException is thrown when reading the first
diff --git a/libhfst/src/HfstLookupFlagDiacritics.cc b/libhfst/src/HfstLookupFlagDiacritics.cc
index 47faab9..3d9ce6a 100644
--- a/libhfst/src/HfstLookupFlagDiacritics.cc
+++ b/libhfst/src/HfstLookupFlagDiacritics.cc
@@ -1,14 +1,16 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "HfstLookupFlagDiacritics.h"
+using hfst::StringVector;
+
DiacriticOperators FlagDiacriticTable::diacritic_operators;
DiacriticFeatures FlagDiacriticTable::diacritic_features;
DiacriticValues FlagDiacriticTable::diacritic_values;
@@ -84,7 +86,7 @@ void FlagDiacriticTable::split_diacritic(const std::string &diacritic_string)
// Third character is always the first fullstop.
size_t first_full_stop_pos = 2;
// Find the second full stop, if there is one.
- size_t second_full_stop_pos =
+ size_t second_full_stop_pos =
diacritic_string.find('.',first_full_stop_pos+1);
size_t last_char_pos = diacritic_string.size() - 1;
if (second_full_stop_pos == std::string::npos)
@@ -93,17 +95,17 @@ void FlagDiacriticTable::split_diacritic(const std::string &diacritic_string)
(diacritic_operators[diacritic_string] == Dop) ||
(diacritic_operators[diacritic_string] == Rop));
diacritic_has_value[diacritic_string] = false;
- diacritic_features[diacritic_string] =
+ diacritic_features[diacritic_string] =
diacritic_string.substr(first_full_stop_pos+1,
last_char_pos - first_full_stop_pos - 1);
}
else
{
diacritic_has_value[diacritic_string] = true;
- diacritic_features[diacritic_string] =
+ diacritic_features[diacritic_string] =
diacritic_string.substr(first_full_stop_pos+1,
second_full_stop_pos-first_full_stop_pos - 1);
- diacritic_values[diacritic_string] =
+ diacritic_values[diacritic_string] =
diacritic_string.substr(second_full_stop_pos+1,
last_char_pos - second_full_stop_pos - 1);
}
@@ -115,7 +117,7 @@ FlagDiacriticTable::FlagDiacriticTable(void):
bool FlagDiacriticTable::is_diacritic(const std::string &symbol)
{ //return diacritic_operators.find(symbol) != diacritic_operators.end(); }
- bool res = is_genuine_diacritic(symbol);
+ bool res = is_genuine_diacritic(symbol);
if (res)
split_diacritic(symbol);
return res;
@@ -150,9 +152,9 @@ void FlagDiacriticTable::require(std::string &feature,
std::string &value)
{
if (feature_values.find(feature) == feature_values.end())
- {
+ {
error_flag = true;
- return;
+ return;
}
else if (feature_values[feature] != value)
{ error_flag = true; }
@@ -172,7 +174,7 @@ void FlagDiacriticTable::unify(std::string &feature,
{ set_positive_value(feature,value); }
// If feature set to something else negatively, set it to value.
else if (feature_values[feature] != value)
- {
+ {
if (! feature_polarities[feature])
{ set_positive_value(feature,value); }
}
@@ -184,7 +186,7 @@ void FlagDiacriticTable::clear(std::string &feature)
feature_polarities.erase(feature);
}
-/*void
+/*void
FlagDiacriticTable::define_diacritic(short diacritic_number,
const std::string &diacritic_string)
{ if (is_genuine_diacritic(diacritic_string))
@@ -286,7 +288,7 @@ void FlagDiacriticTable::display(short diacritic)
}
else
{
- std::cout << diacritic_operators[diacritic]
+ std::cout << diacritic_operators[diacritic]
<< " "
<< diacritic_features[diacritic]
<< " "
@@ -372,8 +374,8 @@ int main(void)
std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == false);
fdt.reset();
- std::cout
- << "\"@P.NeedNoun.ON@ @P.BlaBla.ON@ @C.NeedNoun@ @D.NeedNoun.ON@\""
+ std::cout
+ << "\"@P.NeedNoun.ON@ @P.BlaBla.ON@ @C.NeedNoun@ @D.NeedNoun.ON@\""
<< " should pass:";
fdt.insert_number(1);
fdt.insert_number(7);
@@ -382,7 +384,7 @@ int main(void)
std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == false);
fdt.reset();
- std::cout
+ std::cout
<< "\"@P.NeedNoun.ON@ @P.BlaBla.ON@ @C.NeedNoun@ @D.NeedNoun.ON@ "
<< "@R.BlaBla.ON@\" should pass:";
fdt.insert_number(1);
@@ -393,7 +395,7 @@ int main(void)
std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == false);
fdt.reset();
- std::cout
+ std::cout
<< "\"@P.NeedNoun.ON@ @C.NeedNoun@ @D.NeedNoun.ON@ @R.BlaBla.ON@\""
<< " should fail:";
fdt.insert_number(1);
diff --git a/libhfst/src/HfstLookupFlagDiacritics.h b/libhfst/src/HfstLookupFlagDiacritics.h
index 9a56634..f2c28b9 100644
--- a/libhfst/src/HfstLookupFlagDiacritics.h
+++ b/libhfst/src/HfstLookupFlagDiacritics.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _LOOKUP_FLAG_DIACRITICS_H_
@@ -14,9 +14,9 @@
#include <string>
#include <cassert>
#include <vector>
-#include <iostream>
+#include <iosfwd>
#ifdef DEBUG
-#include <iostream>
+#include <iosfwd>
#endif
enum DiacriticOperator
@@ -30,7 +30,10 @@ typedef std::map<std::string,std::string> DiacriticValues;
typedef std::map<std::string,bool> DiacriticSettingMap;
typedef std::map<std::string,std::string> FeatureValues;
typedef std::map<std::string,bool> FeaturePolarities;
-typedef std::vector<std::string> StringVector;
+
+namespace hfst {
+ typedef std::vector<std::string> StringVector; }
+
class FlagDiacriticTable
{
static DiacriticOperators diacritic_operators;
@@ -65,10 +68,10 @@ class FlagDiacriticTable
void reset(void);
bool fails(void);
static bool is_diacritic(const std::string &symbol);
- bool is_valid_string(const StringVector & input_string);
- StringVector filter_diacritics(const StringVector & input_string);
+ bool is_valid_string(const hfst::StringVector & input_string);
+ hfst::StringVector filter_diacritics(const hfst::StringVector & input_string);
#ifdef DEBUG
- static void display(short diacritic);
+ static void display(short diacritic);
#endif
};
diff --git a/libhfst/src/HfstOutputStream.cc b/libhfst/src/HfstOutputStream.cc
index 45b62a3..d260eae 100644
--- a/libhfst/src/HfstOutputStream.cc
+++ b/libhfst/src/HfstOutputStream.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include <string>
@@ -20,46 +20,46 @@ namespace hfst
{
HfstOutputStream::HfstOutputStream(ImplementationType type, bool hfst_format):
type(type), hfst_format(hfst_format), is_open(false)
- {
+ {
if (! HfstTransducer::is_implementation_type_available(type)) {
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
}
switch(type)
{
#if HAVE_SFST
case SFST_TYPE:
- implementation.sfst =
+ implementation.sfst =
new hfst::implementations::SfstOutputStream();
break;
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
- implementation.tropical_ofst =
+ implementation.tropical_ofst =
new hfst::implementations::TropicalWeightOutputStream(hfst_format);
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
- implementation.log_ofst =
+ implementation.log_ofst =
new hfst::implementations::LogWeightOutputStream();
break;
#endif
#endif
#if HAVE_FOMA
case FOMA_TYPE:
- implementation.foma =
+ implementation.foma =
new hfst::implementations::FomaOutputStream();
break;
#endif
#if HAVE_XFSM
case XFSM_TYPE:
- implementation.xfsm =
+ implementation.xfsm =
new hfst::implementations::XfsmOutputStream(); // throws error, not implemented
break;
#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
- implementation.my_transducer_library =
+ implementation.my_transducer_library =
new hfst::implementations::MyTransducerLibraryOutputStream(hfst_format);
break;
#endif
@@ -78,54 +78,54 @@ namespace hfst
this->is_open=true;
}
- // FIXME: HfstOutputStream takes a string parameter,
+ // FIXME: HfstOutputStream takes a string parameter,
// HfstInputStream a const char*
HfstOutputStream::HfstOutputStream
(const std::string &filename,ImplementationType type, bool hfst_format_):
type(type), hfst_format(hfst_format_), is_open(false)
- {
+ {
if (! HfstTransducer::is_implementation_type_available(type)) {
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
}
switch(type)
{
#if HAVE_SFST
case SFST_TYPE:
- implementation.sfst =
+ implementation.sfst =
new hfst::implementations::SfstOutputStream(filename);
break;
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
// FIXME: this should be done in TropicalWeight layer
- if (filename.compare("") == 0)
- implementation.tropical_ofst =
+ if (filename.compare("") == 0)
+ implementation.tropical_ofst =
new hfst::implementations::TropicalWeightOutputStream(hfst_format);
else
- implementation.tropical_ofst =
+ implementation.tropical_ofst =
new hfst::implementations::TropicalWeightOutputStream
(filename, hfst_format);
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
- implementation.log_ofst =
+ implementation.log_ofst =
new hfst::implementations::LogWeightOutputStream(filename);
break;
#endif
#endif
#if HAVE_FOMA
case FOMA_TYPE:
- implementation.foma =
+ implementation.foma =
new hfst::implementations::FomaOutputStream(filename);
break;
#endif
#if HAVE_XFSM
case XFSM_TYPE:
- /* XFSM api only offers a function that reads transducers that takes a filename argument.
+ /* XFSM api only offers a function that reads transducers that takes a filename argument.
That is why we don't write an HFST header. */
- hfst_format = false;
- implementation.xfsm =
+ hfst_format = false;
+ implementation.xfsm =
new hfst::implementations::XfsmOutputStream(filename);
break;
#endif
@@ -317,22 +317,22 @@ HfstOutputStream::append_implementation_specific_header_data(std::vector<char>&
HfstTransducer&
transducer)
#else
-void
+void
HfstOutputStream::append_implementation_specific_header_data(std::vector<char>&,
HfstTransducer&)
#endif
{
+#if HAVE_SFST
switch(type)
{
-#if HAVE_SFST
case SFST_TYPE:
implementation.sfst->append_implementation_specific_header_data
(header, transducer.implementation.sfst);
break;
-#endif
default:
break;
}
+#endif
}
HfstOutputStream &HfstOutputStream::flush()
@@ -362,7 +362,7 @@ HfstOutputStream::append_implementation_specific_header_data(std::vector<char>&,
if (type != transducer.type)
{
- HFST_THROW_MESSAGE(TransducerTypeMismatchException,
+ HFST_THROW_MESSAGE(TransducerTypeMismatchException,
"operator<<: "
"HfstOutputStream and HfstTransducer do not "
"have the same type");
@@ -374,7 +374,7 @@ HfstOutputStream::append_implementation_specific_header_data(std::vector<char>&,
- the fifth char is a separator: "\0"
- the sixth and seventh char tell the length of the rest of the header
(beginning after the eighth char)
- - the eighth char is a separator and is not counted
+ - the eighth char is a separator and is not counted
to the header length: "\0"
- the rest of the header consists of pairs of attributes and their values
that are each separated by a char "\0"
@@ -387,13 +387,13 @@ HfstOutputStream::append_implementation_specific_header_data(std::vector<char>&,
"type\0" "FOMA\0"
"name\0" "\0"
- This is the header of a version 3.0 HFST transducer whose implementation
- type is foma and whose name is not defined, i.e. is the empty string "".
+ This is the header of a version 3.0 HFST transducer whose implementation
+ type is foma and whose name is not defined, i.e. is the empty string "".
The two bytes "\0\x1c" that form the length field tell that the length of
the rest of the header (i.e. the sequence of bytes
"version\03.0\0type\0FOMA\0name\0\0") is 0 * 256 + 28 * 1 = 28 bytes.
- HFST version 3.0 header must contain at least the attributes 'version',
+ HFST version 3.0 header must contain at least the attributes 'version',
'type' and 'name' and their values. Implementation-specific attributes
can follow after these obligatory attributes.
@@ -457,7 +457,7 @@ HfstOutputStream::append_implementation_specific_header_data(std::vector<char>&,
case TROPICAL_OPENFST_TYPE:
implementation.tropical_ofst->write_transducer
(transducer.implementation.tropical_ofst);
- return *this;
+ return *this;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
implementation.log_ofst->write_transducer
diff --git a/libhfst/src/HfstOutputStream.h b/libhfst/src/HfstOutputStream.h
index 95484cc..a112b28 100644
--- a/libhfst/src/HfstOutputStream.h
+++ b/libhfst/src/HfstOutputStream.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _HFST_OUTPUTSTREAM_H_
@@ -36,10 +36,10 @@ namespace hfst
#endif
#if HAVE_FOMA
class FomaOutputStream;
-#endif
+#endif
#if HAVE_XFSM
class XfsmOutputStream;
-#endif
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
class MyTransducerLibraryOutputStream;
#endif
@@ -47,7 +47,7 @@ namespace hfst
}
- /** \brief A stream for writing binary transducers.
+ /** \brief A stream for writing binary transducers.
An example:
\verbatim
@@ -57,13 +57,13 @@ namespace hfst
// Write three HFST transducers in binary format to file named "testfile"
HfstOutputStream out("testfile", FOMA_TYPE);
- out << foma_transducer1
- << foma_transducer2
+ out << foma_transducer1
+ << foma_transducer2
<< foma_transducer3;
out.close();
\endverbatim
-For more information on HFST transducer structure, see
+For more information on HFST transducer structure, see
<a href="HeaderFormatAndConversions.html">this page</a>.
**/
@@ -89,7 +89,7 @@ For more information on HFST transducer structure, see
#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
- hfst::implementations::MyTransducerLibraryOutputStream *
+ hfst::implementations::MyTransducerLibraryOutputStream *
my_transducer_library;
#endif
@@ -97,7 +97,7 @@ For more information on HFST transducer structure, see
};
ImplementationType type; // type of the stream implementation
// whether an hfst header is written before every transducer
- bool hfst_format;
+ bool hfst_format;
StreamImplementation implementation; // backend implementation
// write data to stream
@@ -113,7 +113,7 @@ For more information on HFST transducer structure, see
// append obligatory HFST header data to \a header
void append_hfst_header_data(std::vector<char> &header);
- /* append implementation-specific header data collected from
+ /* append implementation-specific header data collected from
\a transducer to \a header */
void append_implementation_specific_header_data
(std::vector<char> &header, HfstTransducer &transducer);
@@ -121,24 +121,24 @@ For more information on HFST transducer structure, see
public:
- /** \brief Create a stream to standard output for writing
- binary transducers of type \a type.
- \a hfst_format defines whether transducers are written
- in hfst format or as such in their backend format.
+ /** \brief Create a stream to standard output for writing
+ binary transducers of type \a type.
+ \a hfst_format defines whether transducers are written
+ in hfst format or as such in their backend format.
*/
HFSTDLL HfstOutputStream(ImplementationType type, bool hfst_format=true);
/** \brief Open a stream to file \a filename for writing binary transducers
- of type \a type.
- \a hfst_format defines whether transducers are written in hfst format
+ of type \a type.
+ \a hfst_format defines whether transducers are written in hfst format
or as such in their backend format.
- If the file exists, it is overwritten.
+ If the file exists, it is overwritten.
*/
HFSTDLL HfstOutputStream(const std::string &filename, ImplementationType type, bool hfst_format=true);
/** \brief Destructor. */
- HFSTDLL ~HfstOutputStream(void);
+ HFSTDLL ~HfstOutputStream(void);
/** \brief Flush the stream.
@@ -146,11 +146,11 @@ For more information on HFST transducer structure, see
are actually written to the stream. Else, does nothing. */
HFSTDLL HfstOutputStream &flush();
- /** \brief Write the transducer \a transducer in binary format
- to the stream.
+ /** \brief Write the transducer \a transducer in binary format
+ to the stream.
All transducers must have the same type as the stream, else a
- TransducerTypeMismatchException is thrown.
+ TransducerTypeMismatchException is thrown.
If the stream is of XFSM_TYPE, \a transducer is stored to a list
and written when flush() is called for the stream.
@@ -159,12 +159,12 @@ For more information on HFST transducer structure, see
*/
HFSTDLL HfstOutputStream &operator<< (HfstTransducer &transducer);
- /** @brief An alias for operator<<.
+ /** @brief An alias for operator<<.
@see operator<< */
HFSTDLL HfstOutputStream& redirect (HfstTransducer &transducer);
- /** \brief Close the stream.
+ /** \brief Close the stream.
If the stream points to standard output, nothing is done. */
HFSTDLL void close(void);
diff --git a/libhfst/src/HfstPrintDot.cc b/libhfst/src/HfstPrintDot.cc
index 0fb5d76..a0803a6 100644
--- a/libhfst/src/HfstPrintDot.cc
+++ b/libhfst/src/HfstPrintDot.cc
@@ -1,12 +1,17 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
+#include <cstdio>
+#if defined(_MSC_VER) && _MSC_VER < 1900
+ #include <cstdarg>
+#endif
+
#include "HfstTransducer.h"
using hfst::HfstTransducer;
using hfst::HfstBasicTransducer;
@@ -14,6 +19,37 @@ using hfst::implementations::HfstState;
namespace hfst {
+#if defined(_MSC_VER) && _MSC_VER < 1900
+
+#define snprintf c99_snprintf
+#define vsnprintf c99_vsnprintf
+
+ __inline int c99_vsnprintf(char *outBuf, size_t size, const char *format, va_list ap)
+ {
+ int count = -1;
+
+ if (size != 0)
+ count = _vsnprintf_s(outBuf, size, _TRUNCATE, format, ap);
+ if (count == -1)
+ count = _vscprintf(format, ap);
+
+ return count;
+ }
+
+ __inline int c99_snprintf(char *outBuf, size_t size, const char *format, ...)
+ {
+ int count;
+ va_list ap;
+
+ va_start(ap, format);
+ count = c99_vsnprintf(outBuf, size, format, ap);
+ va_end(ap);
+
+ return count;
+ }
+
+#endif
+
void
print_dot(FILE* out, HfstTransducer& t)
{
@@ -53,7 +89,7 @@ print_dot(FILE* out, HfstTransducer& t)
}
else
{
- fprintf(out, "q%d [label=\"q%d\"] \n",
+ fprintf(out, "q%d [label=\"q%d\"] \n",
s, s);
}
++s;
@@ -64,7 +100,7 @@ print_dot(FILE* out, HfstTransducer& t)
++state)
{
std::map<HfstState, std::string> target_labels;
- for (HfstBasicTransducer::HfstTransitions::const_iterator arc =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator arc =
state->begin();
arc != state->end();
++arc)
@@ -81,7 +117,7 @@ print_dot(FILE* out, HfstTransducer& t)
first = std::string("??");
}
else if (first == hfst::internal_unknown)
- {
+ {
first = std::string("?1");
}
if (second == hfst::internal_epsilon)
@@ -97,7 +133,7 @@ print_dot(FILE* out, HfstTransducer& t)
second = std::string("?2");
}
#define DOT_MAX_LABEL_SIZE 64
- char* l = static_cast<char*>(malloc(sizeof(char) *
+ char* l = static_cast<char*>(malloc(sizeof(char) *
DOT_MAX_LABEL_SIZE));
if (first == second)
{
@@ -113,7 +149,7 @@ print_dot(FILE* out, HfstTransducer& t)
HFST_THROW_MESSAGE(HfstException, "sprinting dot arc label");
}
}
- else
+ else
{
if (snprintf(l, DOT_MAX_LABEL_SIZE,
"%s/%.2f", first.c_str(),
@@ -134,7 +170,7 @@ print_dot(FILE* out, HfstTransducer& t)
HFST_THROW_MESSAGE(HfstException, "sprinting dot arc label");
}
}
- else
+ else
{
if (snprintf(l, DOT_MAX_LABEL_SIZE,
"%s", first.c_str()) < 0)
@@ -143,7 +179,7 @@ print_dot(FILE* out, HfstTransducer& t)
"sprinting dot arc label");
}
} // if old label empty
- } // if weighted
+ } // if weighted
} // if id pair
else
{
@@ -156,7 +192,7 @@ print_dot(FILE* out, HfstTransducer& t)
first.c_str(), second.c_str(),
arc->get_weight()) < 0)
{
- HFST_THROW_MESSAGE(HfstException,
+ HFST_THROW_MESSAGE(HfstException,
"sprinting dot arc label");
}
}
@@ -241,7 +277,7 @@ print_dot(std::ostream & out, HfstTransducer& t)
{
if (mutt->get_final_weight(s) > 0)
{
- out << "q" << s << " [shape=doublecircle," <<
+ out << "q" << s << " [shape=doublecircle," <<
"label=\"q" << s << "/\\n" << mutt->get_final_weight(s) << "\"] " << std::endl;
}
else
@@ -262,7 +298,7 @@ print_dot(std::ostream & out, HfstTransducer& t)
++state)
{
std::map<HfstState, std::string> target_labels;
- for (HfstBasicTransducer::HfstTransitions::const_iterator arc =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator arc =
state->begin();
arc != state->end();
++arc)
@@ -279,7 +315,7 @@ print_dot(std::ostream & out, HfstTransducer& t)
first = std::string("??");
}
else if (first == hfst::internal_unknown)
- {
+ {
first = std::string("?1");
}
if (second == hfst::internal_epsilon)
@@ -295,7 +331,7 @@ print_dot(std::ostream & out, HfstTransducer& t)
second = std::string("?2");
}
#define DOT_MAX_LABEL_SIZE 64
- char* l = static_cast<char*>(malloc(sizeof(char) *
+ char* l = static_cast<char*>(malloc(sizeof(char) *
DOT_MAX_LABEL_SIZE));
if (first == second)
{
@@ -311,7 +347,7 @@ print_dot(std::ostream & out, HfstTransducer& t)
HFST_THROW_MESSAGE(HfstException, "sprinting dot arc label");
}
}
- else
+ else
{
if (snprintf(l, DOT_MAX_LABEL_SIZE,
"%s/%.2f", first.c_str(),
@@ -332,7 +368,7 @@ print_dot(std::ostream & out, HfstTransducer& t)
HFST_THROW_MESSAGE(HfstException, "sprinting dot arc label");
}
}
- else
+ else
{
if (snprintf(l, DOT_MAX_LABEL_SIZE,
"%s", first.c_str()) < 0)
@@ -341,7 +377,7 @@ print_dot(std::ostream & out, HfstTransducer& t)
"sprinting dot arc label");
}
} // if old label empty
- } // if weighted
+ } // if weighted
} // if id pair
else
{
@@ -354,7 +390,7 @@ print_dot(std::ostream & out, HfstTransducer& t)
first.c_str(), second.c_str(),
arc->get_weight()) < 0)
{
- HFST_THROW_MESSAGE(HfstException,
+ HFST_THROW_MESSAGE(HfstException,
"sprinting dot arc label");
}
}
diff --git a/libhfst/src/HfstPrintDot.h b/libhfst/src/HfstPrintDot.h
index cc32dbd..a078390 100644
--- a/libhfst/src/HfstPrintDot.h
+++ b/libhfst/src/HfstPrintDot.h
@@ -1,16 +1,16 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
-#include "HfstTransducer.h"
-#include <iostream>
+#include "HfstDataTypes.h"
+#include <iosfwd>
-namespace hfst
+namespace hfst
{
void print_dot(FILE* out, HfstTransducer& t);
void print_dot(std::ostream & oss, HfstTransducer& t);
diff --git a/libhfst/src/HfstPrintPCKimmo.cc b/libhfst/src/HfstPrintPCKimmo.cc
index 83fc4d1..935770a 100644
--- a/libhfst/src/HfstPrintPCKimmo.cc
+++ b/libhfst/src/HfstPrintPCKimmo.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "HfstTransducer.h"
@@ -25,14 +25,14 @@ print_pckimmo(FILE* out, HfstTransducer& t)
state != mutt->end();
++state)
{
- for (HfstBasicTransducer::HfstTransitions::const_iterator arc =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator arc =
state->begin();
arc != state->end();
++arc)
{
std::string first = arc->get_input_symbol();
std::string second = arc->get_output_symbol();
- pairs.insert(pair<std::string,std::string>(first, second));
+ pairs.insert(std::pair<std::string,std::string>(first, second));
}
++last;
}
@@ -40,10 +40,10 @@ print_pckimmo(FILE* out, HfstTransducer& t)
unsigned int numwidth = 0;
for (unsigned int i = 1; i < last; i *= 10, numwidth++)
{}
- // first line is input symbols per pair
+ // first line is input symbols per pair
// (left corner is digit width + 2)
fprintf(out, "%*s ", numwidth, " ");
- for (std::set<pair<std::string, std::string> >::const_iterator p = pairs.begin();
+ for (std::set<std::pair<std::string, std::string> >::const_iterator p = pairs.begin();
p != pairs.end();
++p)
{
@@ -64,7 +64,7 @@ print_pckimmo(FILE* out, HfstTransducer& t)
fprintf(out, "\n");
// (left corner is digit width + 2)
fprintf(out, "%*s ", numwidth, " ");
- for (std::set<pair<std::string, std::string> >::const_iterator p = pairs.begin();
+ for (std::set<std::pair<std::string, std::string> >::const_iterator p = pairs.begin();
p != pairs.end();
++p)
{
@@ -98,23 +98,23 @@ print_pckimmo(FILE* out, HfstTransducer& t)
}
// map everything to sink state 0 first
std::map<std::pair<std::string,std::string>,HfstState> transitions;
- for(std::set<pair<std::string,std::string> >::const_iterator p = pairs.begin();
+ for(std::set<std::pair<std::string,std::string> >::const_iterator p = pairs.begin();
p != pairs.end();
++p)
{
transitions[*p] = -1;
}
- for (HfstBasicTransducer::HfstTransitions::const_iterator arc =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator arc =
state->begin();
arc != state->end();
++arc)
{
std::string first = arc->get_input_symbol();
std::string second = arc->get_output_symbol();
- transitions[pair<std::string,std::string>(first,second)] =
+ transitions[std::pair<std::string,std::string>(first,second)] =
arc->get_target_state();
}
- for(std::map<std::pair<std::string,std::string>,HfstState>::const_iterator trans =
+ for(std::map<std::pair<std::string,std::string>,HfstState>::const_iterator trans =
transitions.begin();
trans != transitions.end();
++trans)
diff --git a/libhfst/src/HfstPrintPCKimmo.h b/libhfst/src/HfstPrintPCKimmo.h
index 04c5c21..7791d55 100644
--- a/libhfst/src/HfstPrintPCKimmo.h
+++ b/libhfst/src/HfstPrintPCKimmo.h
@@ -1,15 +1,15 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
-#include "HfstTransducer.h"
+#include "HfstDataTypes.h"
-namespace hfst
+namespace hfst
{
void print_pckimmo(FILE* out, HfstTransducer& t);
}
diff --git a/libhfst/src/HfstRules.cc b/libhfst/src/HfstRules.cc
index 6fa5176..b9c7a0a 100644
--- a/libhfst/src/HfstRules.cc
+++ b/libhfst/src/HfstRules.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "HfstTransducer.h"
@@ -15,10 +15,10 @@ namespace hfst
namespace rules
{
- HfstTransducer replace( HfstTransducer &t,
- ReplaceType repl_type,
- bool optional,
- StringPairSet &alphabet )
+ HfstTransducer replace( HfstTransducer &t,
+ ReplaceType repl_type,
+ bool optional,
+ StringPairSet &alphabet )
{
ImplementationType type = t.get_type();
@@ -57,13 +57,13 @@ namespace hfst
return retval;
}
- HfstTransducer replace_transducer(HfstTransducer &t,
- std::string lm, std::string rm,
- ReplaceType repl_type,
+ HfstTransducer replace_transducer(HfstTransducer &t,
+ std::string lm, std::string rm,
+ ReplaceType repl_type,
StringPairSet &alphabet)
{
- t.minimize();
+ t.optimize();
ImplementationType type = t.get_type();
@@ -76,18 +76,18 @@ namespace hfst
tm.concatenate(tc);
tm.concatenate(rmtr);
- tm.minimize();
+ tm.optimize();
HfstTransducer retval = replace(tm, repl_type, false, alphabet);
- retval.minimize();
+ retval.optimize();
return retval;
}
- HfstTransducer replace_context(HfstTransducer &t,
- std::string m1, std::string m2,
+ HfstTransducer replace_context(HfstTransducer &t,
+ std::string m1, std::string m2,
StringPairSet &alphabet)
{
// ct = .* ( m1 >> ( m2 >> t )) || !(.* m1)
@@ -144,16 +144,16 @@ namespace hfst
HfstTransducer retval(pi_star);
retval.subtract(disj);
- retval.minimize();
+ retval.optimize();
return retval;
}
/* identical to ![ .* l [a:. & !a:b] r .* ] */
- HfstTransducer two_level_if(HfstTransducerPair &context,
- StringPairSet &mappings,
- StringPairSet &alphabet)
+ HfstTransducer two_level_if(HfstTransducerPair &context,
+ StringPairSet &mappings,
+ StringPairSet &alphabet)
{
if (context.first.get_type() != context.second.get_type()) {
HFST_THROW_MESSAGE(TransducerTypeMismatchException,
@@ -166,7 +166,7 @@ namespace hfst
// calculate [ a:. ]
StringPairSet input_to_any;
- for (StringPairSet::iterator it = mappings.begin();
+ for (StringPairSet::iterator it = mappings.begin();
it != mappings.end(); it++)
{
for (StringPairSet::iterator alpha_it = alphabet.begin();
@@ -211,12 +211,12 @@ namespace hfst
// equivalent to !(!(.* l) a:b .* | .* a:b !(r .*))
- HfstTransducer two_level_only_if(HfstTransducerPair &context,
- StringPairSet &mappings,
- StringPairSet &alphabet)
- {
+ HfstTransducer two_level_only_if(HfstTransducerPair &context,
+ StringPairSet &mappings,
+ StringPairSet &alphabet)
+ {
if (context.first.get_type() != context.second.get_type()) {
- HFST_THROW_MESSAGE(TransducerTypeMismatchException,
+ HFST_THROW_MESSAGE(TransducerTypeMismatchException,
"rules::two_level_only_if");
}
ImplementationType type = context.first.get_type();
@@ -257,7 +257,7 @@ namespace hfst
HfstTransducer two_level_if_and_only_if(HfstTransducerPair &context,
StringPairSet &mappings,
- StringPairSet &alphabet)
+ StringPairSet &alphabet)
{
HfstTransducer if_rule = two_level_if(context, mappings, alphabet);
HfstTransducer only_if_rule = two_level_only_if
@@ -265,20 +265,20 @@ namespace hfst
return if_rule.intersect(only_if_rule);
}
- HfstTransducer replace_in_context(HfstTransducerPair &context,
- ReplaceType repl_type,
- HfstTransducer &t,
- bool optional,
+ HfstTransducer replace_in_context(HfstTransducerPair &context,
+ ReplaceType repl_type,
+ HfstTransducer &t,
+ bool optional,
StringPairSet &alphabet)
{
// test that all transducers have the same type
- if (context.first.get_type() != context.second.get_type() ||
+ if (context.first.get_type() != context.second.get_type() ||
context.first.get_type() != t.get_type() ) {
HFST_THROW_MESSAGE(TransducerTypeMismatchException,
"rules::replace_in_context");
}
- ImplementationType type = t.get_type();
+ ImplementationType type = t.get_type();
// test that both context transducers are automata
// this could be done more efficiently...
@@ -297,13 +297,13 @@ namespace hfst
// HfstTransducer pi(alphabet, type);
- // Create the insert boundary transducer (.|<>:<L>|<>:<R>)*
+ // Create the insert boundary transducer (.|<>:<L>|<>:<R>)*
StringPairSet pi1 = alphabet;
pi1.insert(StringPair(internal_epsilon, leftm));
pi1.insert(StringPair(internal_epsilon, rightm));
HfstTransducer ibt(pi1, type, true);
- // Create the remove boundary transducer (.|<L>:<>|<R>:<>)*
+ // Create the remove boundary transducer (.|<L>:<>|<R>:<>)*
StringPairSet pi2 = alphabet;
pi2.insert(StringPair(leftm, internal_epsilon));
pi2.insert(StringPair(rightm, internal_epsilon));
@@ -324,39 +324,39 @@ namespace hfst
tmp.concatenate(pi_star);
HfstTransducer cbt(pi_star);
cbt.subtract(tmp);
- cbt.minimize();
+ cbt.optimize();
- // left context transducer .* (<R> >> (<L> >> LEFT_CONTEXT)) || !(.*<L>)
+ // left context transducer .* (<R> >> (<L> >> LEFT_CONTEXT)) || !(.*<L>)
HfstTransducer lct = replace_context
- (context.first, leftm, rightm, alphabet);
+ (context.first, leftm, rightm, alphabet);
- lct.minimize();
+ lct.optimize();
- // right context transducer:
+ // right context transducer:
// reversion( (<R> >> (<L> >> reversion(RIGHT_CONTEXT))) .* || !(<R>.*) )
HfstTransducer right_rev(context.second);
right_rev.reverse();
- right_rev.minimize();
+ right_rev.optimize();
HfstTransducer rct = replace_context(right_rev, rightm, leftm, alphabet);
rct.reverse();
- rct.minimize();
+ rct.optimize();
- // unconditional replace transducer
+ // unconditional replace transducer
HfstTransducer rt(type);
- if (repl_type == REPL_UP || repl_type == REPL_RIGHT ||
+ if (repl_type == REPL_UP || repl_type == REPL_RIGHT ||
repl_type == REPL_LEFT || repl_type == REPL_DOWN_KARTTUNEN)
rt = replace_transducer( t, leftm, rightm, REPL_UP, alphabet );
else
rt = replace_transducer( t, leftm, rightm, REPL_DOWN, alphabet );
- rt.minimize();
+ rt.optimize();
// build the conditional replacement transducer
HfstTransducer result(ibt);
result.compose(cbt);
- result.minimize(); // added
+ result.optimize(); // added
if (repl_type == REPL_UP || repl_type == REPL_RIGHT)
result.compose(rct);
@@ -364,7 +364,7 @@ namespace hfst
if (repl_type == REPL_UP || repl_type == REPL_LEFT)
result.compose(lct);
- result.minimize(); // ADDED
+ result.optimize(); // ADDED
result.compose(rt);
@@ -376,7 +376,7 @@ namespace hfst
repl_type == REPL_DOWN_KARTTUNEN)
result.compose(rct);
- result.minimize(); // ADDED
+ result.optimize(); // ADDED
result.compose(rbt);
@@ -389,64 +389,64 @@ namespace hfst
result.disjunct(pi_star_);
}
- result.minimize();
+ result.optimize();
return result;
}
- HfstTransducer replace_up(HfstTransducerPair &context,
- HfstTransducer &mapping,
- bool optional,
- StringPairSet &alphabet)
- {
+ HfstTransducer replace_up(HfstTransducerPair &context,
+ HfstTransducer &mapping,
+ bool optional,
+ StringPairSet &alphabet)
+ {
return replace_in_context(context, REPL_UP, mapping, optional, alphabet);
}
- HfstTransducer replace_down(HfstTransducerPair &context,
- HfstTransducer &mapping,
- bool optional,
- StringPairSet &alphabet)
- {
+ HfstTransducer replace_down(HfstTransducerPair &context,
+ HfstTransducer &mapping,
+ bool optional,
+ StringPairSet &alphabet)
+ {
return replace_in_context
(context, REPL_DOWN, mapping, optional, alphabet);
}
- HfstTransducer replace_down_karttunen(HfstTransducerPair &context,
- HfstTransducer &mapping,
- bool optional,
- StringPairSet &alphabet)
- {
+ HfstTransducer replace_down_karttunen(HfstTransducerPair &context,
+ HfstTransducer &mapping,
+ bool optional,
+ StringPairSet &alphabet)
+ {
return replace_in_context
(context, REPL_DOWN_KARTTUNEN, mapping, optional, alphabet);
}
- HfstTransducer replace_right(HfstTransducerPair &context,
- HfstTransducer &mapping,
- bool optional,
- StringPairSet &alphabet)
- {
+ HfstTransducer replace_right(HfstTransducerPair &context,
+ HfstTransducer &mapping,
+ bool optional,
+ StringPairSet &alphabet)
+ {
return replace_in_context
(context, REPL_RIGHT, mapping, optional, alphabet);
}
- HfstTransducer replace_left(HfstTransducerPair &context,
- HfstTransducer &mapping,
- bool optional,
+ HfstTransducer replace_left(HfstTransducerPair &context,
+ HfstTransducer &mapping,
+ bool optional,
StringPairSet &alphabet)
- {
+ {
return replace_in_context
(context, REPL_LEFT, mapping, optional, alphabet);
}
- HfstTransducer replace_up(HfstTransducer &mapping,
- bool optional,
+ HfstTransducer replace_up(HfstTransducer &mapping,
+ bool optional,
StringPairSet &alphabet)
{
return replace(mapping, REPL_UP, optional, alphabet);
}
- HfstTransducer replace_down(HfstTransducer &mapping,
- bool optional,
+ HfstTransducer replace_down(HfstTransducer &mapping,
+ bool optional,
StringPairSet &alphabet)
{
return replace(mapping, REPL_DOWN, optional, alphabet);
@@ -548,12 +548,12 @@ namespace hfst
}
}
- HfstTransducer restriction(HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
+ HfstTransducer restriction(HfstTransducerPairVector &contexts,
+ HfstTransducer &mapping,
StringPairSet &alphabet,
- TwolType twol_type,
- int direction )
- {
+ TwolType twol_type,
+ int direction )
+ {
// Make sure that contexts contains at least one transducer pair and that
// all transducers in the set have the same type.
ImplementationType type=ERROR_TYPE;
@@ -564,8 +564,8 @@ namespace hfst
if (! type_defined) {
type = it->first.get_type();
type_defined=true;
- }
- else {
+ }
+ else {
if (type != it->first.get_type()) {
HFST_THROW_MESSAGE(TransducerTypeMismatchException,
"rules::restriction");
@@ -602,7 +602,7 @@ namespace hfst
tmp.compose(mapping.output_project());
}
- // context transducer
+ // context transducer
// pi_star + left[i] + mt + tmp + mt + + right[i] + pi_star
HfstTransducer l2(type);
for (HfstTransducerPairVector::const_iterator it = contexts.begin();
@@ -611,7 +611,7 @@ namespace hfst
HfstTransducer ct(internal_epsilon, type);
ct.concatenate(pi_star);
ct.concatenate(it->first);
- ct.concatenate(mt);
+ ct.concatenate(mt);
ct.concatenate(tmp);
ct.concatenate(mt);
ct.concatenate(it->second);
@@ -659,66 +659,66 @@ namespace hfst
}
else {
assert(false);
- return HfstTransducer(type); // make compiler happy
+ return HfstTransducer(type); // make compiler happy
}
}
- HfstTransducer restriction(HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
+ HfstTransducer restriction(HfstTransducerPairVector &contexts,
+ HfstTransducer &mapping,
StringPairSet &alphabet) {
- return restriction(contexts, mapping, alphabet, twol_right, 0);
+ return restriction(contexts, mapping, alphabet, twol_right, 0);
}
- HfstTransducer coercion(HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
- StringPairSet &alphabet) {
- return restriction(contexts, mapping, alphabet, twol_left, 0);
+ HfstTransducer coercion(HfstTransducerPairVector &contexts,
+ HfstTransducer &mapping,
+ StringPairSet &alphabet) {
+ return restriction(contexts, mapping, alphabet, twol_left, 0);
}
HfstTransducer restriction_and_coercion(HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
+ HfstTransducer &mapping,
StringPairSet &alphabet) {
- return restriction(contexts, mapping, alphabet, twol_both, 0);
+ return restriction(contexts, mapping, alphabet, twol_both, 0);
}
HfstTransducer surface_restriction(HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
- StringPairSet &alphabet) {
- return restriction(contexts, mapping, alphabet, twol_right, 1);
+ HfstTransducer &mapping,
+ StringPairSet &alphabet) {
+ return restriction(contexts, mapping, alphabet, twol_right, 1);
}
- HfstTransducer surface_coercion(HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
- StringPairSet &alphabet) {
- return restriction(contexts, mapping, alphabet, twol_left, 1);
+ HfstTransducer surface_coercion(HfstTransducerPairVector &contexts,
+ HfstTransducer &mapping,
+ StringPairSet &alphabet) {
+ return restriction(contexts, mapping, alphabet, twol_left, 1);
}
HfstTransducer surface_restriction_and_coercion
- (HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
- StringPairSet &alphabet) {
- return restriction(contexts, mapping, alphabet, twol_both, 1);
+ (HfstTransducerPairVector &contexts,
+ HfstTransducer &mapping,
+ StringPairSet &alphabet) {
+ return restriction(contexts, mapping, alphabet, twol_both, 1);
}
HfstTransducer deep_restriction(HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
- StringPairSet &alphabet) {
- return restriction(contexts, mapping, alphabet, twol_right, -1);
+ HfstTransducer &mapping,
+ StringPairSet &alphabet) {
+ return restriction(contexts, mapping, alphabet, twol_right, -1);
}
- HfstTransducer deep_coercion(HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
- StringPairSet &alphabet) {
- return restriction(contexts, mapping, alphabet, twol_left, -1);
+ HfstTransducer deep_coercion(HfstTransducerPairVector &contexts,
+ HfstTransducer &mapping,
+ StringPairSet &alphabet) {
+ return restriction(contexts, mapping, alphabet, twol_left, -1);
}
HfstTransducer deep_restriction_and_coercion
- (HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
- StringPairSet &alphabet) {
- return restriction(contexts, mapping, alphabet, twol_both, -1);
+ (HfstTransducerPairVector &contexts,
+ HfstTransducer &mapping,
+ StringPairSet &alphabet) {
+ return restriction(contexts, mapping, alphabet, twol_both, -1);
}
}
diff --git a/libhfst/src/HfstStrings2FstTokenizer.cc b/libhfst/src/HfstStrings2FstTokenizer.cc
index c5010f3..ee24f95 100644
--- a/libhfst/src/HfstStrings2FstTokenizer.cc
+++ b/libhfst/src/HfstStrings2FstTokenizer.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "HfstStrings2FstTokenizer.h"
@@ -55,7 +55,7 @@ void HfstStrings2FstTokenizer::add_multichar_symbol_head
{
if (multichar_symbol.empty())
{ throw EmptyMulticharSymbol(); }
- StringVector tokenized_multichar_symbol =
+ StringVector tokenized_multichar_symbol =
tokenizer.tokenize_one_level(multichar_symbol);
std::string multichar_symbol_head(*tokenized_multichar_symbol.begin());
tokenizer.add_multichar_symbol
@@ -71,7 +71,7 @@ StringPairVector HfstStrings2FstTokenizer::tokenize_pair_string
else
{
tokenized_str = tokenizer.tokenize_one_level(str);
- StringVector::iterator new_end =
+ StringVector::iterator new_end =
std::remove(tokenized_str.begin(),tokenized_str.end(),BACKSLASH);
tokenized_str.erase(new_end,tokenized_str.end());
}
@@ -105,17 +105,17 @@ StringPairVector HfstStrings2FstTokenizer::make_pair_vector
++it)
{
if (! is_pair_input_symbol(it,v.end()))
- {
+ {
std::string symbol = unescape(*it);
- symbol = (symbol.empty() || symbol == eps ?
+ symbol = (symbol.empty() || symbol == eps ?
EPSILON_SYMBOL : symbol);
spv.push_back(StringPair(symbol,symbol)); }
else
{
- std::string input = (it->empty() || *it == eps ?
+ std::string input = (it->empty() || *it == eps ?
EPSILON_SYMBOL : unescape(*it));
++(++it);
- std::string output = (it->empty() || *it == eps ?
+ std::string output = (it->empty() || *it == eps ?
EPSILON_SYMBOL : unescape(*it));
spv.push_back(StringPair(input,output));
}
@@ -130,15 +130,15 @@ StringPairVector HfstStrings2FstTokenizer::make_pair_vector
StringVector::const_iterator input_it = input.begin();
StringVector::const_iterator output_it = output.begin();
while (input_it != input.end() && output_it != output.end())
- {
+ {
std::string input_symbol = unescape(*input_it);
std::string output_symbol = unescape(*output_it);
spv.push_back
- (StringPair(input_symbol.empty() || input_symbol == eps ?
+ (StringPair(input_symbol.empty() || input_symbol == eps ?
EPSILON_SYMBOL : input_symbol,
- output_symbol.empty() || output_symbol == eps ?
- EPSILON_SYMBOL : output_symbol));
+ output_symbol.empty() || output_symbol == eps ?
+ EPSILON_SYMBOL : output_symbol));
++input_it;
++output_it;
}
@@ -147,14 +147,14 @@ StringPairVector HfstStrings2FstTokenizer::make_pair_vector
for ( ; output_it != output.end(); ++output_it)
{ spv.push_back
(StringPair(EPSILON_SYMBOL,
- output_it->empty() || *output_it == eps ?
+ output_it->empty() || *output_it == eps ?
EPSILON_SYMBOL : unescape(*output_it))); }
}
else
{
for ( ; input_it != input.end(); ++input_it)
{ spv.push_back
- (StringPair(input_it->empty() || *input_it == eps ?
+ (StringPair(input_it->empty() || *input_it == eps ?
EPSILON_SYMBOL : unescape(*input_it),
EPSILON_SYMBOL)); }
}
@@ -219,7 +219,7 @@ void HfstStrings2FstTokenizer::check_cols(const std::string &symbol)
{ throw UnescapedColsFound(); }
size_t pos = 0;
while ((pos = symbol.find(COL_CHAR,pos+1)) != std::string::npos)
- {
+ {
if (symbol[pos-1] != BACKSLASH_CHAR)
{ throw UnescapedColsFound(); }
if (pos > 1 && symbol[pos-2] == BACKSLASH_CHAR)
@@ -237,7 +237,7 @@ int HfstStrings2FstTokenizer::get_col_pos(const std::string &str)
for (size_t i = 1; i < str.size(); ++i)
{
if (str[i] == COL_CHAR && str[i-1] != BACKSLASH_CHAR)
- { return i; }
+ { return (int)i; }
}
return -1;
}
@@ -250,7 +250,7 @@ StringVector HfstStrings2FstTokenizer::split_at_spaces(const std::string &str)
for (StringVector::const_iterator it = sv.begin(); it != sv.end(); ++it)
{
if (*it == SPACE && ! symbol.empty())
- {
+ {
res.push_back(symbol);
while (it + 1 != sv.end() && *(it + 1) == SPACE)
{ ++it; }
@@ -259,12 +259,12 @@ StringVector HfstStrings2FstTokenizer::split_at_spaces(const std::string &str)
{ break; }
}
else if (* it == SPACE)
- {
+ {
while (it + 1 != sv.end() && *(it + 1) == SPACE)
{ ++it; }
}
else if (*it == COL && ! symbol.empty())
- {
+ {
res.push_back(symbol);
res.push_back(COL);
symbol = EMPTY;
@@ -291,8 +291,8 @@ void test_ps
for (StringPairVector::const_iterator it = spv.begin();
it != spv.end();
++it)
- {
- if (it->first != it->second)
+ {
+ if (it->first != it->second)
{ std::cout << it->first << " : " << it->second << std::endl; }
else
{ std::cout << it->first << std::endl; }
@@ -309,8 +309,8 @@ void test_sp
for (StringPairVector::const_iterator it = spv.begin();
it != spv.end();
++it)
- {
- if (it->first != it->second)
+ {
+ if (it->first != it->second)
{ std::cout << it->first << " : " << it->second << std::endl; }
else
{ std::cout << it->first << std::endl; }
diff --git a/libhfst/src/HfstStrings2FstTokenizer.h b/libhfst/src/HfstStrings2FstTokenizer.h
index 655bab3..3153916 100644
--- a/libhfst/src/HfstStrings2FstTokenizer.h
+++ b/libhfst/src/HfstStrings2FstTokenizer.h
@@ -1,13 +1,13 @@
#ifndef HEADER_STRINGS_2_FST_TOKENIZER_H
#define HEADER_STRINGS_2_FST_TOKENIZER_H
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include <vector>
@@ -53,13 +53,13 @@ class HfstStrings2FstTokenizer
HfstStrings2FstTokenizer
(StringVector &multichar_symbols,const std::string &eps);
- //! Return a StringPairVector representation of the pair string str.
- //! The parameter spaces tells whether spaces have been used to separate
+ //! Return a StringPairVector representation of the pair string str.
+ //! The parameter spaces tells whether spaces have been used to separate
//! symbol pairs.
StringPairVector tokenize_pair_string(const std::string &str,bool spaces);
- //! Return a StringPairVector representation of the string pair str.
- //! The parameter spaces tells whether spaces have been used to separate
+ //! Return a StringPairVector representation of the string pair str.
+ //! The parameter spaces tells whether spaces have been used to separate
//! symbols.
StringPairVector tokenize_string_pair(const std::string &str,bool spaces);
@@ -98,7 +98,7 @@ class HfstStrings2FstTokenizer
StringVector split_at_spaces(const std::string &str);
//! Throw UnescapedColsFound, if symbols contains unescaped ':'-symbols.
- void check_cols(const std::string &symbol);
+ void check_cols(const std::string &symbol);
};
} // namespace hfst
diff --git a/libhfst/src/HfstSymbolDefs.cc b/libhfst/src/HfstSymbolDefs.cc
index 4892997..3a98eda 100644
--- a/libhfst/src/HfstSymbolDefs.cc
+++ b/libhfst/src/HfstSymbolDefs.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "HfstSymbolDefs.h"
@@ -155,7 +155,7 @@ bool is_default(const char * str)
for (HfstTwoLevelPaths::const_iterator it = paths.begin();
it != paths.end(); it++)
{
- unsigned int length = it->second.size();
+ unsigned int length = (unsigned int)it->second.size();
max_path_length = (length > max_path_length)? length : max_path_length;
}
return (int)max_path_length;
@@ -169,14 +169,14 @@ bool is_default(const char * str)
for (HfstTwoLevelPaths::const_iterator it = paths.begin();
it != paths.end(); it++)
{
- unsigned int length = it->second.size();
+ unsigned int length = (unsigned int)it->second.size();
max_path_length = (length > max_path_length)? length : max_path_length;
}
for (HfstTwoLevelPaths::const_iterator it = paths.begin();
it != paths.end(); it++)
{
- unsigned int length = it->second.size();
+ unsigned int length = (unsigned int)it->second.size();
if (length == max_path_length)
{
result.insert(*it);
diff --git a/libhfst/src/HfstSymbolDefs.h b/libhfst/src/HfstSymbolDefs.h
index cf387be..97a8b5a 100644
--- a/libhfst/src/HfstSymbolDefs.h
+++ b/libhfst/src/HfstSymbolDefs.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _SYMBOL_DEFS_H_
@@ -17,10 +17,10 @@
#include "hfstdll.h"
/** @file HfstSymbolDefs.h
- \brief Typedefs and functions for symbols, symbol pairs and
+ \brief Typedefs and functions for symbols, symbol pairs and
sets of symbols. */
-/* Macros that can be used instead of hfsT::internal_epsilon etc
+/* Macros that can be used instead of hfsT::internal_epsilon etc
if there are problems with variable initialization. */
#define INTERNAL_EPSILON "@_EPSILON_SYMBOL_@"
#define INTERNAL_UNKNOWN "@_UNKNOWN_SYMBOL_@"
@@ -31,27 +31,27 @@
namespace hfst
{
- /** \brief A UTF-8 symbol in a transition.
+ /** \brief A UTF-8 symbol in a transition.
- Strings <i>"@_EPSILON_SYMBOL_@"</i>, <i>"@_UNKNOWN_SYMBOL_@"</i>
+ Strings <i>"@_EPSILON_SYMBOL_@"</i>, <i>"@_UNKNOWN_SYMBOL_@"</i>
and <i>"@_IDENTITY_SYMBOL_@"</i> are reserved.
- - <i>"@_EPSILON_SYMBOL_@"</i> denotes the epsilon (in AT&T formalism
- <tt>[0]</tt>).
- - <i>"@_UNKNOWN_SYMBOL_@"</i> denotes an unknown symbol,
- i.e. any symbol that
- does not occur or has not occurred in the transducer
- (in AT&T formalism <tt>[?], [?:x], [x:?]</tt>).
+ - <i>"@_EPSILON_SYMBOL_@"</i> denotes the epsilon (in AT&T formalism
+ <tt>[0]</tt>).
+ - <i>"@_UNKNOWN_SYMBOL_@"</i> denotes an unknown symbol,
+ i.e. any symbol that
+ does not occur or has not occurred in the transducer
+ (in AT&T formalism <tt>[?], [?:x], [x:?]</tt>).
- <i>"@_IDENTITY_SYMBOL_@"</i> denotes any unknown symbol that is the same
- on the input and output side of a transition (in AT&T formalism
+ on the input and output side of a transition (in AT&T formalism
<tt>[?:?]</tt>).
For an example of special symbols, see HFST transducer
<a href="classhfst_1_1HfstTransducer.html#symbols">
documentation</a>.
- Strings of form <CODE> @[PNDRCU][.][A-Z]+([.][A-Z]+)?@ </CODE>
- are reserved for flag diacritics. For an example of flag diacritics, see
+ Strings of form <CODE> @[PNDRCU][.][A-Z]+([.][A-Z]+)?@ </CODE>
+ are reserved for flag diacritics. For an example of flag diacritics, see
#hfst::HfstTransducer::extract_paths_fd(hfst::HfstTwoLevelPaths&, int, int, bool) const.
For more information on flag diacritics, see Finite State Morphology
(Beesley & Karttunen, 2003).
@@ -61,34 +61,34 @@ namespace hfst
/* A set of strings. */
typedef std::set<String> StringSet;
- typedef std::vector<String> StringVector;
+ typedef std::vector<String> StringVector;
- /** \brief A symbol pair in a transition.
+ /** \brief A symbol pair in a transition.
- @see HfstTransducer::substitute(const StringPair&, const StringPair&)
+ @see HfstTransducer::substitute(const StringPair&, const StringPair&)
and other substitute functions */
typedef std::pair<String, String> StringPair;
- /** \brief A vector of transitions that represents a path in a transducer.
+ /** \brief A vector of transitions that represents a path in a transducer.
@see HfstTokenizer */
typedef std::vector<StringPair> StringPairVector;
- /** \brief A set of symbol pairs used in substituting symbol pairs
- and in rule functions.
+ /** \brief A set of symbol pairs used in substituting symbol pairs
+ and in rule functions.
@see HfstTransducer::substitute(const StringPair&, const StringPairSet &)
#hfst::rules */
typedef std::set<StringPair> StringPairSet;
/** \brief A map of substitutions used when performing multiple
- symbol-to-symbol substitutions.
+ symbol-to-symbol substitutions.
@see HfstTransducer::substitute(const HfstSymbolSubstitutions&) */
typedef std::map<String, String> HfstSymbolSubstitutions;
/** \brief A map of substitutions used when performing multiple
- symbol pair-to-symbol pair substitutions.
+ symbol pair-to-symbol pair substitutions.
@see HfstTransducer::substitute(const HfstSymbolPairSubstitutions&) */
typedef std::map<StringPair, StringPair> HfstSymbolPairSubstitutions;
@@ -97,10 +97,10 @@ namespace hfst
typedef std::set<HfstTwoLevelPath> HfstTwoLevelPaths;
/* The internal representations */
- const std::string internal_epsilon = "@_EPSILON_SYMBOL_@";
- const std::string internal_unknown = "@_UNKNOWN_SYMBOL_@";
- const std::string internal_identity = "@_IDENTITY_SYMBOL_@";
- const std::string internal_default = "@_DEFAULT_SYMBOL_@";
+ const std::string internal_epsilon("@_EPSILON_SYMBOL_@");
+ const std::string internal_unknown("@_UNKNOWN_SYMBOL_@");
+ const std::string internal_identity("@_IDENTITY_SYMBOL_@");
+ const std::string internal_default("@_DEFAULT_SYMBOL_@");
/* Check whether a string is equal to reserved internal representation. */
HFSTDLL bool is_epsilon(std::string const & str);
diff --git a/libhfst/src/HfstTokenizer.cc b/libhfst/src/HfstTokenizer.cc
index 9be665f..6321b6b 100644
--- a/libhfst/src/HfstTokenizer.cc
+++ b/libhfst/src/HfstTokenizer.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "HfstTokenizer.h"
@@ -15,7 +15,7 @@
#ifndef MAIN_TEST
using std::string;
-namespace hfst
+namespace hfst
{
bool MultiCharSymbolTrie::is_end_of_string(const char * p) const
{ return *(p+1) == 0; }
@@ -27,7 +27,7 @@ namespace hfst
{ return is_leaf[(unsigned char)(*p)]; }
void MultiCharSymbolTrie::init_symbol_rests(const char * p)
- {
+ {
if (symbol_rests[(unsigned char)(*p)] == NULL)
{ symbol_rests[(unsigned char)(*p)] = new MultiCharSymbolTrie(); }
}
@@ -65,14 +65,14 @@ namespace hfst
{
MultiCharSymbolTrie * symbol_rest_trie = get_symbol_rest_trie(p);
if (symbol_rest_trie == NULL)
- {
+ {
if (is_symbol_end(p))
{ return p+1; }
- return NULL;
+ return NULL;
}
const char * symbol_end = symbol_rest_trie->find(p+1);
if (symbol_end == NULL)
- {
+ {
if (is_symbol_end(p))
{ return p+1; }
}
@@ -88,11 +88,11 @@ const
if (! *symbol)
{ return 0; }
- const char * multi_char_symbol_end = multi_char_symbols.find(symbol);
+ const char * multi_char_symbol_end = multi_char_symbols.find(symbol);
/* The string begins with a multi character symbol */
if (multi_char_symbol_end != NULL)
- { return multi_char_symbol_end - symbol; }
+ { return (int)(multi_char_symbol_end - symbol); }
if ((128 & *symbol) == 0)
{ return 1; }
else if ((32 & *symbol) == 0)
@@ -118,7 +118,7 @@ void
HfstTokenizer::add_skip_symbol(const std::string &symbol)
{ if (symbol == "")
{ return; }
- multi_char_symbols.add(symbol.c_str());
+ multi_char_symbols.add(symbol.c_str());
skip_symbol_set.insert(symbol.c_str()); }
StringPairVector HfstTokenizer::tokenize
@@ -181,7 +181,7 @@ StringPairVector HfstTokenizer::tokenize_space_separated(const std::string & str
{
symbol_pos = pos;
}
- else
+ else
{}
++pos;
}
@@ -252,7 +252,7 @@ StringPairVector HfstTokenizer::tokenize
for (StringPairVector::iterator it = input_spv.begin();
it != input_spv.end();
++it)
- {
+ {
StringPair sp(it->first, jt->first);
warn_about_pair(sp);
spv.push_back(sp);
@@ -302,7 +302,7 @@ StringPairVector HfstTokenizer::tokenize_and_align_flag_diacritics
StringPair sp("", ""); // string pair to push back to the result
StringPair sp_cont("", ""); // possible continuation in case of missaligned flags
- if (it == input_spv.end())
+ if (it == input_spv.end())
{
if (FdOperation::is_diacritic(jt->first)) // copy diacritic to other side
{
@@ -314,7 +314,7 @@ StringPairVector HfstTokenizer::tokenize_and_align_flag_diacritics
}
jt++;
}
- else if (jt == output_spv.end())
+ else if (jt == output_spv.end())
{
if (FdOperation::is_diacritic(it->first)) // copy diacritic to other side
{
@@ -329,7 +329,7 @@ StringPairVector HfstTokenizer::tokenize_and_align_flag_diacritics
else
{
// take from both vectors (cases foo:bar, foo:foo, flag1:flag1)
- if ((!FdOperation::is_diacritic(it->first) && !FdOperation::is_diacritic(jt->first)) ||
+ if ((!FdOperation::is_diacritic(it->first) && !FdOperation::is_diacritic(jt->first)) ||
*it == *jt)
{
sp = StringPair(it->first, jt->first);
@@ -347,22 +347,29 @@ StringPairVector HfstTokenizer::tokenize_and_align_flag_diacritics
jt++;
}
- spv.push_back(sp);
+ spv.push_back(sp);
if (sp_cont.first.size() != 0 && sp_cont.second.size() != 0)
{
spv.push_back(sp_cont);
- }
+ }
}
return spv;
}
+
+void
+HfstTokenizer::check_utf8_correctness(const std::string &input_string)
+{
+ (void)check_utf8_correctness_and_calculate_length(input_string);
+}
- void
- HfstTokenizer::check_utf8_correctness(const std::string &input_string)
+ unsigned int
+ HfstTokenizer::check_utf8_correctness_and_calculate_length(const std::string &input_string)
{
+ unsigned int retval = 0;
// Check that input_string is made up from utf-8 sequences.
- for (std::string::const_iterator it = input_string.begin();
+ for (std::string::const_iterator it = input_string.begin();
it != input_string.end();
++it)
{
@@ -373,14 +380,14 @@ StringPairVector HfstTokenizer::tokenize_and_align_flag_diacritics
// The bytes 192, 193, 245, 246 and 247 are invalid in utf8.
if (initial_char == 192 || initial_char == 193 ||
initial_char == 245 || initial_char == 246 || initial_char == 247)
- { HFST_THROW_MESSAGE(IncorrectUtf8CodingException,
+ { HFST_THROW_MESSAGE(IncorrectUtf8CodingException,
"leading octet in [192, 193, 245, 246, 247]"); }
// Case 0xxxxxxx, i.e. ASCII byte.
else if ((128 & initial_char) == 0)
{ additional_chars = 0; }
- // Case 10xxxxxx cannot be an initial byte.
+ // Case 10xxxxxx cannot be an initial byte.
else if ((64 & initial_char) == 0)
- { HFST_THROW_MESSAGE(IncorrectUtf8CodingException,
+ { HFST_THROW_MESSAGE(IncorrectUtf8CodingException,
"leading octet & 10000000b"); }
// Case 110xxxxx, i.e. read one more byte.
else if ((32 & initial_char) == 0)
@@ -389,11 +396,11 @@ StringPairVector HfstTokenizer::tokenize_and_align_flag_diacritics
else if ((16 & initial_char) == 0)
{ additional_chars = 2; }
// Case 11110xxx, i.e. read three more bytes.
- else if ((8 & initial_char) == 0)
+ else if ((8 & initial_char) == 0)
{ additional_chars = 3; }
// Case 11111xxx is not allowed in utf8.
else
- { HFST_THROW_MESSAGE(IncorrectUtf8CodingException,
+ { HFST_THROW_MESSAGE(IncorrectUtf8CodingException,
"leading octet & 11111000b"); }
// Read the continuation bytes.
@@ -410,7 +417,9 @@ StringPairVector HfstTokenizer::tokenize_and_align_flag_diacritics
{ HFST_THROW_MESSAGE(IncorrectUtf8CodingException,
"not continuation octet & 100000000b"); }
}
+ retval = retval + 1;
}
+ return retval;
}
}
diff --git a/libhfst/src/HfstTokenizer.h b/libhfst/src/HfstTokenizer.h
index 605f300..76cf927 100644
--- a/libhfst/src/HfstTokenizer.h
+++ b/libhfst/src/HfstTokenizer.h
@@ -11,7 +11,7 @@
#define _HFST_TOKENIZER_H_
#include "HfstSymbolDefs.h"
#include "HfstExceptionDefs.h"
-#include <iostream>
+#include <iosfwd>
#include <climits>
#include <string>
@@ -52,16 +52,16 @@ namespace hfst
HFSTDLL MultiCharSymbolTrie(void);
HFSTDLL ~MultiCharSymbolTrie(void);
HFSTDLL void add(const char * p);
- HFSTDLL const char * find(const char * p) const;
+ HFSTDLL const char * find(const char * p) const;
};
/** \brief A tokenizer for creating transducers from UTF-8 strings.
Strings are tokenized from left to right using longest match tokenization.
- For example, if the tokenizer contains a multicharacter symbol
+ For example, if the tokenizer contains a multicharacter symbol
"foo" and a skip symbol "fo",
the string "foo" is tokenized as "foo:foo".
- If the tokenizer contains a multicharacter symbol "fo" and a skip
+ If the tokenizer contains a multicharacter symbol "fo" and a skip
symbol "foo",
the string "foo" is tokenized as an empty string.
@@ -76,13 +76,13 @@ namespace hfst
// A:A <br />:<br /> p:p a:a r:r a:a g:g r:r a:a p:p h:h !:!
\endverbatim
- @note The tokenizer only tokenizes utf-8 strings.
- Special symbols (see #String) are not included in the tokenizer
+ @note The tokenizer only tokenizes utf-8 strings.
+ Special symbols (see #String) are not included in the tokenizer
unless added to it.
@see hfst::HfstTransducer::HfstTransducer(const std::string&, const HfstTokenizer&, ImplementationType type) */
class HfstTokenizer
- {
+ {
private:
MultiCharSymbolTrie multi_char_symbols;
StringSet skip_symbol_set;
@@ -94,18 +94,18 @@ namespace hfst
/** \brief Create a tokenizer that recognizes utf-8 symbols. */
HFSTDLL HfstTokenizer();
- /** \brief Add a symbol to be skipped to this tokenizer.
+ /** \brief Add a symbol to be skipped to this tokenizer.
After skipping a symbol, tokenization is always started again.
- For example if we have a multicharacter symbol "foo" and a
- skip symbol "bar", the string "fobaro" will be tokenized
+ For example if we have a multicharacter symbol "foo" and a
+ skip symbol "bar", the string "fobaro" will be tokenized
"f" "o" "o", not "foo". */
HFSTDLL void add_skip_symbol(const std::string &symbol);
- /** \brief Add a multicharacter symbol \a symbol to this tokenizer.
+ /** \brief Add a multicharacter symbol \a symbol to this tokenizer.
If a multicharacter symbol has a skip symbol inside it, it is
- not considered a multicharacter symbol. For example if we have
+ not considered a multicharacter symbol. For example if we have
a multicharacter symbol "foo" and a skip symbol "bar", the string
"fobaro" will be tokenized "f" "o" "o", not "foo". */
HFSTDLL void add_multichar_symbol(const std::string& symbol);
@@ -118,7 +118,7 @@ namespace hfst
HFSTDLL static StringPairVector tokenize_space_separated(const std::string & str);
- /** \brief Tokenize the string pair \a input_string : \a output_string.
+ /** \brief Tokenize the string pair \a input_string : \a output_string.
If one string has more tokens than the other, epsilons will be
inserted to the end of the tokenized string with less tokens
@@ -139,21 +139,23 @@ namespace hfst
//! \brief If @a input_String is not valid utf-8, throw an
//! @a IncorrectUtf8CodingException.
//!
- //! A string is non-valid if:
+ //! A string is non-valid if:
//! - It contains one of the unsigned bytes 192, 193, 245, 246 and 247.
- //! - If it is not made up of sequences of one initial byte (0xxxxxxx,
- //! 110xxxxx, 1110xxxx or 11110xxx) followed by an appropriate number
+ //! - If it is not made up of sequences of one initial byte (0xxxxxxx,
+ //! 110xxxxx, 1110xxxx or 11110xxx) followed by an appropriate number
//! of continuation bytes (10xxxxxx).
//! -# Initial bytes 0xxxxxxx represent ASCII chars and may not be
//! followed by a continuation byte.
- //! -# Initial bytes 110xxxxx are followed by exactly one
+ //! -# Initial bytes 110xxxxx are followed by exactly one
//! continuation byte.
//! -# Initial bytes 1110xxxx are followed by exactly two continuation
//! bytes.
- //! -# Initial bytes 11110xxx are followed by exactly three
+ //! -# Initial bytes 11110xxx are followed by exactly three
//! continuation bytes.
//! (For reference: http://en.wikipedia.org/wiki/UTF-8)
HFSTDLL static void check_utf8_correctness(const std::string &input_string);
+
+ HFSTDLL static unsigned int check_utf8_correctness_and_calculate_length(const std::string &input_string);
};
}
#endif
diff --git a/libhfst/src/HfstTransducer.cc b/libhfst/src/HfstTransducer.cc
index 7e9a811..b86a146 100644
--- a/libhfst/src/HfstTransducer.cc
+++ b/libhfst/src/HfstTransducer.cc
@@ -1,20 +1,21 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
/* @file HfstTransducer.cc
- \brief Implementations of functions declared in file HfstTransducer.h
+ \brief Implementations of functions declared in file HfstTransducer.h
The implementations call backend implementations that are declared in
files in the directory implementations. */
#include <string>
#include <map>
+#include <cassert>
using std::string;
using std::map;
@@ -26,45 +27,44 @@ using std::map;
using hfst::implementations::ConversionFunctions;
-
#ifndef MAIN_TEST
namespace hfst
{
-// -----------------------------------------------------------------------
-//
-// Interfaces through which the backend implementations are called
-//
-// -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
+ //
+ // Interfaces through which the backend implementations are called
+ //
+ // -----------------------------------------------------------------------
#if HAVE_SFST
-hfst::implementations::SfstTransducer HfstTransducer::sfst_interface;
+ hfst::implementations::SfstTransducer HfstTransducer::sfst_interface;
#endif
#if HAVE_OPENFST
-hfst::implementations::TropicalWeightTransducer
+ hfst::implementations::TropicalWeightTransducer
HfstTransducer::tropical_ofst_interface;
#if HAVE_OPENFST_LOG
-hfst::implementations::LogWeightTransducer
+ hfst::implementations::LogWeightTransducer
HfstTransducer::log_ofst_interface;
#endif
#endif
#if HAVE_FOMA
-hfst::implementations::FomaTransducer HfstTransducer::foma_interface;
+ hfst::implementations::FomaTransducer HfstTransducer::foma_interface;
#endif
hfst::implementations::HfstOlTransducer HfstTransducer::hfst_ol_interface;
-/* Add here the interface between HFST and your transducer library. */
-//#if HAVE_MY_TRANSDUCER_LIBRARY
-//hfst::implementations::MyTransducerLibraryTransducer
-// HfstTransducer::my_transducer_library_interface;
-//#endif
+ /* Add here the interface between HFST and your transducer library. */
+ //#if HAVE_MY_TRANSDUCER_LIBRARY
+ //hfst::implementations::MyTransducerLibraryTransducer
+ // HfstTransducer::my_transducer_library_interface;
+ //#endif
-// -----------------------------------------------------------------------
-//
-// Testing and optimization functions
-//
-// -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
+ //
+ // Testing and optimization functions
+ //
+ // -----------------------------------------------------------------------
#if HAVE_XFSM
void initialize_xfsm()
@@ -80,22 +80,25 @@ hfst::implementations::FomaTransducer HfstTransducer::foma_interface;
InitializeXfsm dummy;
#endif
-/* The default minimization algorithm if Hopcroft. */
-MinimizationAlgorithm minimization_algorithm=HOPCROFT;
- /* By default, we do not minimize transducers that are already minimal.
- This variable is for debugging and profiling. */
-bool minimize_even_if_already_minimal=false;
-/* By default, weights are not encoded in minimization. */
-bool encode_weights=false;
-/* By default, harmonization is not optimized. */
-bool harmonize_smaller=true;
-/* By default, unknown symbols are used. */
-bool unknown_symbols_in_use=true;
-
-/* Xerox-style composition where flag diacritics match unknowns and identities. */
-bool xerox_composition=false;
-/* Xerox option where flag diacritic are treated as epsilons in composition. */
-bool flag_is_epsilon_in_composition=false;
+ /* The default minimization algorithm if Hopcroft. */
+ MinimizationAlgorithm minimization_algorithm=HOPCROFT;
+ /* By default, we do not minimize transducers that are already minimal.
+ This variable is for debugging and profiling. */
+ bool minimize_even_if_already_minimal=false;
+ /* By default, weights are not encoded in minimization. */
+ bool encode_weights=false;
+ /* Allow minimization of intermediary results, used in some more complex functions.
+ A false value indicates that only epsilon removal and determinization is allowed. */
+ bool can_minimize=true;
+ /* By default, harmonization is optimized. */
+ bool harmonize_smaller=true;
+ /* By default, unknown symbols are used. */
+ bool unknown_symbols_in_use=true;
+
+ /* Xerox-style composition where flag diacritics match unknowns and identities. */
+ bool xerox_composition=false;
+ /* Xerox option where flag diacritic are treated as epsilons in composition. */
+ bool flag_is_epsilon_in_composition=false;
void set_xerox_composition(bool value) {
xerox_composition=value;
@@ -105,6 +108,14 @@ bool flag_is_epsilon_in_composition=false;
return xerox_composition;
}
+ void set_minimization(bool value) {
+ can_minimize=value;
+ }
+
+ bool get_minimization() {
+ return can_minimize;
+ }
+
void set_minimize_even_if_already_minimal(bool value) {
minimize_even_if_already_minimal=value;
#if HAVE_XFSM
@@ -142,16 +153,24 @@ void set_encode_weights(bool value) {
void set_warning_stream(std::ostream * os)
{
+#if HAVE_OPENFST
hfst::implementations::TropicalWeightTransducer::set_warning_stream(os);
+#else
+ (void)os;
+#endif
}
std::ostream * get_warning_stream()
{
+#if HAVE_OPENFST
return hfst::implementations::TropicalWeightTransducer::get_warning_stream();
+#else
+ HFST_THROW_MESSAGE(FunctionNotImplementedException, "get_warning_stream");
+#endif
}
void set_minimization_algorithm(MinimizationAlgorithm a) {
- minimization_algorithm=a;
+ minimization_algorithm=a;
#if HAVE_SFST
if (minimization_algorithm == HOPCROFT)
hfst::implementations::sfst_set_hopcroft(true);
@@ -170,7 +189,7 @@ void set_minimization_algorithm(MinimizationAlgorithm a) {
hfst::implementations::openfst_log_set_hopcroft(false);
#endif
#endif
- // in foma, Hopcroft is always used
+ // in foma, Hopcroft is always used
}
MinimizationAlgorithm get_minimization_algorithm() {
@@ -198,7 +217,7 @@ float HfstTransducer::get_profile_seconds(ImplementationType type)
// -----------------------------------------------------------------------
//
-// Alphabet and harmonization
+// Alphabet and harmonization
//
// -----------------------------------------------------------------------
@@ -213,7 +232,7 @@ StringPairSet HfstTransducer::get_symbol_pairs()
HFST_THROW_MESSAGE(FunctionNotImplementedException, "get_symbol_pairs");
}
-void HfstTransducer::insert_to_alphabet(const std::string &symbol)
+void HfstTransducer::insert_to_alphabet(const std::string &symbol)
{
HfstTokenizer::check_utf8_correctness(symbol);
@@ -228,7 +247,7 @@ void HfstTransducer::insert_to_alphabet(const std::string &symbol)
#endif
if (this->type != XFSM_TYPE)
{
- hfst::implementations::HfstBasicTransducer * net
+ hfst::implementations::HfstBasicTransducer * net
= convert_to_basic_transducer();
net->add_symbol_to_alphabet(symbol);
convert_to_hfst_transducer(net);
@@ -236,14 +255,14 @@ void HfstTransducer::insert_to_alphabet(const std::string &symbol)
#if HAVE_XFSM
this->xfsm_interface.add_symbol_to_alphabet(this->implementation.xfsm, symbol);
#else
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, XFSM_TYPE);
#endif
}
}
-void HfstTransducer::insert_to_alphabet(const std::set<std::string> &symbols)
+void HfstTransducer::insert_to_alphabet(const std::set<std::string> &symbols)
{
- for (std::set<std::string>::const_iterator it = symbols.begin();
+ for (std::set<std::string>::const_iterator it = symbols.begin();
it != symbols.end(); it++)
{
HfstTokenizer::check_utf8_correctness(*it);
@@ -253,7 +272,7 @@ void HfstTransducer::insert_to_alphabet(const std::set<std::string> &symbols)
if (this->type != XFSM_TYPE)
{
- hfst::implementations::HfstBasicTransducer * net
+ hfst::implementations::HfstBasicTransducer * net
= convert_to_basic_transducer();
net->add_symbols_to_alphabet(symbols);
convert_to_hfst_transducer(net);
@@ -263,14 +282,14 @@ void HfstTransducer::insert_to_alphabet(const std::set<std::string> &symbols)
#if HAVE_XFSM
this->xfsm_interface.add_symbols_to_alphabet(this->implementation.xfsm, symbols);
#else
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, XFSM_TYPE);
#endif
}
}
-void HfstTransducer::remove_from_alphabet(const std::string &symbol)
+void HfstTransducer::remove_from_alphabet(const std::string &symbol)
{
HfstTokenizer::check_utf8_correctness(symbol);
@@ -278,14 +297,14 @@ void HfstTransducer::remove_from_alphabet(const std::string &symbol)
if (symbol == "")
HFST_THROW_MESSAGE(EmptyStringException, "remove_from_alphabet");
- hfst::implementations::HfstBasicTransducer * net
+ hfst::implementations::HfstBasicTransducer * net
= convert_to_basic_transducer();
net->remove_symbol_from_alphabet(symbol);
convert_to_hfst_transducer(net);
}
-void HfstTransducer::remove_from_alphabet(const std::set<std::string> &symbols)
+void HfstTransducer::remove_from_alphabet(const std::set<std::string> &symbols)
{
for (std::set<std::string>::const_iterator it = symbols.begin();
it != symbols.end(); it++)
@@ -302,12 +321,13 @@ void HfstTransducer::remove_symbols_from_alphabet(const StringSet & symbols)
#if HAVE_XFSM
this->xfsm_interface.remove_symbols_from_alphabet(this->implementation.xfsm, symbols);
#endif
+ (void)symbols;
}
HfstTransducer &HfstTransducer::prune_alphabet(bool force)
{
- hfst::implementations::HfstBasicTransducer * net
+ hfst::implementations::HfstBasicTransducer * net
= convert_to_basic_transducer();
net->prune_alphabet(force);
return convert_to_hfst_transducer(net);;
@@ -343,7 +363,7 @@ StringSet HfstTransducer::get_first_input_symbols() const
HFST_THROW_MESSAGE(FunctionNotImplementedException, "get_first_input_symbols");
default:
HFST_THROW_MESSAGE(FunctionNotImplementedException, "get_first_input_symbols");
- }
+ }
}
StringSet HfstTransducer::get_alphabet() const
@@ -378,44 +398,7 @@ StringSet HfstTransducer::get_alphabet() const
return hfst_ol_interface.get_alphabet(implementation.hfst_ol);
default:
HFST_THROW_MESSAGE(FunctionNotImplementedException, "get_alphabet");
- }
-}
-
-unsigned int HfstTransducer::get_symbol_number(const std::string &symbol)
-{
- switch(type)
- {
-#if HAVE_SFST
- case SFST_TYPE:
- return sfst_interface.get_symbol_number(implementation.sfst, symbol);
-#endif
-#if HAVE_OPENFST
- case TROPICAL_OPENFST_TYPE:
- return tropical_ofst_interface.get_symbol_number
- (implementation.tropical_ofst, symbol);
-#if HAVE_OPENFST_LOG
- case LOG_OPENFST_TYPE:
- return log_ofst_interface.get_symbol_number(implementation.log_ofst,
- symbol);
-#endif
-#endif
-#if HAVE_FOMA
- case FOMA_TYPE:
- return foma_interface.get_symbol_number(implementation.foma,
- symbol);
-#endif
-#if HAVE_XFSM
- case XFSM_TYPE:
- HFST_THROW_MESSAGE(FunctionNotImplementedException, "get_symbol_number");
-#endif
- case ERROR_TYPE:
- HFST_THROW(TransducerHasWrongTypeException);
- case HFST_OL_TYPE:
- case HFST_OLW_TYPE:
- default:
- HFST_THROW_MESSAGE(FunctionNotImplementedException,
- "get_symbol_number");
- }
+ }
}
/*
@@ -424,32 +407,15 @@ unsigned int HfstTransducer::get_symbol_number(const std::string &symbol)
*/
HfstTransducer * HfstTransducer::harmonize_symbol_encodings(const HfstTransducer &another)
{
- /*
- std::cerr << "== 1 ==" << std::endl;
- HfstBasicTransducer * another_basic = another.get_basic_transducer();
- std::cerr << "== 2 ==" << std::endl;
- HfstBasicTransducer * this_basic = this->convert_to_basic_transducer(); // OpenFst fails
-
- std::cerr << "== 3 ==" << std::endl;
- this->convert_to_hfst_transducer(this_basic);
- std::cerr << "== 4 ==" << std::endl;
- HfstTransducer * another_harmonized
- = new HfstTransducer(*another_basic, this->type);
- std::cerr << "== 5 ==" << std::endl;
- delete another_basic;
- std::cerr << "== 6 ==" << std::endl;
-
- return another_harmonized; */
-
HfstBasicTransducer another_basic(another);
HfstBasicTransducer this_basic(*this);
*this = HfstTransducer(this_basic, this->get_type());
return new HfstTransducer(another_basic, another.get_type());
}
-/*
+/*
Harmonize this transducer with a copy of another.
- another is not modifed, but a modified copy of it is returned.
+ another is not modifed, but a modified copy of it is returned.
Flag diacritics from the alphabet of this transducer are inserted
to the alphabet of the copy of another, so that they are excluded
from harmonization.
@@ -528,7 +494,7 @@ HfstTransducer * HfstTransducer::harmonize_(const HfstTransducer &another)
this_basic->harmonize(*another_basic);
this->convert_to_hfst_transducer(this_basic);
- HfstTransducer * another_harmonized
+ HfstTransducer * another_harmonized
= new HfstTransducer(*another_basic, this->type);
delete another_basic;
@@ -540,16 +506,16 @@ HfstTransducer * HfstTransducer::harmonize_(const HfstTransducer &another)
default:
HFST_THROW(TransducerHasWrongTypeException);
}
-
+ return NULL; // make compiler happy
}
-/* Harmonize symbol-to-number encodings and expand unknown and
- identity symbols.
+/* Harmonize symbol-to-number encodings and expand unknown and
+ identity symbols.
In the case of foma transducers, does nothing because foma's own functions
- take care of harmonizing. If harmonization is needed,
+ take care of harmonizing. If harmonization is needed,
FomaTransducer::harmonize can be used instead. */
-void HfstTransducer::harmonize(HfstTransducer &another)
+ void HfstTransducer::harmonize(HfstTransducer &another, bool force/*=false*/)
{
using namespace implementations;
if (this->type != another.type) {
@@ -588,7 +554,8 @@ void HfstTransducer::harmonize(HfstTransducer &another)
#if HAVE_FOMA
case (FOMA_TYPE):
// no need to harmonize as foma's functions take care of harmonizing
- break;
+ if (!force)
+ return;
#endif // HAVE_FOMA
#if HAVE_XFSM
case (XFSM_TYPE):
@@ -603,7 +570,7 @@ void HfstTransducer::harmonize(HfstTransducer &another)
#endif
{
HfstBasicTransducer * this_basic = this->convert_to_basic_transducer();
- HfstBasicTransducer * another_basic =
+ HfstBasicTransducer * another_basic =
another.convert_to_basic_transducer();
this_basic->harmonize(*another_basic);
@@ -617,12 +584,6 @@ void HfstTransducer::harmonize(HfstTransducer &another)
default:
HFST_THROW(TransducerHasWrongTypeException);
}
-
- if (false)
- {
- this->write_in_att_format(stderr);
- another.write_in_att_format(stderr);
- }
}
// test function
@@ -658,7 +619,7 @@ void HfstTransducer::print_alphabet()
// -----------------------------------------------------------------------
//
-// Lookup functions
+// Lookup functions
//
// -----------------------------------------------------------------------
@@ -674,6 +635,18 @@ HfstOneLevelPaths * HfstTransducer::lookup(const std::string & s,
return lookup_fd(s, limit, time_cutoff);
}
+HfstTwoLevelPaths * HfstTransducer::lookup_pairs(const std::string & s,
+ ssize_t limit, double time_cutoff) const
+{
+ switch(this->type) {
+ case (HFST_OL_TYPE):
+ case (HFST_OLW_TYPE):
+ return this->implementation.hfst_ol->lookup_fd_pairs(s, limit, time_cutoff);
+ default:
+ HFST_THROW(FunctionNotImplementedException);
+ }
+}
+
HfstOneLevelPaths * HfstTransducer::lookup_fd(const StringVector& s,
ssize_t limit, double time_cutoff) const
{
@@ -687,18 +660,6 @@ HfstOneLevelPaths * HfstTransducer::lookup_fd(const StringVector& s,
HFST_THROW(TransducerHasWrongTypeException);
default:
HFST_THROW(FunctionNotImplementedException);
-
- /*
- hfst::implementations::HfstBasicTransducer net(*this);
- HfstTransducer * tmp;
- if (this->type == TROPICAL_OPENFST_TYPE) {
- tmp = new HfstTransducer(net, HFST_OLW_TYPE); }
- else {
- tmp = new HfstTransducer(net, HFST_OL_TYPE); }
- HfstOneLevelPaths * retval = tmp->lookup_fd(s, limit);
- delete tmp;
- return retval;
- */
}
}
@@ -716,22 +677,11 @@ HfstOneLevelPaths * HfstTransducer::lookup_fd(const std::string & s,
default:
HFST_THROW(FunctionNotImplementedException);
- /*
- hfst::implementations::HfstBasicTransducer net(*this);
- HfstTransducer * tmp;
- if (this->type == TROPICAL_OPENFST_TYPE) {
- tmp = new HfstTransducer(net, HFST_OLW_TYPE); }
- else {
- tmp = new HfstTransducer(net, HFST_OL_TYPE); }
- HfstOneLevelPaths * retval = tmp->lookup_fd(s, limit);
- delete tmp;
- return retval;
- */
}
}
HfstOneLevelPaths * HfstTransducer::lookup(const HfstTokenizer& tok,
- const std::string &s,
+ const std::string &s,
ssize_t limit, double time_cutoff) const
{
StringVector sv = tok.tokenize_one_level(s);
@@ -777,7 +727,7 @@ bool HfstTransducer::is_lookup_infinitely_ambiguous(const StringVector& s)
is_lookup_infinitely_ambiguous(s);
default:
(void)s;
- HFST_THROW(FunctionNotImplementedException);
+ HFST_THROW(FunctionNotImplementedException);
}
}
@@ -791,7 +741,7 @@ bool HfstTransducer::is_lookup_infinitely_ambiguous(const std::string & s)
is_lookup_infinitely_ambiguous(s);
default:
(void)s;
- HFST_THROW(FunctionNotImplementedException);
+ HFST_THROW(FunctionNotImplementedException);
}
}
@@ -820,7 +770,7 @@ bool HfstTransducer::is_infinitely_ambiguous()
// -----------------------------------------------------------------------
//
-// Transducer constructors and destructor
+// Transducer constructors and destructor
//
// -----------------------------------------------------------------------
@@ -833,7 +783,7 @@ HfstTransducer::HfstTransducer(ImplementationType type):
type(type),anonymous(false),is_trie(true), name("")
{
if (! is_implementation_type_available(type))
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
switch (type)
{
@@ -844,13 +794,13 @@ HfstTransducer::HfstTransducer(ImplementationType type):
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
- implementation.tropical_ofst =
+ implementation.tropical_ofst =
tropical_ofst_interface.create_empty_transducer();
this->type = TROPICAL_OPENFST_TYPE;
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
- implementation.log_ofst =
+ implementation.log_ofst =
log_ofst_interface.create_empty_transducer();
break;
#endif
@@ -868,7 +818,7 @@ HfstTransducer::HfstTransducer(ImplementationType type):
/* Add here your implementation. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//case MY_TRANSDUCER_LIBRARY_TYPE:
- //implementation.my_transducer_library
+ //implementation.my_transducer_library
// = my_transducer_library_interface.create_empty_transducer();
//break;
//#endif
@@ -885,22 +835,22 @@ HfstTransducer::HfstTransducer(ImplementationType type):
}
-HfstTransducer::HfstTransducer(const std::string& utf8_str,
- const HfstTokenizer
+HfstTransducer::HfstTransducer(const std::string& utf8_str,
+ const HfstTokenizer
&multichar_symbol_tokenizer,
ImplementationType type):
type(type),anonymous(false),is_trie(true), name("")
{
if (! is_implementation_type_available(type))
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
if (utf8_str == "")
HFST_THROW_MESSAGE
- (EmptyStringException,
+ (EmptyStringException,
"HfstTransducer(const std::string&, const HfstTokenizer&,"
" ImplementationType)");
- StringPairVector spv =
+ StringPairVector spv =
multichar_symbol_tokenizer.tokenize(utf8_str);
switch (type)
{
@@ -911,13 +861,13 @@ HfstTransducer::HfstTransducer(const std::string& utf8_str,
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
- implementation.tropical_ofst =
+ implementation.tropical_ofst =
tropical_ofst_interface.define_transducer(spv);
this->type = TROPICAL_OPENFST_TYPE;
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
- implementation.log_ofst =
+ implementation.log_ofst =
log_ofst_interface.define_transducer(spv);
break;
#endif
@@ -953,19 +903,19 @@ HfstTransducer::HfstTransducer(const StringVector & sv,
*this = HfstTransducer(spv, type);
}
-HfstTransducer::HfstTransducer(const StringPairVector & spv,
+HfstTransducer::HfstTransducer(const StringPairVector & spv,
ImplementationType type):
type(type), anonymous(false), is_trie(false), name("")
{
if (! is_implementation_type_available(type))
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
for (StringPairVector::const_iterator it = spv.begin();
it != spv.end(); it++)
{
if (it->first == "" || it->second == "")
HFST_THROW_MESSAGE
- (EmptyStringException,
+ (EmptyStringException,
"HfstTransducer(const StringPairVector&, ImplementationType)");
}
@@ -979,13 +929,13 @@ HfstTransducer::HfstTransducer(const StringPairVector & spv,
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
- implementation.tropical_ofst =
+ implementation.tropical_ofst =
tropical_ofst_interface.define_transducer(spv);
this->type = TROPICAL_OPENFST_TYPE;
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
- implementation.log_ofst =
+ implementation.log_ofst =
log_ofst_interface.define_transducer(spv);
this->type = LOG_OPENFST_TYPE;
break;
@@ -1012,20 +962,20 @@ HfstTransducer::HfstTransducer(const StringPairVector & spv,
}
}
-HfstTransducer::HfstTransducer(const StringPairSet & sps,
- ImplementationType type,
+HfstTransducer::HfstTransducer(const StringPairSet & sps,
+ ImplementationType type,
bool cyclic):
type(type),anonymous(false),is_trie(false), name("")
{
if (! is_implementation_type_available(type))
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
for (StringPairSet::const_iterator it = sps.begin();
it != sps.end(); it++)
{
if (it->first == "" || it->second == "")
HFST_THROW_MESSAGE
- (EmptyStringException,
+ (EmptyStringException,
"HfstTransducer(const StringPairSet&, ImplementationType, bool)");
}
@@ -1039,13 +989,13 @@ HfstTransducer::HfstTransducer(const StringPairSet & sps,
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
- implementation.tropical_ofst =
+ implementation.tropical_ofst =
tropical_ofst_interface.define_transducer(sps,cyclic);
this->type = TROPICAL_OPENFST_TYPE;
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
- implementation.log_ofst =
+ implementation.log_ofst =
log_ofst_interface.define_transducer(sps,cyclic);
this->type = LOG_OPENFST_TYPE;
break;
@@ -1077,7 +1027,7 @@ HfstTransducer::HfstTransducer(const std::vector<StringPairSet> & spsv,
type(type),anonymous(false),is_trie(false), name("")
{
if (! is_implementation_type_available(type))
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
for (std::vector<StringPairSet>::const_iterator it = spsv.begin();
it != spsv.end(); it++)
@@ -1087,7 +1037,7 @@ HfstTransducer::HfstTransducer(const std::vector<StringPairSet> & spsv,
{
if (IT->first == "" || IT->second == "")
HFST_THROW_MESSAGE
- (EmptyStringException,
+ (EmptyStringException,
"HfstTransducer(const std::vector<StringPairSet>&, "
" ImplementationType)");
}
@@ -1103,13 +1053,13 @@ HfstTransducer::HfstTransducer(const std::vector<StringPairSet> & spsv,
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
- implementation.tropical_ofst =
+ implementation.tropical_ofst =
tropical_ofst_interface.define_transducer(spsv);
this->type = TROPICAL_OPENFST_TYPE;
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
- implementation.log_ofst =
+ implementation.log_ofst =
log_ofst_interface.define_transducer(spsv);
this->type = LOG_OPENFST_TYPE;
break;
@@ -1138,22 +1088,22 @@ HfstTransducer::HfstTransducer(const std::vector<StringPairSet> & spsv,
HfstTransducer::HfstTransducer(const std::string& upper_utf8_str,
const std::string& lower_utf8_str,
- const HfstTokenizer
+ const HfstTokenizer
&multichar_symbol_tokenizer,
ImplementationType type):
type(type),anonymous(false),is_trie(true), name("")
{
if (! is_implementation_type_available(type))
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
- if (upper_utf8_str == "" ||
+ if (upper_utf8_str == "" ||
lower_utf8_str == "")
HFST_THROW_MESSAGE
(EmptyStringException,
"HfstTransducer(const std::string&, const std::string&, "
- "const HfstTokenizer&, ImplementationType");
+ "const HfstTokenizer&, ImplementationType");
- StringPairVector spv =
+ StringPairVector spv =
multichar_symbol_tokenizer.tokenize
(upper_utf8_str,lower_utf8_str);
switch (type)
@@ -1165,13 +1115,13 @@ HfstTransducer::HfstTransducer(const std::string& upper_utf8_str,
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
- implementation.tropical_ofst =
+ implementation.tropical_ofst =
tropical_ofst_interface.define_transducer(spv);
this->type = TROPICAL_OPENFST_TYPE;
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
- implementation.log_ofst =
+ implementation.log_ofst =
log_ofst_interface.define_transducer(spv);
break;
#endif
@@ -1191,7 +1141,7 @@ HfstTransducer::HfstTransducer(const std::string& upper_utf8_str,
case ERROR_TYPE:
HFST_THROW(SpecifiedTypeRequiredException);
default:
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
}
}
@@ -1200,10 +1150,10 @@ HfstTransducer::HfstTransducer(HfstInputStream &in):
type(in.type), anonymous(false),is_trie(false), name("")
{
if (! is_implementation_type_available(type)) {
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
}
- in.read_transducer(*this);
+ in.read_transducer(*this);
}
HfstTransducer::HfstTransducer(const HfstTransducer &another):
@@ -1211,7 +1161,8 @@ HfstTransducer::HfstTransducer(const HfstTransducer &another):
is_trie(another.is_trie), name("")
{
if (! is_implementation_type_available(type))
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
+
for (map<string,string>::const_iterator prop = another.props.begin();
prop != another.props.end();
++prop)
@@ -1251,12 +1202,12 @@ HfstTransducer::HfstTransducer(const HfstTransducer &another):
break;
#endif
case HFST_OL_TYPE:
- implementation.hfst_ol
+ implementation.hfst_ol
= another.implementation.hfst_ol->copy
(another.implementation.hfst_ol, false);
break;
case HFST_OLW_TYPE:
- implementation.hfst_ol
+ implementation.hfst_ol
= another.implementation.hfst_ol->copy
(another.implementation.hfst_ol, true);
break;
@@ -1273,38 +1224,38 @@ HfstTransducer::HfstTransducer
type(type),anonymous(false),is_trie(false), name("")
{
if (! is_implementation_type_available(type))
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
switch (type)
{
#if HAVE_SFST
case SFST_TYPE:
- implementation.sfst =
+ implementation.sfst =
ConversionFunctions::hfst_basic_transducer_to_sfst(&net);
break;
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
- implementation.tropical_ofst =
+ implementation.tropical_ofst =
ConversionFunctions::hfst_basic_transducer_to_tropical_ofst(&net);
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
- implementation.log_ofst =
+ implementation.log_ofst =
ConversionFunctions::hfst_basic_transducer_to_log_ofst(&net);
break;
#endif
#endif
#if HAVE_FOMA
case FOMA_TYPE:
- implementation.foma =
+ implementation.foma =
ConversionFunctions::hfst_basic_transducer_to_foma(&net);
break;
#endif
#if HAVE_XFSM
// this is slow!
case XFSM_TYPE:
- implementation.xfsm =
+ implementation.xfsm =
ConversionFunctions::hfst_basic_transducer_to_xfsm(&net);
break;
#endif
@@ -1321,27 +1272,28 @@ HfstTransducer::HfstTransducer
default:
HFST_THROW(FunctionNotImplementedException);
}
+ name = net.name;
}
HfstTransducer::~HfstTransducer(void)
{
if (! is_implementation_type_available(type))
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
switch (type)
{
#if HAVE_SFST
case SFST_TYPE:
- delete implementation.sfst;
+ sfst_interface.delete_transducer(implementation.sfst);
break;
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
- delete implementation.tropical_ofst;
+ tropical_ofst_interface.delete_transducer(implementation.tropical_ofst);
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
- delete implementation.log_ofst;
+ log_ofst_interface.delete_transducer(implementation.log_ofst);
break;
#endif
#endif
@@ -1368,18 +1320,18 @@ HfstTransducer::~HfstTransducer(void)
}
-HfstTransducer::HfstTransducer(const std::string &symbol,
- ImplementationType type):
+HfstTransducer::HfstTransducer(const std::string &symbol,
+ ImplementationType type):
type(type),anonymous(false),is_trie(false), name("")
{
if (! is_implementation_type_available(type))
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
HfstTokenizer::check_utf8_correctness(symbol);
if (symbol == "")
HFST_THROW_MESSAGE
- (EmptyStringException,
+ (EmptyStringException,
"HfstTransducer(const std::string&, ImplementationType)");
switch (this->type)
@@ -1391,7 +1343,7 @@ HfstTransducer::HfstTransducer(const std::string &symbol,
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
- implementation.tropical_ofst =
+ implementation.tropical_ofst =
tropical_ofst_interface.define_transducer(symbol);
this->type = TROPICAL_OPENFST_TYPE;
break;
@@ -1420,20 +1372,20 @@ HfstTransducer::HfstTransducer(const std::string &symbol,
}
}
-HfstTransducer::HfstTransducer(const std::string &isymbol,
- const std::string &osymbol,
+HfstTransducer::HfstTransducer(const std::string &isymbol,
+ const std::string &osymbol,
ImplementationType type):
type(type),anonymous(false),is_trie(false), name("")
{
if (! is_implementation_type_available(type))
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
HfstTokenizer::check_utf8_correctness(isymbol);
HfstTokenizer::check_utf8_correctness(osymbol);
if (isymbol == "" || osymbol == "")
HFST_THROW_MESSAGE
- (EmptyStringException,
+ (EmptyStringException,
"HfstTransducer(const std::string&, const std::string&, "
" ImplementationType)");
@@ -1441,33 +1393,33 @@ HfstTransducer::HfstTransducer(const std::string &isymbol,
{
#if HAVE_SFST
case SFST_TYPE:
- implementation.sfst
+ implementation.sfst
= sfst_interface.define_transducer(isymbol, osymbol);
break;
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
- implementation.tropical_ofst
+ implementation.tropical_ofst
= tropical_ofst_interface.define_transducer(isymbol, osymbol);
this->type = TROPICAL_OPENFST_TYPE;
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
- implementation.log_ofst
+ implementation.log_ofst
= log_ofst_interface.define_transducer(isymbol, osymbol);
break;
#endif
#endif
#if HAVE_FOMA
case FOMA_TYPE:
- implementation.foma
+ implementation.foma
= foma_interface.define_transducer(isymbol, osymbol);
// should the char*:s be deleted?
break;
#endif
#if HAVE_XFSM
case XFSM_TYPE:
- implementation.xfsm
+ implementation.xfsm
= xfsm_interface.define_transducer(isymbol, osymbol);
// should the char*:s be deleted?
break;
@@ -1489,23 +1441,23 @@ HfstTransducer::HfstTransducer(const std::string &isymbol,
ImplementationType HfstTransducer::get_type(void) const {
return this->type; }
void HfstTransducer::set_name(const std::string &name) {
- this->set_property("name", name);
-}
+ this->set_property(std::string("name"), name);
+}
std::string HfstTransducer::get_name() const {
- return this->get_property("name"); }
+ return this->get_property(std::string("name")); }
void
-HfstTransducer::set_property(const string& property, const string& name)
+HfstTransducer::set_property(const std::string& property, const std::string& name)
{
HfstTokenizer::check_utf8_correctness(name);
this->props[property] = name;
- if (property == "name")
+ if (property == std::string("name"))
{
this->name = name;
}
}
-string HfstTransducer::get_property(const string& property) const
+std::string HfstTransducer::get_property(const std::string& property) const
{
if (this->props.find(property) != this->props.end())
{
@@ -1513,10 +1465,10 @@ string HfstTransducer::get_property(const string& property) const
}
else
{
- return "";
+ return std::string("");
}
}
-const std::map<string,string>&
+const std::map<std::string,std::string>&
HfstTransducer::get_properties() const
{
return this->props;
@@ -1531,7 +1483,7 @@ HfstTransducer::get_properties() const
bool HfstTransducer::compare(const HfstTransducer &another, bool harmonize) const
{
if (this->type != another.type)
- HFST_THROW_MESSAGE(TransducerTypeMismatchException,
+ HFST_THROW_MESSAGE(TransducerTypeMismatchException,
"HfstTransducer::compare");
HfstTransducer one_copy(*this);
@@ -1539,7 +1491,7 @@ bool HfstTransducer::compare(const HfstTransducer &another, bool harmonize) cons
/* prevent harmonization, if needed */
if (! harmonize)
- {
+ {
one_copy.insert_missing_symbols_to_alphabet_from(another_copy);
another_copy.insert_missing_symbols_to_alphabet_from(one_copy);
}
@@ -1554,38 +1506,39 @@ bool HfstTransducer::compare(const HfstTransducer &another, bool harmonize) cons
delete tmp;
}
- one_copy.minimize();
+ one_copy.determinize();
+ another_copy.determinize();
switch (one_copy.type)
{
#if HAVE_SFST
case SFST_TYPE:
return one_copy.sfst_interface.are_equivalent(
- one_copy.implementation.sfst,
+ one_copy.implementation.sfst,
another_copy.implementation.sfst);
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
return one_copy.tropical_ofst_interface.are_equivalent(
- one_copy.implementation.tropical_ofst,
+ one_copy.implementation.tropical_ofst,
another_copy.implementation.tropical_ofst);
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
return one_copy.log_ofst_interface.are_equivalent(
- one_copy.implementation.log_ofst,
+ one_copy.implementation.log_ofst,
another_copy.implementation.log_ofst);
#endif
#endif
#if HAVE_FOMA
case FOMA_TYPE:
return one_copy.foma_interface.are_equivalent(
- one_copy.implementation.foma,
+ one_copy.implementation.foma,
another_copy.implementation.foma);
#endif
#if HAVE_XFSM
case XFSM_TYPE:
return one_copy.xfsm_interface.are_equivalent(
- one_copy.implementation.xfsm,
+ one_copy.implementation.xfsm,
another_copy.implementation.xfsm);
#endif
case ERROR_TYPE:
@@ -1622,7 +1575,7 @@ bool HfstTransducer::is_automaton(void) const
#endif
#if HAVE_XFSM
case XFSM_TYPE:
- HFST_THROW(FunctionNotImplementedException);
+ HFST_THROW(FunctionNotImplementedException);
#endif
case ERROR_TYPE:
HFST_THROW(TransducerHasWrongTypeException);
@@ -1723,9 +1676,9 @@ unsigned int HfstTransducer::number_of_arcs() const
// -----------------------------------------------------------------------
// if (required): return ~[(?* FAIL_FLAGS) ~$SUCCEED_FLAGS SELF ?*]
-// if (! required): return ~[?* FAIL_FLAGS ~$SUCCEED_FLAGS SELF ?*]
+// if (! required): return ~[?* FAIL_FLAGS ~$SUCCEED_FLAGS SELF ?*]
static HfstTransducer * new_filter
- (const HfstTransducer & fail_flags, const HfstTransducer & succeed_flags,
+ (const HfstTransducer & fail_flags, const HfstTransducer & succeed_flags,
const HfstTransducer & self, bool required)
{
ImplementationType type = fail_flags.get_type();
@@ -1782,89 +1735,74 @@ static void substitute_escaped_flags(HfstTransducer * filter)
static int flag_build
(int ftype, char *fname, char *fvalue, int fftype, char *ffname, char *ffvalue) {
- int eq, selfnull;
- selfnull = 0; /* If current flag has no value, e.g. @R.A@ */
if (strcmp(fname,ffname) != 0)
return FLAG_NONE;
-
- bool fvalue_allocated = false; // make scan-build happy..
- bool ffvalue_allocated = false;
-
- if (fvalue == NULL) {
- fvalue = strdup("");
- fvalue_allocated = true;
- selfnull = 1;
- }
-
- if (ffvalue == NULL) {
- ffvalue = strdup("");
- ffvalue_allocated = true;
- }
- eq = strcmp(fvalue, ffvalue);
+ bool selfnull = false; /* If current flag has no value, e.g. @R.A@ */
+ if (strcmp(fvalue, "") == 0)
+ selfnull = true;
- if (fvalue_allocated) { free(fvalue); } // make scan-build happy..
- if (ffvalue_allocated) { free(ffvalue); }
+ int eq = strcmp(fvalue, ffvalue);
/* U flags */
- if (ftype == FLAG_UNIFY && fftype == FLAG_POSITIVE && eq == 0)
+ if ((ftype == FLAG_UNIFY) && (fftype == FLAG_POSITIVE) && (eq == 0))
return FLAG_SUCCEED;
- if (ftype == FLAG_UNIFY && fftype == FLAG_CLEAR)
+ if ((ftype == FLAG_UNIFY) && (fftype == FLAG_CLEAR))
return FLAG_SUCCEED;
- if (ftype == FLAG_UNIFY && fftype == FLAG_UNIFY && eq != 0)
+ if ((ftype == FLAG_UNIFY) && (fftype == FLAG_UNIFY) && (eq != 0))
return FLAG_FAIL;
- if (ftype == FLAG_UNIFY && fftype == FLAG_POSITIVE && eq != 0)
+ if ((ftype == FLAG_UNIFY) && (fftype == FLAG_POSITIVE) && (eq != 0))
return FLAG_FAIL;
- if (ftype == FLAG_UNIFY && fftype == FLAG_NEGATIVE && eq == 0)
+ if ((ftype == FLAG_UNIFY) && (fftype == FLAG_NEGATIVE) && (eq == 0))
return FLAG_FAIL;
/* R flag with value = 0 */
- if (ftype == FLAG_REQUIRE && fftype == FLAG_UNIFY && selfnull)
+ if ((ftype == FLAG_REQUIRE) && (fftype == FLAG_UNIFY) && selfnull)
return FLAG_SUCCEED;
- if (ftype == FLAG_REQUIRE && fftype == FLAG_POSITIVE && selfnull)
+ if ((ftype == FLAG_REQUIRE) && (fftype == FLAG_POSITIVE) && selfnull)
return FLAG_SUCCEED;
- if (ftype == FLAG_REQUIRE && fftype == FLAG_NEGATIVE && selfnull)
+ if ((ftype == FLAG_REQUIRE) && (fftype == FLAG_NEGATIVE) && selfnull)
return FLAG_SUCCEED;
- if (ftype == FLAG_REQUIRE && fftype == FLAG_CLEAR && selfnull)
+ if ((ftype == FLAG_REQUIRE) && (fftype == FLAG_CLEAR) && selfnull)
return FLAG_FAIL;
/* R flag with value */
- if (ftype == FLAG_REQUIRE && fftype == FLAG_POSITIVE && eq == 0 && !selfnull)
+ if ((ftype == FLAG_REQUIRE) && (fftype == FLAG_POSITIVE) && (eq == 0) && !selfnull)
return FLAG_SUCCEED;
- if (ftype == FLAG_REQUIRE && fftype == FLAG_UNIFY && eq == 0 && !selfnull)
+ if ((ftype == FLAG_REQUIRE) && (fftype == FLAG_UNIFY) && (eq == 0) && !selfnull)
return FLAG_SUCCEED;
- if (ftype == FLAG_REQUIRE && fftype == FLAG_POSITIVE && eq != 0 && !selfnull)
+ if ((ftype == FLAG_REQUIRE) && (fftype == FLAG_POSITIVE) && (eq != 0) && !selfnull)
return FLAG_FAIL;
- if (ftype == FLAG_REQUIRE && fftype == FLAG_UNIFY && eq != 0 && !selfnull)
+ if ((ftype == FLAG_REQUIRE) && (fftype == FLAG_UNIFY) && (eq != 0) && !selfnull)
return FLAG_FAIL;
- if (ftype == FLAG_REQUIRE && fftype == FLAG_NEGATIVE && !selfnull)
+ if ((ftype == FLAG_REQUIRE) && (fftype == FLAG_NEGATIVE) && !selfnull)
return FLAG_FAIL;
- if (ftype == FLAG_REQUIRE && fftype == FLAG_CLEAR && !selfnull)
+ if ((ftype == FLAG_REQUIRE) && (fftype == FLAG_CLEAR) && !selfnull)
return FLAG_FAIL;
/* D flag with value = 0 */
- if (ftype == FLAG_DISALLOW && fftype == FLAG_CLEAR && selfnull)
- return FLAG_SUCCEED;
- if (ftype == FLAG_DISALLOW && fftype == FLAG_POSITIVE && selfnull)
- return FLAG_FAIL;
- if (ftype == FLAG_DISALLOW && fftype == FLAG_UNIFY && selfnull)
- return FLAG_FAIL;
- if (ftype == FLAG_DISALLOW && fftype == FLAG_NEGATIVE && selfnull)
- return FLAG_FAIL;
+ if ((ftype == FLAG_DISALLOW) && (fftype == FLAG_CLEAR) && selfnull)
+ return FLAG_SUCCEED;
+ if ((ftype == FLAG_DISALLOW) && (fftype == FLAG_POSITIVE) && selfnull)
+ return FLAG_FAIL;
+ if ((ftype == FLAG_DISALLOW) && (fftype == FLAG_UNIFY) && selfnull)
+ return FLAG_FAIL;
+ if ((ftype == FLAG_DISALLOW) && (fftype == FLAG_NEGATIVE) && selfnull)
+ return FLAG_FAIL;
/* D flag with value */
- if (ftype == FLAG_DISALLOW && fftype == FLAG_POSITIVE && eq != 0 && !selfnull)
+ if ((ftype == FLAG_DISALLOW) && (fftype == FLAG_POSITIVE) && (eq != 0) && !selfnull)
return FLAG_SUCCEED;
- if (ftype == FLAG_DISALLOW && fftype == FLAG_CLEAR && !selfnull)
+ if ((ftype == FLAG_DISALLOW) && (fftype == FLAG_CLEAR) && !selfnull)
return FLAG_SUCCEED;
- if (ftype == FLAG_DISALLOW && fftype == FLAG_NEGATIVE && eq == 0 && !selfnull)
+ if ((ftype == FLAG_DISALLOW) && (fftype == FLAG_NEGATIVE ) && (eq == 0) && !selfnull)
return FLAG_SUCCEED;
- if (ftype == FLAG_DISALLOW && fftype == FLAG_POSITIVE && eq == 0 && !selfnull)
+ if ((ftype == FLAG_DISALLOW) && (fftype == FLAG_POSITIVE) && (eq == 0) && !selfnull)
return FLAG_FAIL;
- if (ftype == FLAG_DISALLOW && fftype == FLAG_UNIFY && eq == 0 && !selfnull)
+ if ((ftype == FLAG_DISALLOW) && (fftype == FLAG_UNIFY) && (eq == 0) && !selfnull)
return FLAG_FAIL;
- if (ftype == FLAG_DISALLOW && fftype == FLAG_NEGATIVE && eq != 0 && !selfnull)
+ if ((ftype == FLAG_DISALLOW) && (fftype == FLAG_NEGATIVE ) && (eq != 0) && !selfnull)
return FLAG_FAIL;
return FLAG_NONE;
@@ -1908,10 +1846,10 @@ static int is_valid_flag_combination
}
-/* @brief Get flag filter for transducer \a transducer.
- @param transducer The transducer that is going to be filtered.
- @param flags The set of all flags in \a transducer.
- @param flag The flag that is filtered, if empty string, all flags are filtered.
+/* @brief Get flag filter for transducer \a transducer.
+ @param transducer The transducer that is going to be filtered.
+ @param flags The set of all flags in \a transducer.
+ @param flag The flag that is filtered, if empty string, all flags are filtered.
@return The flag filter, if there is nothing to filter, return NULL. */
static HfstTransducer * get_flag_filter
(const HfstTransducer * transducer, const StringSet & flags, const String & flag)
@@ -1928,8 +1866,8 @@ static HfstTransducer * get_flag_filter
HfstTransducer fail_flags(type);
char op = FdOperation::get_operator(*f)[0];
- if ((flag == "" || FdOperation::get_feature(*f) == flag)
- && (op == 'U' || op == 'R' || op == 'D')) // Equal flag?
+ if ((flag == "" || FdOperation::get_feature(*f) == flag)
+ && (op == 'U' || op == 'R' || op == 'D')) // Equal flag?
{
for (StringSet::const_iterator ff = flags.begin();
ff != flags.end(); ff++)
@@ -1943,7 +1881,7 @@ static HfstTransducer * get_flag_filter
succeed_flags.disjunct(HfstTransducer("_" + *ff, type));
flag_found = true;
} else {
- succeed_flags.disjunct(HfstTransducer("_" + *ff, type));
+ ;
}
}
@@ -1967,7 +1905,7 @@ static HfstTransducer * get_flag_filter
if (filter != NULL)
{
substitute_escaped_flags(filter); // unescape the flags
- filter->minimize();
+ filter->optimize();
}
return filter;
@@ -2008,7 +1946,7 @@ HfstTransducer &HfstTransducer::eliminate_flags()
StringSet flags = basic.get_flags();
HfstTransducer * filter = get_flag_filter(this, flags, "");
- if (filter != NULL)
+ if (filter != NULL)
{
HfstTransducer filter_copy(*filter);
filter_copy.compose(*this).compose(*filter);
@@ -2021,12 +1959,11 @@ HfstTransducer &HfstTransducer::eliminate_flags()
flag_purge(*this, "");
}
- return this->minimize();
+ return this->optimize();
}
HfstTransducer &HfstTransducer::eliminate_flag(const std::string & flag)
{
-
HfstBasicTransducer basic(*this);
StringSet flags = basic.get_flags();
bool feature_found = false;
@@ -2063,7 +2000,7 @@ HfstTransducer &HfstTransducer::eliminate_flag(const std::string & flag)
#endif
HfstTransducer * filter = get_flag_filter(this, flags, flag);
- if (filter != NULL)
+ if (filter != NULL)
{
HfstTransducer filter_copy(*filter);
filter_copy.compose(*this).compose(*filter);
@@ -2073,10 +2010,10 @@ HfstTransducer &HfstTransducer::eliminate_flag(const std::string & flag)
}
else
{
- flag_purge(*this, "");
+ flag_purge(*this, flag);
}
- return this->minimize();
+ return this->optimize();
}
HfstTransducer &HfstTransducer::remove_epsilons()
@@ -2110,7 +2047,7 @@ HfstTransducer &HfstTransducer::prune()
this->convert(TROPICAL_OPENFST_TYPE);
fst::StdVectorFst * temp = hfst::implementations::TropicalWeightTransducer::prune
(this->implementation.tropical_ofst);
- delete this->implementation.tropical_ofst;
+ this->tropical_ofst_interface.delete_transducer(this->implementation.tropical_ofst);
this->implementation.tropical_ofst = temp;
return *this;
#endif
@@ -2140,7 +2077,7 @@ HfstTransducer &HfstTransducer::determinize()
NULL,
#endif
/* Add here your implementation. */
- false ); }
+ false ); }
HfstTransducer &HfstTransducer::minimize()
{ is_trie = false;
@@ -2164,7 +2101,13 @@ HfstTransducer &HfstTransducer::minimize()
false );
}
-
+HfstTransducer &HfstTransducer::optimize()
+{
+ if (can_minimize)
+ return this->minimize();
+ else
+ return this->determinize();
+}
// -----------------------------------------------------------------------
//
@@ -2191,11 +2134,11 @@ HfstTransducer &HfstTransducer::repeat_star()
NULL,
#endif
/* Add here your implementation. */
- false ); }
+ false ); }
HfstTransducer &HfstTransducer::repeat_plus()
{ is_trie = false;
- return apply(
+ return apply(
#if HAVE_SFST
&hfst::implementations::SfstTransducer::repeat_plus,
#endif
@@ -2212,7 +2155,7 @@ HfstTransducer &HfstTransducer::repeat_plus()
NULL,
#endif
/* Add here your implementation. */
- false ); }
+ false ); }
HfstTransducer &HfstTransducer::repeat_n(unsigned int n)
{ is_trie = false; // This could be done so that is_trie is preserved
@@ -2236,7 +2179,7 @@ HfstTransducer &HfstTransducer::repeat_n(unsigned int n)
//#if HAVE_MY_TRANSDUCER_LIBRARY
//&hfst::implementations::MyTransducerLibraryTransducer::repeat_n,
//#endif
- n ); }
+ n ); }
HfstTransducer &HfstTransducer::repeat_n_plus(unsigned int n)
{ is_trie = false; // This could be done so that is_trie is preserved
@@ -2270,7 +2213,7 @@ HfstTransducer &HfstTransducer::repeat_n_minus(unsigned int n)
NULL,
#endif
/* Add here your implementation. */
- n ); }
+ n ); }
HfstTransducer &HfstTransducer::repeat_n_to_k(unsigned int n, unsigned int k)
{ is_trie = false; // This could be done so that is_trie is preserved
@@ -2312,7 +2255,7 @@ HfstTransducer &HfstTransducer::optionalize()
&hfst::implementations::XfsmTransducer::optionalize,
#endif
/* Add here your implementation. */
- false ); }
+ false ); }
HfstTransducer &HfstTransducer::invert()
{ is_trie = false; // This could be done so that is_trie is preserved
@@ -2333,7 +2276,7 @@ HfstTransducer &HfstTransducer::invert()
&hfst::implementations::XfsmTransducer::invert,
#endif
/* Add here your implementation. */
- false ); }
+ false ); }
HfstTransducer &HfstTransducer::reverse()
{ is_trie = false; // This could be done so that is_trie is preserved
@@ -2354,7 +2297,7 @@ HfstTransducer &HfstTransducer::reverse()
&hfst::implementations::XfsmTransducer::reverse,
#endif
/* Add here your implementation. */
- false ); }
+ false ); }
HfstTransducer &HfstTransducer::input_project()
{ is_trie = false; // This could be done so that is_trie is preserved
@@ -2414,13 +2357,13 @@ std::vector<HfstTransducer*> HfstTransducer::extract_path_transducers()
std::vector<HfstTransducer*> hfst_paths;
#if HAVE_SFST
- std::vector<SFST::Transducer*> sfst_paths
+ std::vector<SFST::Transducer*> sfst_paths
= sfst_interface.extract_path_transducers(this->implementation.sfst);
- for (std::vector<SFST::Transducer*>::iterator it
+ for (std::vector<SFST::Transducer*>::iterator it
= sfst_paths.begin(); it != sfst_paths.end(); it++)
{
HfstTransducer *tr = new HfstTransducer(SFST_TYPE);
- delete tr->implementation.sfst;
+ sfst_interface.delete_transducer(tr->implementation.sfst);
tr->implementation.sfst = *it;
hfst_paths.push_back(tr);
}
@@ -2430,7 +2373,7 @@ std::vector<HfstTransducer*> HfstTransducer::extract_path_transducers()
void HfstTransducer::extract_paths(ExtractStringsCb& callback, int cycles)
const
-{
+{
switch (this->type)
{
#if HAVE_OPENFST
@@ -2470,17 +2413,17 @@ void HfstTransducer::extract_paths(ExtractStringsCb& callback, int cycles)
}
}
-void HfstTransducer::extract_paths_fd(ExtractStringsCb& callback,
+void HfstTransducer::extract_paths_fd(ExtractStringsCb& callback,
int cycles, bool filter_fd)
const
-{
+{
switch (this->type)
{
#if HAVE_OPENFST
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
{
- FdTable<int64>* t_log_ofst
+ FdTable<int64>* t_log_ofst
= hfst::implementations::LogWeightTransducer::get_flag_diacritics
(implementation.log_ofst);
hfst::implementations::LogWeightTransducer::extract_paths
@@ -2491,7 +2434,7 @@ void HfstTransducer::extract_paths_fd(ExtractStringsCb& callback,
#endif
case TROPICAL_OPENFST_TYPE:
{
- FdTable<int64>* t_tropical_ofst
+ FdTable<int64>* t_tropical_ofst
= hfst::implementations::TropicalWeightTransducer::
get_flag_diacritics(implementation.tropical_ofst);
hfst::implementations::TropicalWeightTransducer::extract_paths
@@ -2504,7 +2447,7 @@ void HfstTransducer::extract_paths_fd(ExtractStringsCb& callback,
#if HAVE_SFST
case SFST_TYPE:
{
- FdTable<SFST::Character>* t_sfst
+ FdTable<SFST::Character>* t_sfst
= hfst::implementations::SfstTransducer::get_flag_diacritics
(implementation.sfst);
hfst::implementations::SfstTransducer::extract_paths
@@ -2516,7 +2459,7 @@ void HfstTransducer::extract_paths_fd(ExtractStringsCb& callback,
#if HAVE_FOMA
case FOMA_TYPE:
{
- FdTable<int>* t_foma
+ FdTable<int>* t_foma
= hfst::implementations::FomaTransducer::get_flag_diacritics
(implementation.foma);
hfst::implementations::FomaTransducer::extract_paths
@@ -2529,7 +2472,7 @@ void HfstTransducer::extract_paths_fd(ExtractStringsCb& callback,
case HFST_OL_TYPE:
case HFST_OLW_TYPE:
{
- const FdTable<hfst_ol::SymbolNumber>* t_hfst_ol
+ const FdTable<hfst_ol::SymbolNumber>* t_hfst_ol
= hfst::implementations::HfstOlTransducer::get_flag_diacritics
(implementation.hfst_ol);
hfst::implementations::HfstOlTransducer::extract_paths
@@ -2547,11 +2490,10 @@ void HfstTransducer::extract_paths_fd(ExtractStringsCb& callback,
class ExtractStringsCb_ : public ExtractStringsCb
{
public:
- //WeightedPaths<float>::Set& paths;
HfstTwoLevelPaths& paths;
int max_num;
- ExtractStringsCb_(HfstTwoLevelPaths& p, int max):
+ ExtractStringsCb_(HfstTwoLevelPaths& p, int max):
paths(p), max_num(max) {}
RetVal operator()(HfstTwoLevelPath& path, bool final)
{
@@ -2582,13 +2524,13 @@ int HfstTransducer::longest_path_size(bool obey_flags) const
return -1;
}
// else, there is at least one path
- return results.begin()->second.size();
+ return (int)results.begin()->second.size();
}
static std::string match_any_n_times(unsigned int n, const StringSet & flags)
{
std::string match_any(" [ ? ");
- for (StringSet::const_iterator it = flags.begin();
+ for (StringSet::const_iterator it = flags.begin();
it != flags.end(); it++)
{
match_any = match_any + "| \"" + *it + "\" ";
@@ -2634,7 +2576,7 @@ bool HfstTransducer::extract_longest_paths
// filter out the paths of current length and extract them
length_tr->compose(*this);
- length_tr->minimize();
+ length_tr->optimize();
if (obey_flags)
{
length_tr->extract_paths_fd(results);
@@ -2648,35 +2590,7 @@ bool HfstTransducer::extract_longest_paths
// if paths were found
if (results.size() > 0)
{
- //if (show_flags)
return true;
- /*
- // it is possible that there are longer paths after flags have been removed
- else
- {
- results = remove_flags(results);
- results = get_longest_paths(results);
- int max_length = longest_path_length(results);
- if (max_length < 0 || max_length > *length) // this should not happen
- {
- HFST_THROW(HfstFatalException);
- }
- if (max_length == *length)
- {
- return true;
- }
- // see if there could be longer paths
- length++;
- if (*length == path_lengths.end())
- {
- return true;
- }
- if (*length < max_length)
- {
- return true;
- }
- else this gets a bit complicated...
- }*/
}
} // lengths of accepted paths gone through
@@ -2793,13 +2707,12 @@ void HfstTransducer::extract_random_paths_fd
HFST_THROW(FunctionNotImplementedException);
}
-HfstTransducer &HfstTransducer::n_best(unsigned int n)
+HfstTransducer &HfstTransducer::n_best(unsigned int n)
{
if (! is_implementation_type_available(TROPICAL_OPENFST_TYPE)) {
(void)n;
- HFST_THROW_MESSAGE(ImplementationTypeNotAvailableException,
- "HfstTransducer::n_best implemented only for "
- "TROPICAL_OPENFST_TYPE");
+ throw ImplementationTypeNotAvailableException("HfstTransducer::n_best implemented only for "
+ "TROPICAL_OPENFST_TYPE", __FILE__, __LINE__, type);
}
#if HAVE_OPENFST
@@ -2817,7 +2730,7 @@ HfstTransducer &HfstTransducer::n_best(unsigned int n)
fst::StdVectorFst * temp =
hfst::implementations::TropicalWeightTransducer::n_best
(implementation.tropical_ofst,(int)n);
- delete implementation.tropical_ofst;
+ tropical_ofst_interface.delete_transducer(implementation.tropical_ofst);
implementation.tropical_ofst = temp;
break;
}
@@ -2827,7 +2740,7 @@ HfstTransducer &HfstTransducer::n_best(unsigned int n)
hfst::implementations::LogFst * temp =
hfst::implementations::LogWeightTransducer::n_best
(implementation.log_ofst,(int)n);
- delete implementation.log_ofst;
+ log_ofst_interface.delete_transducer(implementation.log_ofst);
implementation.log_ofst = temp;
break;
}
@@ -2839,7 +2752,7 @@ HfstTransducer &HfstTransducer::n_best(unsigned int n)
case HFST_OLW_TYPE:
default:
HFST_THROW(FunctionNotImplementedException);
- break;
+ break;
}
#if HAVE_OPENFST
this->convert(original_type);
@@ -2852,7 +2765,7 @@ bool HfstTransducer::is_special_symbol(const std::string &symbol)
{
if (symbol.size() < 4)
return false;
- if (symbol[0] == '@' && symbol[symbol.size()-1] == '@' &&
+ if (symbol[0] == '@' && symbol[symbol.size()-1] == '@' &&
symbol[1] == '_' && symbol[symbol.size()-2] == '_')
return true;
return false;
@@ -2868,7 +2781,7 @@ StringSet HfstTransducer::insert_missing_diacritics_to_alphabet_from(const HfstT
it != another_alphabet.end(); it++)
{
if (this_alphabet.find(*it) == this_alphabet.end())
- {
+ {
if (FdOperation::is_diacritic(*it))
missing_flags.insert(*it);
}
@@ -2887,7 +2800,7 @@ void HfstTransducer::insert_missing_symbols_to_alphabet_from(const HfstTransduce
it != another_alphabet.end(); it++)
{
if (this_alphabet.find(*it) == this_alphabet.end())
- {
+ {
if (! only_special_symbols)
{
missing_symbols.insert(*it);
@@ -2909,10 +2822,10 @@ void HfstTransducer::insert_missing_symbols_to_alphabet_from(const HfstTransduce
//
// -----------------------------------------------------------------------
-/*
- Check for missing flag diacritics (FG), i.e. FGs that are present in the
- alphabet of \a another but not in the alphabet of this transducer and insert
- them to \a missing_flags. \a return_on_first_miss defines whether function
+/*
+ Check for missing flag diacritics (FG), i.e. FGs that are present in the
+ alphabet of \a another but not in the alphabet of this transducer and insert
+ them to \a missing_flags. \a return_on_first_miss defines whether function
returns after first missing FG is found and inserted to \a missing_flags.
@ retval Whether any missing FGs where found.
*/
@@ -2928,7 +2841,7 @@ bool HfstTransducer::check_for_missing_flags_in
for (StringSet::const_iterator it = another_alphabet.begin();
it != another_alphabet.end(); it++)
{
- if ( FdOperation::is_diacritic(*it) &&
+ if ( FdOperation::is_diacritic(*it) &&
(this_alphabet.find(*it) == this_alphabet.end()) )
{
missing_flags.insert(*it);
@@ -2941,7 +2854,7 @@ bool HfstTransducer::check_for_missing_flags_in
}
void HfstTransducer::insert_freely_missing_flags_from
-(const HfstTransducer &another)
+(const HfstTransducer &another)
{
StringSet missing_flags;
if (check_for_missing_flags_in(another, missing_flags,
@@ -2949,7 +2862,7 @@ void HfstTransducer::insert_freely_missing_flags_from
{
HfstBasicTransducer basic(*this);
- for (size_t s = 0; s <= basic.get_max_state(); ++s)
+ for (unsigned int s = 0; s <= (unsigned int)basic.get_max_state(); ++s)
{
for (StringSet::const_iterator it = missing_flags.begin();
it != missing_flags.end(); it++)
@@ -2972,7 +2885,7 @@ bool has_flags(const HfstTransducer &fst)
if (FdOperation::is_diacritic(*it))
{ return true; }
}
- return false;
+ return false;
}
bool HfstTransducer::has_flag_diacritics(void) const
@@ -2981,7 +2894,7 @@ bool HfstTransducer::has_flag_diacritics(void) const
std::string add_suffix_to_feature_name(const std::string &flag_diacritic,
const std::string &suffix)
{
- return
+ return
"@" +
FdOperation::get_operator(flag_diacritic) +
"." +
@@ -3008,7 +2921,7 @@ void rename_flag_diacritics(HfstTransducer &fst,const std::string &suffix)
it != basic_fst.end();
++it)
{
- for (HfstBasicTransducer::HfstTransitions::const_iterator jt =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator jt =
it->begin();
jt != it->end();
++jt)
@@ -3049,7 +2962,7 @@ void HfstTransducer::twosided_flag_diacritics()
it != basic_fst.end();
++it)
{
- for (HfstBasicTransducer::HfstTransitions::const_iterator jt =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator jt =
it->begin();
jt != it->end();
++jt)
@@ -3059,8 +2972,8 @@ void HfstTransducer::twosided_flag_diacritics()
bool istr_is_flag = FdOperation::is_diacritic(istr);
bool ostr_is_flag = FdOperation::is_diacritic(ostr);
- bool extra_transition_needed
- = (istr_is_flag || ostr_is_flag) && (istr != ostr);
+ bool extra_transition_needed
+ = (istr_is_flag || ostr_is_flag) && (istr != ostr);
if (extra_transition_needed)
{
@@ -3134,7 +3047,7 @@ void encode_flag_diacritics(HfstTransducer &fst)
it != basic_fst.end();
it++)
{
- for (HfstBasicTransducer::HfstTransitions::const_iterator jt =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator jt =
it->begin();
jt != it->end();
jt++)
@@ -3201,7 +3114,7 @@ void decode_flag_diacritics(HfstTransducer &fst)
it != basic_fst.end();
it++)
{
- for (HfstBasicTransducer::HfstTransitions::const_iterator jt =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator jt =
it->begin();
jt != it->end();
jt++)
@@ -3257,7 +3170,7 @@ bool is_flag_suffix
if (flag_end_pos < suffix.size())
{ return false; }
- if (flag_diacritic.substr(flag_end_pos - suffix.size(),suffix.size())
+ if (flag_diacritic.substr(flag_end_pos - suffix.size(),suffix.size())
!= suffix)
{ return false; }
@@ -3295,7 +3208,7 @@ bool HfstTransducer::check_for_missing_flags_in
(const HfstTransducer &another) const
{
StringSet foo; /* An obligatory argument that is not used. */
- return check_for_missing_flags_in(another, foo,
+ return check_for_missing_flags_in(another, foo,
true /* return on first miss */);
}
@@ -3304,14 +3217,10 @@ static bool substitute_input_flag_with_epsilon(const StringPair &sp, StringPairS
{
if (FdOperation::is_diacritic(sp.first))
{
- //if (sp.second != sp.first)
- // throw "Error: flags must be identities";
StringPair new_pair(hfst::internal_epsilon, sp.second);
sps.insert(new_pair);
return true;
}
- //if (FdOperation::is_diacritic(sp.second))
- // throw "Error: flags must be identities";
return false;
}
@@ -3319,14 +3228,10 @@ static bool substitute_output_flag_with_epsilon(const StringPair &sp, StringPair
{
if (FdOperation::is_diacritic(sp.second))
{
- //if (sp.first != sp.second)
- // throw "Error: flags must be identities";
StringPair new_pair(sp.first, hfst::internal_epsilon);
sps.insert(new_pair);
return true;
}
- //if (FdOperation::is_diacritic(sp.first))
- // throw "Error: flags must be identities";
return false;
}
@@ -3363,7 +3268,7 @@ HfstTransducer &HfstTransducer::insert_freely
if (symbol_pair.first == "" || symbol_pair.second == "")
HFST_THROW_MESSAGE
- (EmptyStringException,
+ (EmptyStringException,
"insert_freely(const StringPair&)");
HfstTransducer tr(symbol_pair.first, symbol_pair.second, this->get_type());
@@ -3375,7 +3280,7 @@ HfstTransducer &HfstTransducer::insert_freely
{
if (this->type != tr.type)
HFST_THROW_MESSAGE(TransducerTypeMismatchException,
- "HfstTransducer::insert_freely");
+ "HfstTransducer::insert_freely");
// Segfaults in xfst command line tool...
#if HAVE_XFSM
@@ -3383,7 +3288,7 @@ HfstTransducer &HfstTransducer::insert_freely
{
this->xfsm_interface.insert_freely(this->implementation.xfsm, tr.implementation.xfsm);
return *this;
- }
+ }
#endif
/* In this function, this transducer must always be harmonized
@@ -3401,24 +3306,24 @@ HfstTransducer &HfstTransducer::insert_freely
tr_harmonized = new HfstTransducer(tr);
}
- switch (this->type)
+ switch (this->type)
{
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
{
- hfst::implementations::HfstBasicTransducer * net =
+ hfst::implementations::HfstBasicTransducer * net =
ConversionFunctions::tropical_ofst_to_hfst_basic_transducer
(implementation.tropical_ofst);
- delete implementation.tropical_ofst;
+ tropical_ofst_interface.delete_transducer(implementation.tropical_ofst);
- hfst::implementations::HfstBasicTransducer * substituting_net =
+ hfst::implementations::HfstBasicTransducer * substituting_net =
ConversionFunctions::tropical_ofst_to_hfst_basic_transducer
(tr_harmonized->implementation.tropical_ofst);
delete tr_harmonized;
net->insert_freely(*substituting_net);
delete substituting_net;
- implementation.tropical_ofst =
+ implementation.tropical_ofst =
ConversionFunctions::hfst_basic_transducer_to_tropical_ofst(net);
delete net;
return *this;
@@ -3427,19 +3332,19 @@ HfstTransducer &HfstTransducer::insert_freely
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
{
- hfst::implementations::HfstBasicTransducer * net =
+ hfst::implementations::HfstBasicTransducer * net =
ConversionFunctions::log_ofst_to_hfst_basic_transducer
(implementation.log_ofst);
- delete implementation.log_ofst;
+ log_ofst_interface.delete_transducer(implementation.log_ofst);
- hfst::implementations::HfstBasicTransducer * substituting_net =
+ hfst::implementations::HfstBasicTransducer * substituting_net =
ConversionFunctions::log_ofst_to_hfst_basic_transducer
(tr_harmonized->implementation.log_ofst);
delete tr_harmonized;
net->insert_freely(*substituting_net);
delete substituting_net;
- implementation.log_ofst =
+ implementation.log_ofst =
ConversionFunctions::hfst_basic_transducer_to_log_ofst(net);
delete net;
return *this;
@@ -3453,25 +3358,25 @@ HfstTransducer &HfstTransducer::insert_freely
// HfstTransducer::harmonize does nothing to a foma transducer,
// because foma's own functions take care of harmonizing.
// Now we need to harmonize because we are using internal transducers.
- if (harmonize)
+ if (harmonize)
{
this->foma_interface.harmonize
(implementation.foma,tr_harmonized->implementation.foma);
}
- hfst::implementations::HfstBasicTransducer * net =
+ hfst::implementations::HfstBasicTransducer * net =
ConversionFunctions::foma_to_hfst_basic_transducer
(implementation.foma);
this->foma_interface.delete_foma(implementation.foma);
- hfst::implementations::HfstBasicTransducer * substituting_net =
+ hfst::implementations::HfstBasicTransducer * substituting_net =
ConversionFunctions::foma_to_hfst_basic_transducer
(tr_harmonized->implementation.foma);
delete tr_harmonized;
net->insert_freely(*substituting_net);
delete substituting_net;
- implementation.foma =
+ implementation.foma =
ConversionFunctions::hfst_basic_transducer_to_foma(net);
delete net;
return *this;
@@ -3481,19 +3386,19 @@ HfstTransducer &HfstTransducer::insert_freely
#if HAVE_SFST
case SFST_TYPE:
{
- hfst::implementations::HfstBasicTransducer * net =
+ hfst::implementations::HfstBasicTransducer * net =
ConversionFunctions::sfst_to_hfst_basic_transducer
(implementation.sfst);
- delete implementation.sfst;
+ sfst_interface.delete_transducer(implementation.sfst);
- hfst::implementations::HfstBasicTransducer * substituting_net =
+ hfst::implementations::HfstBasicTransducer * substituting_net =
ConversionFunctions::sfst_to_hfst_basic_transducer
(tr_harmonized->implementation.sfst);
delete tr_harmonized;
net->insert_freely(*substituting_net);
delete substituting_net;
- implementation.sfst =
+ implementation.sfst =
ConversionFunctions::hfst_basic_transducer_to_sfst(net);
delete net;
return *this;
@@ -3523,7 +3428,7 @@ HfstTransducer &HfstTransducer::substitute
if (this->type == XFSM_TYPE)
HFST_THROW(FunctionNotImplementedException);
#endif
- hfst::implementations::HfstBasicTransducer * net
+ hfst::implementations::HfstBasicTransducer * net
= convert_to_basic_transducer();
net->substitute(func);
return convert_to_hfst_transducer(net);
@@ -3540,7 +3445,7 @@ HfstTransducer &HfstTransducer::substitute
// empty strings are not accepted
if (old_symbol == "" || new_symbol == "")
HFST_THROW_MESSAGE
- (EmptyStringException,
+ (EmptyStringException,
"substitute(const std::string&, const std::string&, bool, bool)");
// if there are implementations available, use them
@@ -3552,7 +3457,7 @@ HfstTransducer &HfstTransducer::substitute
hfst::implementations::Transducer * tmp =
this->sfst_interface.substitute
(implementation.sfst, old_symbol, new_symbol);
- delete implementation.sfst;
+ sfst_interface.delete_transducer(implementation.sfst);
implementation.sfst = tmp;
return *this;
}
@@ -3564,7 +3469,7 @@ HfstTransducer &HfstTransducer::substitute
fst::StdVectorFst * tmp =
this->tropical_ofst_interface.substitute
(implementation.tropical_ofst, old_symbol, new_symbol);
- delete implementation.tropical_ofst;
+ tropical_ofst_interface.delete_transducer(implementation.tropical_ofst);
implementation.tropical_ofst = tmp;
return *this;
}
@@ -3574,7 +3479,7 @@ HfstTransducer &HfstTransducer::substitute
hfst::implementations::LogFst * tmp =
this->log_ofst_interface.substitute
(implementation.log_ofst, old_symbol, new_symbol);
- delete implementation.log_ofst;
+ log_ofst_interface.delete_transducer(implementation.log_ofst);
implementation.log_ofst = tmp;
return *this;
}
@@ -3592,9 +3497,9 @@ HfstTransducer &HfstTransducer::substitute
HfstTransducer &HfstTransducer::substitute
-(const StringPair &old_symbol_pair,
+(const StringPair &old_symbol_pair,
const StringPair &new_symbol_pair)
-{
+{
#if HAVE_XFSM
if (this->type == XFSM_TYPE)
HFST_THROW(FunctionNotImplementedException);
@@ -3603,10 +3508,10 @@ HfstTransducer &HfstTransducer::substitute
if (old_symbol_pair.first == "" || old_symbol_pair.second == "" ||
new_symbol_pair.first == "" || new_symbol_pair.second == "")
HFST_THROW_MESSAGE
- (EmptyStringException,
+ (EmptyStringException,
"substitute(const StringPair&, const StringPair&)");
- hfst::implementations::HfstBasicTransducer * net
+ hfst::implementations::HfstBasicTransducer * net
= convert_to_basic_transducer();
net->substitute(old_symbol_pair, new_symbol_pair);
convert_to_hfst_transducer(net);
@@ -3614,22 +3519,22 @@ HfstTransducer &HfstTransducer::substitute
}
HfstTransducer &HfstTransducer::substitute
-(const StringPair &old_symbol_pair,
+(const StringPair &old_symbol_pair,
const StringPairSet &new_symbol_pair_set)
-{
+{
#if HAVE_XFSM
if (this->type == XFSM_TYPE)
HFST_THROW(FunctionNotImplementedException);
#endif
if(old_symbol_pair.first == "" || old_symbol_pair.second == "")
HFST_THROW_MESSAGE
- (EmptyStringException,
+ (EmptyStringException,
"substitute(const StringPair&, const StringPairSet&");
- hfst::implementations::HfstBasicTransducer * net
+ hfst::implementations::HfstBasicTransducer * net
= convert_to_basic_transducer();
net->substitute(old_symbol_pair, new_symbol_pair_set);
- return convert_to_hfst_transducer(net);
+ return convert_to_hfst_transducer(net);
}
@@ -3653,20 +3558,21 @@ HfstTransducer &HfstTransducer::substitute
if (this->type == XFSM_TYPE)
HFST_THROW(FunctionNotImplementedException);
#endif
- hfst::implementations::HfstBasicTransducer * net
+ hfst::implementations::HfstBasicTransducer * net
= convert_to_basic_transducer();
try {
net->substitute(substitutions);
- }
+ }
catch (const FunctionNotImplementedException & e) {
+ (void)e;
for (HfstSymbolSubstitutions::const_iterator it =
substitutions.begin(); it != substitutions.end(); it++) {
net->substitute(it->first, it->second, true, true);
}
}
- return convert_to_hfst_transducer(net);
+ return convert_to_hfst_transducer(net);
}
HfstTransducer &HfstTransducer::substitute_symbol_pairs
@@ -3677,21 +3583,21 @@ HfstTransducer &HfstTransducer::substitute_symbol_pairs
HfstTransducer &HfstTransducer::substitute
(const HfstSymbolPairSubstitutions &substitutions)
-{
+{
#if HAVE_XFSM
if (this->type == XFSM_TYPE)
HFST_THROW(FunctionNotImplementedException);
#endif
- hfst::implementations::HfstBasicTransducer * net
+ hfst::implementations::HfstBasicTransducer * net
= convert_to_basic_transducer();
net->substitute(substitutions);
- return convert_to_hfst_transducer(net);
+ return convert_to_hfst_transducer(net);
}
HfstTransducer &HfstTransducer::substitute
(const StringPair &symbol_pair,
HfstTransducer &transducer, bool harmonize)
-{
+{
#if HAVE_XFSM
if (this->type == XFSM_TYPE)
HFST_THROW(FunctionNotImplementedException);
@@ -3702,7 +3608,7 @@ HfstTransducer &HfstTransducer::substitute
if (symbol_pair.first == "" || symbol_pair.second == "")
HFST_THROW_MESSAGE
- (EmptyStringException,
+ (EmptyStringException,
"substitute(const StringPair&, HfstTransducer&)");
#if HAVE_SFST && HAVE_OPENFST
@@ -3752,18 +3658,18 @@ HfstTransducer &HfstTransducer::substitute
this->foma_interface.harmonize
(implementation.foma,transducer.implementation.foma);
- hfst::implementations::HfstBasicTransducer * net =
+ hfst::implementations::HfstBasicTransducer * net =
ConversionFunctions::foma_to_hfst_basic_transducer
(implementation.foma);
this->foma_interface.delete_foma(implementation.foma);
- hfst::implementations::HfstBasicTransducer * substituting_net =
+ hfst::implementations::HfstBasicTransducer * substituting_net =
ConversionFunctions::foma_to_hfst_basic_transducer
(transducer.implementation.foma);
net->substitute(symbol_pair, *substituting_net);
delete substituting_net;
- implementation.foma =
+ implementation.foma =
ConversionFunctions::hfst_basic_transducer_to_foma(net);
delete net;
return *this;
@@ -3816,7 +3722,7 @@ HfstTransducer &HfstTransducer::set_final_weights(float weight, bool increment)
implementation.log_ofst =
this->log_ofst_interface.set_final_weights
(this->implementation.log_ofst, weight);
- return *this;
+ return *this;
}
#endif
#endif
@@ -3828,12 +3734,12 @@ HfstTransducer &HfstTransducer::push_weights(PushType push_type)
{
#if HAVE_OPENFST
bool to_initial_state = (push_type == TO_INITIAL_STATE);
- if (this->type == TROPICAL_OPENFST_TYPE)
+ if (this->type == TROPICAL_OPENFST_TYPE)
{
hfst::implementations::StdVectorFst * tmp =
this->tropical_ofst_interface.push_weights
(this->implementation.tropical_ofst, to_initial_state);
- delete this->implementation.tropical_ofst;
+ this->tropical_ofst_interface.delete_transducer(this->implementation.tropical_ofst);
this->implementation.tropical_ofst = tmp;
return *this;
}
@@ -3843,7 +3749,7 @@ HfstTransducer &HfstTransducer::push_weights(PushType push_type)
hfst::implementations::LogFst * tmp =
this->log_ofst_interface.push_weights
(this->implementation.log_ofst, to_initial_state);
- delete this->implementation.log_ofst;
+ this->log_ofst_interface.delete_transducer(this->implementation.log_ofst);
this->implementation.log_ofst = tmp;
return *this;
}
@@ -3891,13 +3797,13 @@ bool substitute_single_identity_with_the_other_symbol
std::string isymbol = sp.first;
std::string osymbol = sp.second;
- if (isymbol.compare("@_IDENTITY_SYMBOL_@") == 0 &&
+ if (isymbol.compare("@_IDENTITY_SYMBOL_@") == 0 &&
(osymbol.compare("@_IDENTITY_SYMBOL_@") != 0)) {
isymbol = std::string("@_UNKNOWN_SYMBOL_@");
sps.insert(StringPair(isymbol, osymbol));
return true;
}
- else if (osymbol.compare("@_IDENTITY_SYMBOL_@") == 0 &&
+ else if (osymbol.compare("@_IDENTITY_SYMBOL_@") == 0 &&
(isymbol.compare("@_IDENTITY_SYMBOL_@") != 0)) {
osymbol = std::string("@_UNKNOWN_SYMBOL_@");
sps.insert(StringPair(isymbol, osymbol));
@@ -3913,7 +3819,7 @@ bool substitute_unknown_identity_pairs
std::string isymbol = sp.first;
std::string osymbol = sp.second;
- if (isymbol.compare("@_UNKNOWN_SYMBOL_@") == 0 &&
+ if (isymbol.compare("@_UNKNOWN_SYMBOL_@") == 0 &&
osymbol.compare("@_IDENTITY_SYMBOL_@") == 0) {
isymbol = std::string("@_IDENTITY_SYMBOL_@");
osymbol = std::string("@_IDENTITY_SYMBOL_@");
@@ -3936,7 +3842,7 @@ HfstTransducer &HfstTransducer::merge
std::set<std::string> markers_added;
HfstBasicTransducer result = hfst::implementations::HfstBasicTransducer::merge(this_basic, another_basic, args.list_definitions, markers_added);
HfstTransducer initial_merge(result, this->get_type());
- initial_merge.minimize();
+ initial_merge.optimize();
// filter non-optimal paths
// [ ? | #V ?:? ]* %#V:V ?:0 [ ? | #V ?:? | %#V:V ?:0 ]*
@@ -3951,12 +3857,12 @@ HfstTransducer &HfstTransducer::merge
HfstTransducer * worsener = xre_.compile(worsener_string);
assert(worsener != NULL);
- worsener->minimize();
+ worsener->optimize();
HfstTransducer cp(initial_merge);
- cp.compose(*worsener).output_project().minimize();
+ cp.compose(*worsener).output_project().optimize();
delete worsener;
- initial_merge.subtract(cp).minimize();
+ initial_merge.subtract(cp).optimize();
initial_merge.substitute(marker, internal_epsilon);
HfstBasicTransducer fsm(initial_merge);
@@ -3981,10 +3887,6 @@ HfstTransducer &HfstTransducer::compose
HfstTransducer * another_copy = new HfstTransducer(another);
- //if (this->type != another_copy->type) {
- // another_copy->convert(this->type);
- //}
-
/* If we want flag diacritcs to be handled in the same way as epsilons
in composition, we substitute output flags of first transducer with
epsilons and input flags of second transducer with epsilons. NOTE:
@@ -3996,13 +3898,14 @@ HfstTransducer &HfstTransducer::compose
already when set_flag_is_epsilon_in_composition() has been called */
if (flag_is_epsilon_in_composition && this->type != XFSM_TYPE)
{
- try
+ try
{
this->substitute(&substitute_output_flag_with_epsilon);
another_copy->substitute(&substitute_input_flag_with_epsilon);
}
catch (const char * msg)
{
+ (void)msg;
HFST_THROW(FlagDiacriticsAreNotIdentitiesException);
}
}
@@ -4032,9 +3935,9 @@ HfstTransducer &HfstTransducer::compose
{
if (this->type == XFSM_TYPE)
{
- diacritics_added_from_another_to_this
+ diacritics_added_from_another_to_this
= this->insert_missing_diacritics_to_alphabet_from(*another_copy);
- diacritics_added_from_this_to_another
+ diacritics_added_from_this_to_another
= another_copy->insert_missing_diacritics_to_alphabet_from(*this);
}
}
@@ -4080,7 +3983,7 @@ HfstTransducer &HfstTransducer::compose
SFST::Transducer * sfst_temp =
this->sfst_interface.compose
(implementation.sfst,another_copy->implementation.sfst);
- delete implementation.sfst;
+ sfst_interface.delete_transducer(implementation.sfst);
implementation.sfst = sfst_temp;
break;
}
@@ -4092,7 +3995,7 @@ HfstTransducer &HfstTransducer::compose
this->tropical_ofst_interface.compose
(this->implementation.tropical_ofst,
another_copy->implementation.tropical_ofst);
- delete implementation.tropical_ofst;
+ tropical_ofst_interface.delete_transducer(implementation.tropical_ofst);
implementation.tropical_ofst = tropical_ofst_temp;
break;
}
@@ -4103,7 +4006,7 @@ HfstTransducer &HfstTransducer::compose
this->log_ofst_interface.compose
(implementation.log_ofst,
another_copy->implementation.log_ofst);
- delete implementation.log_ofst;
+ log_ofst_interface.delete_transducer(implementation.log_ofst);
implementation.log_ofst = log_ofst_temp;
break;
}
@@ -4123,8 +4026,8 @@ HfstTransducer &HfstTransducer::compose
#if HAVE_XFSM
case XFSM_TYPE:
{
- this->implementation.xfsm =
- this->xfsm_interface.compose(this->implementation.xfsm,
+ this->implementation.xfsm =
+ this->xfsm_interface.compose(this->implementation.xfsm,
another_copy->implementation.xfsm);
break;
}
@@ -4165,7 +4068,7 @@ HfstTransducer &HfstTransducer::compose
this->substitute(&substitute_one_sided_flags);
}
- if ( (this->type != FOMA_TYPE && this->type != XFSM_TYPE) && unknown_symbols_in_use)
+ if ( (this->type != FOMA_TYPE && this->type != XFSM_TYPE) && unknown_symbols_in_use)
{
this->substitute(&substitute_single_identity_with_the_other_symbol);
(const_cast<HfstTransducer*>(another_copy))->
@@ -4257,7 +4160,7 @@ HfstTransducer get_flag_path_restriction(const StringSet &_1_flags,
}
HfstTransducer &HfstTransducer::remove_illegal_flag_paths(void)
-{
+{
StringSet alphabet = this->get_alphabet();
StringSet _1_flags;
StringSet _2_flags;
@@ -4271,12 +4174,12 @@ HfstTransducer &HfstTransducer::remove_illegal_flag_paths(void)
{ continue; }
if (it->find("_1.") != std::string::npos)
- {
+ {
_1_flags.insert(*it);
}
if (it->find("_2.") != std::string::npos)
- {
+ {
_2_flags.insert(*it);
}
}
@@ -4345,7 +4248,7 @@ HfstTransducer &HfstTransducer::lenient_composition( const HfstTransducer &anoth
HfstTransducer retval(*this);
// true is a dummy variable, false means do not encode epsilons
- retval.compose(another).minimize().priority_union(*this).minimize();
+ retval.compose(another).optimize().priority_union(*this).optimize();
*this = retval;
return *this;
@@ -4400,19 +4303,19 @@ HfstTransducer &HfstTransducer::cross_product( const HfstTransducer &another, bo
HfstTransducer MarkToEpsilon(EpsilonToMark);
MarkToEpsilon.invert();
- UnknownToMark.repeat_star().minimize();
- EpsilonToMark.repeat_star().minimize();
- MarkToUnknown.repeat_star().minimize();
- MarkToEpsilon.repeat_star().minimize();
+ UnknownToMark.repeat_star().minimize(); // minimization is safe
+ EpsilonToMark.repeat_star().minimize(); // minimization is safe
+ MarkToUnknown.repeat_star().minimize(); // minimization is safe
+ MarkToEpsilon.repeat_star().minimize(); // minimization is safe
HfstTransducer a1(automata1);
- a1.compose(UnknownToMark).minimize().concatenate(EpsilonToMark).minimize();
+ a1.compose(UnknownToMark).optimize().concatenate(EpsilonToMark).optimize();
HfstTransducer b1(MarkToUnknown);
- b1.compose(automata2).minimize().concatenate(MarkToEpsilon).minimize();
+ b1.compose(automata2).optimize().concatenate(MarkToEpsilon).optimize();
HfstTransducer retval(a1);
- retval.compose(b1).minimize();
+ retval.compose(b1).optimize();
// Expand ?:? transitions to ?:?|?
StringPairSet id_or_unk;
@@ -4433,10 +4336,10 @@ HfstTransducer &HfstTransducer::cross_product( const HfstTransducer &another, bo
// A flag to indicate that there was an error during shuffle.
static bool shuffle_failed=false;
-// Possible cases for function code_symbols_for_shuffle.
-enum ShuffleCoding { ENCODE_FIRST_SHUFFLE_ARGUMENT,
- ENCODE_SECOND_SHUFFLE_ARGUMENT,
- DECODE_AFTER_SHUFFLE };
+// Possible cases for function code_symbols_for_shuffle.
+enum ShuffleCoding { ENCODE_FIRST_SHUFFLE_ARGUMENT,
+ ENCODE_SECOND_SHUFFLE_ARGUMENT,
+ DECODE_AFTER_SHUFFLE };
// The current case in function code_symbols_for_shuffle.
static ShuffleCoding shuffle_coding_case;
@@ -4459,7 +4362,7 @@ bool code_symbols_for_shuffle(const StringPair &sp, StringPairSet &sps)
}
switch (shuffle_coding_case)
{
- // substitute each symbol foo in the first argument transducer
+ // substitute each symbol foo in the first argument transducer
// with a symbol @1foo
case ENCODE_FIRST_SHUFFLE_ARGUMENT:
{
@@ -4467,7 +4370,7 @@ bool code_symbols_for_shuffle(const StringPair &sp, StringPairSet &sps)
StringPair new_sp(symbol_escaped, symbol_escaped);
sps.insert(new_sp);
break;
- }
+ }
// substitute each symbol bar in the second argument transducer
// with a symbol @2bar
case ENCODE_SECOND_SHUFFLE_ARGUMENT:
@@ -4531,7 +4434,7 @@ HfstTransducer &HfstTransducer::shuffle(const HfstTransducer &another, bool)
shuffle_failed=false;
HFST_THROW_MESSAGE(TransducersAreNotAutomataException,
"HfstTransducer::shuffle(const HfstTransducer&)");
- }
+ }
// The new alphabets of transducers where each symbol is prefixed
// with "@1" or "@2"
@@ -4561,7 +4464,7 @@ HfstTransducer &HfstTransducer::shuffle(const HfstTransducer &another, bool)
HfstTransducer another1(another_basic, another.get_type());
this1.intersect(another1);
- this1.minimize();
+ this1.optimize();
// We use HfstBasicTransducers again
HfstBasicTransducer this1_basic(this1);
@@ -4594,23 +4497,23 @@ HfstTransducer &HfstTransducer::priority_union (const HfstTransducer &another)
#endif
if ( this->type != another.type )
{
- HFST_THROW_MESSAGE(HfstTransducerTypeMismatchException,
+ HFST_THROW_MESSAGE(HfstTransducerTypeMismatchException,
"HfstTransducer::priority_union");
}
HfstTransducer t1(*this);
HfstTransducer t2(another);
HfstTransducer t1upper(t1);
- t1upper.input_project().minimize();
+ t1upper.input_project().optimize();
HfstTransducer complement = HfstTransducer::identity_pair( this->type );
- complement.repeat_star().minimize();
+ complement.repeat_star().optimize();
complement.subtract(t1upper).prune_alphabet(false);
- complement.compose(t2).minimize();
+ complement.compose(t2).optimize();
HfstTransducer retval(t1);
- retval.disjunct(complement).minimize();
+ retval.disjunct(complement).optimize();
*this = retval;
return *this;
@@ -4627,8 +4530,8 @@ HfstTransducer &HfstTransducer::compose_intersect
// transducers, convert to openfst type.
bool convert_to_openfst = false;
if (get_type() == FOMA_TYPE)
- {
- convert_to_openfst = true;
+ {
+ convert_to_openfst = true;
this->convert(TROPICAL_OPENFST_TYPE);
}
@@ -4639,12 +4542,12 @@ HfstTransducer &HfstTransducer::compose_intersect
const HfstTransducer &first = *v.begin();
- // If rule transducers contain word boundaries, add word boundaries to
- // the lexicon unless the lexicon already contains them.
+ // If rule transducers contain word boundaries, add word boundaries to
+ // the lexicon unless the lexicon already contains them.
std::set<std::string> rule_alphabet = first.get_alphabet();
if (rule_alphabet.find("@#@") != rule_alphabet.end())
- {
+ {
std::set<std::string> lexicon_alphabet = get_alphabet();
HfstTokenizer tokenizer;
tokenizer.add_multichar_symbol("@#@");
@@ -4658,7 +4561,7 @@ HfstTransducer &HfstTransducer::compose_intersect
basic_this.add_symbol_to_alphabet("@#@");
*this = HfstTransducer(basic_this,this->get_type());
- wb.concatenate(*this).concatenate(wb_copy).minimize();
+ wb.concatenate(*this).concatenate(wb_copy).optimize();
*this = wb;
}
@@ -4673,8 +4576,8 @@ HfstTransducer &HfstTransducer::compose_intersect
{ harmonized_lexicon = new HfstTransducer(*this); }
if (invert)
- {
- harmonized_lexicon->invert();
+ {
+ harmonized_lexicon->invert();
harmonized_lexicon->substitute(StringPair("@#@",internal_epsilon),
StringPair(internal_epsilon,"@#@"));
}
@@ -4682,27 +4585,27 @@ HfstTransducer &HfstTransducer::compose_intersect
harmonized_lexicon->substitute(internal_identity,"||_IDENTITY_SYMBOL_||");
harmonized_lexicon->substitute(internal_unknown,"||_UNKNOWN_SYMBOL_||");
- if (v.size() == 1)
+ if (v.size() == 1)
{
HfstTransducer rule_fst = v.at(0);
if (convert_to_openfst)
{ rule_fst.convert(TROPICAL_OPENFST_TYPE); }
if (invert)
- {
- rule_fst.invert();
+ {
+ rule_fst.invert();
rule_fst.substitute(StringPair(internal_epsilon,"@#@"),
StringPair("@#@",internal_epsilon));
}
// In case there is only onw rule, compose with that.
implementations::ComposeIntersectRule rule(rule_fst);
- // Create a ComposeIntersectLexicon from *this.
+ // Create a ComposeIntersectLexicon from *this.
//implementations::ComposeIntersectLexicon lexicon(*this);
implementations::ComposeIntersectLexicon lexicon(*harmonized_lexicon);
- hfst::implementations::HfstBasicTransducer res =
+ hfst::implementations::HfstBasicTransducer res =
lexicon.compose_with_rules(&rule);
res.prune_alphabet();
@@ -4711,7 +4614,7 @@ HfstTransducer &HfstTransducer::compose_intersect
else
{
- // In case there are many rules, build a ComposeIntersectRulePair
+ // In case there are many rules, build a ComposeIntersectRulePair
// recursively and compose with that.
HfstTransducer first_rule_fst = v.at(0);
@@ -4719,8 +4622,8 @@ HfstTransducer &HfstTransducer::compose_intersect
{ first_rule_fst.convert(TROPICAL_OPENFST_TYPE); }
if (invert)
- {
- first_rule_fst.invert();
+ {
+ first_rule_fst.invert();
first_rule_fst.substitute(StringPair(internal_epsilon,"@#@"),
StringPair("@#@",internal_epsilon));
}
@@ -4730,43 +4633,43 @@ HfstTransducer &HfstTransducer::compose_intersect
{ second_rule_fst.convert(TROPICAL_OPENFST_TYPE); }
if (invert)
- {
- second_rule_fst.invert();
+ {
+ second_rule_fst.invert();
second_rule_fst.substitute(StringPair(internal_epsilon,"@#@"),
StringPair("@#@",internal_epsilon));
}
std::vector<implementations::ComposeIntersectRule*> rule_vector;
- implementations::ComposeIntersectRule * first_rule =
+ implementations::ComposeIntersectRule * first_rule =
new implementations::ComposeIntersectRule(first_rule_fst);
- implementations::ComposeIntersectRule * second_rule =
+ implementations::ComposeIntersectRule * second_rule =
new implementations::ComposeIntersectRule(second_rule_fst);
- implementations::ComposeIntersectRulePair * rules =
+ implementations::ComposeIntersectRulePair * rules =
new implementations::ComposeIntersectRulePair
(first_rule,second_rule);
for (HfstTransducerVector::const_iterator it = v.begin() + 2;
it != v.end();
++it)
- {
+ {
HfstTransducer rule_fst(*it);
if (convert_to_openfst)
{ rule_fst.convert(TROPICAL_OPENFST_TYPE); }
if (invert)
- {
- rule_fst.invert();
+ {
+ rule_fst.invert();
rule_fst.substitute(StringPair(internal_epsilon,"@#@"),
StringPair("@#@",internal_epsilon));
}
rules = new implementations::ComposeIntersectRulePair
- (new implementations::ComposeIntersectRule(rule_fst),rules);
+ (new implementations::ComposeIntersectRule(rule_fst),rules);
}
- // Create a ComposeIntersectLexicon from *this.
+ // Create a ComposeIntersectLexicon from *this.
implementations::ComposeIntersectLexicon lexicon(*harmonized_lexicon);
- hfst::implementations::HfstBasicTransducer res =
+ hfst::implementations::HfstBasicTransducer res =
lexicon.compose_with_rules(rules);
res.prune_alphabet();
@@ -4844,7 +4747,7 @@ HfstTransducer &HfstTransducer::disjunct(const StringPairVector &spv)
default:
assert(false);
}
- return *this;
+ return *this;
}
// TODO...
@@ -4876,7 +4779,7 @@ HfstTransducer &HfstTransducer::disjunct_as_tries(HfstTransducer &another,
default:
assert(false);
}
- return *this;
+ return *this;
}
HfstTransducer &HfstTransducer::disjunct
@@ -4959,7 +4862,7 @@ get_basic_transducer() const
#if HAVE_SFST
if (this->type == SFST_TYPE)
{
- hfst::implementations::HfstBasicTransducer * net =
+ hfst::implementations::HfstBasicTransducer * net =
ConversionFunctions::sfst_to_hfst_basic_transducer
(implementation.sfst);
return net;
@@ -4968,7 +4871,7 @@ get_basic_transducer() const
#if HAVE_OPENFST
if (this->type == TROPICAL_OPENFST_TYPE)
{
- hfst::implementations::HfstBasicTransducer * net =
+ hfst::implementations::HfstBasicTransducer * net =
ConversionFunctions::tropical_ofst_to_hfst_basic_transducer
(implementation.tropical_ofst);
return net;
@@ -4976,7 +4879,7 @@ get_basic_transducer() const
#if HAVE_OPENFST_LOG
if (this->type == LOG_OPENFST_TYPE)
{
- hfst::implementations::HfstBasicTransducer * net =
+ hfst::implementations::HfstBasicTransducer * net =
ConversionFunctions::log_ofst_to_hfst_basic_transducer
(implementation.log_ofst);
return net;
@@ -4986,7 +4889,7 @@ get_basic_transducer() const
#if HAVE_FOMA
if (this->type == FOMA_TYPE)
{
- hfst::implementations::HfstBasicTransducer * net =
+ hfst::implementations::HfstBasicTransducer * net =
ConversionFunctions::foma_to_hfst_basic_transducer
(implementation.foma);
return net;
@@ -5005,29 +4908,29 @@ convert_to_basic_transducer()
#if HAVE_SFST
if (this->type == SFST_TYPE)
{
- hfst::implementations::HfstBasicTransducer * net =
+ hfst::implementations::HfstBasicTransducer * net =
ConversionFunctions::sfst_to_hfst_basic_transducer
(implementation.sfst);
- delete implementation.sfst;
+ sfst_interface.delete_transducer(implementation.sfst);
return net;
}
#endif
#if HAVE_OPENFST
if (this->type == TROPICAL_OPENFST_TYPE)
{
- hfst::implementations::HfstBasicTransducer * net =
+ hfst::implementations::HfstBasicTransducer * net =
ConversionFunctions::tropical_ofst_to_hfst_basic_transducer
(implementation.tropical_ofst);
- delete implementation.tropical_ofst;
+ tropical_ofst_interface.delete_transducer(implementation.tropical_ofst);
return net;
}
#if HAVE_OPENFST_LOG
if (this->type == LOG_OPENFST_TYPE)
{
- hfst::implementations::HfstBasicTransducer * net =
+ hfst::implementations::HfstBasicTransducer * net =
ConversionFunctions::log_ofst_to_hfst_basic_transducer
(implementation.log_ofst);
- delete implementation.log_ofst;
+ log_ofst_interface.delete_transducer(implementation.log_ofst);
return net;
}
#endif
@@ -5035,10 +4938,10 @@ convert_to_basic_transducer()
#if HAVE_FOMA
if (this->type == FOMA_TYPE)
{
- hfst::implementations::HfstBasicTransducer * net =
+ hfst::implementations::HfstBasicTransducer * net =
ConversionFunctions::foma_to_hfst_basic_transducer
(implementation.foma);
- free(implementation.foma);
+ foma_interface.delete_foma(implementation.foma);
return net;
}
#endif
@@ -5055,8 +4958,9 @@ convert_to_hfst_transducer(implementations::HfstBasicTransducer *t)
#if HAVE_SFST
if (this->type == SFST_TYPE)
{
- implementation.sfst =
+ implementation.sfst =
ConversionFunctions::hfst_basic_transducer_to_sfst(t);
+ name = t->name;
delete t;
return *this;
}
@@ -5064,16 +4968,18 @@ convert_to_hfst_transducer(implementations::HfstBasicTransducer *t)
#if HAVE_OPENFST
if (this->type == TROPICAL_OPENFST_TYPE)
{
- implementation.tropical_ofst =
+ implementation.tropical_ofst =
ConversionFunctions::hfst_basic_transducer_to_tropical_ofst(t);
+ name = t->name;
delete t;
return *this;
}
#if HAVE_OPENFST_LOG
if (this->type == LOG_OPENFST_TYPE)
{
- implementation.log_ofst =
+ implementation.log_ofst =
ConversionFunctions::hfst_basic_transducer_to_log_ofst(t);
+ name = t->name;
delete t;
return *this;
}
@@ -5082,8 +4988,9 @@ convert_to_hfst_transducer(implementations::HfstBasicTransducer *t)
#if HAVE_FOMA
if (this->type == FOMA_TYPE)
{
- implementation.foma =
+ implementation.foma =
ConversionFunctions::hfst_basic_transducer_to_foma(t);
+ name = t->name;
delete t;
return *this;
}
@@ -5098,27 +5005,25 @@ HfstTransducer &HfstTransducer::convert(const HfstTransducer &t,
ImplementationType type)
{
if (type == ERROR_TYPE)
- {
+ {
HFST_THROW_MESSAGE
(SpecifiedTypeRequiredException, "HfstTransducer::convert");
}
if (type == t.type)
{ return *(new HfstTransducer(t)); }
if (! is_implementation_type_available(type)) {
- HFST_THROW_MESSAGE
- (ImplementationTypeNotAvailableException,
- "HfstTransducer::convert");
+ throw ImplementationTypeNotAvailableException("HfstTransducer::convert", __FILE__, __LINE__, type);
}
- hfst::implementations::HfstBasicTransducer net(t);
+ hfst::implementations::HfstBasicTransducer net(t);
- HfstTransducer * retval = new HfstTransducer(net, type);
+ HfstTransducer * retval = new HfstTransducer(net, type);
return *retval;
}
-/* ERROR_TYPE or UNSPECIFIED_TYPE returns true, so they must be handled
+/* ERROR_TYPE or UNSPECIFIED_TYPE returns true, so they must be handled
separately */
bool HfstTransducer::is_implementation_type_available
(ImplementationType type) {
@@ -5143,7 +5048,7 @@ bool HfstTransducer::is_implementation_type_available
//if (type == MY_TRANSDUCER_LIBRARY_TYPE)
// return false;
//#endif
- (void)type;
+ (void)type;
return true;
}
@@ -5157,14 +5062,13 @@ HfstTransducer &HfstTransducer::convert(ImplementationType type,
}
if (type == ERROR_TYPE)
- {
+ {
HFST_THROW_MESSAGE(SpecifiedTypeRequiredException,
"HfstTransducer::convert"); }
if (type == this->type)
{ return *this; }
if (! is_implementation_type_available(type)) {
- HFST_THROW_MESSAGE(ImplementationTypeNotAvailableException,
- "HfstTransducer::convert");
+ throw ImplementationTypeNotAvailableException("HfstTransducer::convert", __FILE__, __LINE__, type);
}
hfst::implementations::HfstBasicTransducer * internal=NULL;
@@ -5198,10 +5102,10 @@ HfstTransducer &HfstTransducer::convert(ImplementationType type,
//#endif
#if HAVE_SFST
case SFST_TYPE:
- internal =
+ internal =
ConversionFunctions::sfst_to_hfst_basic_transducer
(implementation.sfst);
- delete implementation.sfst;
+ sfst_interface.delete_transducer(implementation.sfst);
break;
#endif
#if HAVE_OPENFST
@@ -5210,14 +5114,14 @@ HfstTransducer &HfstTransducer::convert(ImplementationType type,
ConversionFunctions::tropical_ofst_to_hfst_basic_transducer
(implementation.tropical_ofst);
assert(internal != NULL);
- delete implementation.tropical_ofst;
+ tropical_ofst_interface.delete_transducer(implementation.tropical_ofst);
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
internal =
ConversionFunctions::log_ofst_to_hfst_basic_transducer
(implementation.log_ofst);
- delete implementation.log_ofst;
+ log_ofst_interface.delete_transducer(implementation.log_ofst);
break;
#endif
case HFST_OL_TYPE:
@@ -5230,6 +5134,7 @@ HfstTransducer &HfstTransducer::convert(ImplementationType type,
#endif
case ERROR_TYPE:
default:
+ (void)internal;
HFST_THROW(TransducerHasWrongTypeException);
break;
}
@@ -5238,7 +5143,7 @@ HfstTransducer &HfstTransducer::convert(ImplementationType type,
{
#if HAVE_SFST
case SFST_TYPE:
- implementation.sfst =
+ implementation.sfst =
ConversionFunctions::hfst_basic_transducer_to_sfst(internal);
delete internal;
break;
@@ -5246,7 +5151,7 @@ HfstTransducer &HfstTransducer::convert(ImplementationType type,
/* Add here your implementation. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//case MY_TRANSDUCER_LIBRARY_TYPE:
- //implementation.my_transducer_library =
+ //implementation.my_transducer_library =
// ConversionFunctions::
// hfst_basic_transducer_to_my_transducer_library_transducer
// (internal);
@@ -5269,7 +5174,7 @@ HfstTransducer &HfstTransducer::convert(ImplementationType type,
#endif
case HFST_OL_TYPE:
case HFST_OLW_TYPE:
- implementation.hfst_ol =
+ implementation.hfst_ol =
ConversionFunctions::hfst_basic_transducer_to_hfst_ol
(internal, this->type==HFST_OLW_TYPE?true:false, options);
delete internal;
@@ -5300,7 +5205,7 @@ HfstTransducer &HfstTransducer::convert(ImplementationType type,
void HfstTransducer::write_in_att_format
(const std::string &filename, bool print_weights) const
{
- FILE * ofile = fopen(filename.c_str(), "wb");
+ FILE * ofile = hfst::hfst_fopen(filename.c_str(), "wb");
if (ofile == NULL) {
std::string message(filename);
HFST_THROW_MESSAGE(StreamCannotBeWrittenException, message);
@@ -5353,6 +5258,7 @@ void HfstTransducer::write_xfsm_transducer_in_att_format(const char * filename)
hfst::implementations::XfsmTransducer::write_in_att_format
(const_cast<NETptr>(this->implementation.xfsm), filename);
#endif
+ (void)filename;
}
/* Implemented only for XFSM_TYPE. */
@@ -5364,6 +5270,7 @@ void HfstTransducer::write_xfsm_transducer_in_prolog_format(const char * filenam
hfst::implementations::XfsmTransducer::write_in_prolog_format
(const_cast<NETptr>(this->implementation.xfsm), filename);
#endif
+ (void)filename;
}
void HfstTransducer::write_in_prolog_format(FILE * file, const std::string & name,
@@ -5383,11 +5290,12 @@ HfstTransducer * HfstTransducer::prolog_file_to_xfsm_transducer(const char * fil
retval->implementation.xfsm = XfsmTransducer::prolog_file_to_xfsm_transducer(filename);
return retval;
#else
+ (void)filename;
HFST_THROW(FunctionNotImplementedException);
#endif
}
-HfstTransducer::HfstTransducer(FILE * ifile,
+HfstTransducer::HfstTransducer(FILE * ifile,
ImplementationType type,
const std::string &epsilon_symbol):
type(type),anonymous(false),is_trie(false), name("")
@@ -5399,61 +5307,59 @@ HfstTransducer::HfstTransducer(FILE * ifile,
unsigned int linecount=0;
if (! is_implementation_type_available(type))
- HFST_THROW_MESSAGE(ImplementationTypeNotAvailableException,
- "HfstTransducer::HfstTransducer"
- "(FILE*, ImplementationType, const std::string&)");
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
HfstTokenizer::check_utf8_correctness(epsilon_symbol);
// Implemented only for internal transducer format.
hfst::implementations::HfstBasicTransducer net =
- hfst::implementations::HfstTransitionGraph<hfst::implementations::
- HfstTropicalTransducerTransitionData>::
- read_in_att_format(ifile, std::string(epsilon_symbol), linecount);
+ /*hfst::implementations::HfstTransitionGraph<hfst::implementations::
+ HfstTropicalTransducerTransitionData>::*/
+ HfstBasicTransducer::read_in_att_format(ifile, std::string(epsilon_symbol), linecount);
// Conversion is done here.
switch (type)
{
#if HAVE_SFST
case SFST_TYPE:
- implementation.sfst =
+ implementation.sfst =
ConversionFunctions::hfst_basic_transducer_to_sfst(&net);
break;
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
- implementation.tropical_ofst
+ implementation.tropical_ofst
= ConversionFunctions::hfst_basic_transducer_to_tropical_ofst(&net);
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
- implementation.log_ofst
+ implementation.log_ofst
= ConversionFunctions::hfst_basic_transducer_to_log_ofst(&net);
break;
#endif
#endif
#if HAVE_FOMA
case FOMA_TYPE:
- implementation.foma =
+ implementation.foma =
ConversionFunctions::hfst_basic_transducer_to_foma(&net);
break;
#endif
#if HAVE_HFSTOL
case HFST_OL_TYPE:
- implementation.hfst_ol
+ implementation.hfst_ol
= ConversionFunctions::hfst_basic_transducer_to_hfst_ol
(&net, false);
break;
case HFST_OLW_TYPE:
- implementation.hfst_ol
+ implementation.hfst_ol
= ConversionFunctions::hfst_basic_transducer_to_hfst_ol(&net, true);
break;
#endif
/* Add here your implementation. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//case MY_TRANSDUCER_LIBRARY_TYPE:
- //implementation.my_transducer_library =
+ //implementation.my_transducer_library =
// ConversionFunctions::
// hfst_basic_transducer_to_my_transducer_library_transducer(&net);
//break;
@@ -5465,7 +5371,7 @@ HfstTransducer::HfstTransducer(FILE * ifile,
}
}
-HfstTransducer::HfstTransducer(FILE * ifile,
+HfstTransducer::HfstTransducer(FILE * ifile,
ImplementationType type,
const std::string &epsilon_symbol,
unsigned int & linecount):
@@ -5477,60 +5383,58 @@ HfstTransducer::HfstTransducer(FILE * ifile,
#endif
if (! is_implementation_type_available(type))
- HFST_THROW_MESSAGE(ImplementationTypeNotAvailableException,
- "HfstTransducer::HfstTransducer"
- "(FILE*, ImplementationType, const std::string&)");
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
HfstTokenizer::check_utf8_correctness(epsilon_symbol);
// Implemented only for internal transducer format.
hfst::implementations::HfstBasicTransducer net =
- hfst::implementations::HfstTransitionGraph<hfst::implementations::
- HfstTropicalTransducerTransitionData>::
- read_in_att_format(ifile, std::string(epsilon_symbol), linecount);
+ /*hfst::implementations::HfstTransitionGraph<hfst::implementations::
+ HfstTropicalTransducerTransitionData>::*/
+ HfstBasicTransducer::read_in_att_format(ifile, std::string(epsilon_symbol), linecount);
// Conversion is done here.
switch (type)
{
#if HAVE_SFST
case SFST_TYPE:
- implementation.sfst =
+ implementation.sfst =
ConversionFunctions::hfst_basic_transducer_to_sfst(&net);
break;
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
- implementation.tropical_ofst
- = ConversionFunctions::hfst_basic_transducer_to_tropical_ofst(&net);
+ implementation.tropical_ofst
+ = ConversionFunctions::hfst_basic_transducer_to_tropical_ofst(&net);
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
- implementation.log_ofst
+ implementation.log_ofst
= ConversionFunctions::hfst_basic_transducer_to_log_ofst(&net);
break;
#endif
#endif
#if HAVE_FOMA
case FOMA_TYPE:
- implementation.foma =
+ implementation.foma =
ConversionFunctions::hfst_basic_transducer_to_foma(&net);
break;
#endif
#if HAVE_HFSTOL
case HFST_OL_TYPE:
- implementation.hfst_ol
+ implementation.hfst_ol
= ConversionFunctions::hfst_basic_transducer_to_hfst_ol
(&net, false);
break;
case HFST_OLW_TYPE:
- implementation.hfst_ol
+ implementation.hfst_ol
= ConversionFunctions::hfst_basic_transducer_to_hfst_ol(&net, true);
break;
#endif
/* Add here your implementation. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//case MY_TRANSDUCER_LIBRARY_TYPE:
- //implementation.my_transducer_library =
+ //implementation.my_transducer_library =
// ConversionFunctions::
// hfst_basic_transducer_to_my_transducer_library_transducer(&net);
//break;
@@ -5545,12 +5449,12 @@ HfstTransducer::HfstTransducer(FILE * ifile,
HfstTransducer &HfstTransducer::read_in_att_format
-(const std::string &filename, ImplementationType type,
+(const std::string &filename, ImplementationType type,
const std::string &epsilon_symbol)
{
if (type == XFSM_TYPE)
{ HFST_THROW(FunctionNotImplementedException); }
- FILE * ifile = fopen(filename.c_str(), "rb");
+ FILE * ifile = hfst::hfst_fopen(filename.c_str(), "rb");
if (ifile == NULL) {
std::string message(filename);
HFST_THROW_MESSAGE(StreamNotReadableException, message);
@@ -5569,16 +5473,15 @@ HfstTransducer &HfstTransducer::read_in_att_format
{ HFST_THROW(FunctionNotImplementedException); }
if (! is_implementation_type_available(type))
- HFST_THROW_MESSAGE(ImplementationTypeNotAvailableException,
- "HfstTransducer::read_in_att_format");
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
HfstTokenizer::check_utf8_correctness(epsilon_symbol);
unsigned int foo = 0;
hfst::implementations::HfstBasicTransducer net =
- hfst::implementations::HfstTransitionGraph<hfst::implementations::
- HfstTropicalTransducerTransitionData>
- ::read_in_att_format(ifile, std::string(epsilon_symbol), foo);
+ /*hfst::implementations::HfstTransitionGraph<hfst::implementations::
+ HfstTropicalTransducerTransitionData>*/
+ HfstBasicTransducer::read_in_att_format(ifile, std::string(epsilon_symbol), foo);
HfstTransducer *retval = new HfstTransducer(net,type);
(void)foo;
return *retval;
@@ -5588,7 +5491,7 @@ HfstTransducer &HfstTransducer::read_in_att_format
// -----------------------------------------------------------------------
//
-// Miscellaneous functions
+// Miscellaneous functions
//
// -----------------------------------------------------------------------
@@ -5635,7 +5538,7 @@ HfstTransducer &HfstTransducer::operator=(const HfstTransducer &another)
if (&another == this)
{ return *this; }
- if (this->type != UNSPECIFIED_TYPE &&
+ if (this->type != UNSPECIFIED_TYPE &&
this->type != another.type) {
HFST_THROW_MESSAGE
(TransducerTypeMismatchException, "HfstTransducer::operator=");
@@ -5651,28 +5554,26 @@ HfstTransducer &HfstTransducer::operator=(const HfstTransducer &another)
{
#if HAVE_FOMA
case FOMA_TYPE:
- free(implementation.foma);
+ foma_interface.delete_foma(implementation.foma);
break;
#endif
#if HAVE_SFST
case SFST_TYPE:
- delete implementation.sfst;
+ sfst_interface.delete_transducer(implementation.sfst);
break;
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
- delete implementation.tropical_ofst;
+ tropical_ofst_interface.delete_transducer(implementation.tropical_ofst);
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
- delete implementation.log_ofst;
+ log_ofst_interface.delete_transducer(implementation.log_ofst);
break;
#endif
#endif
case HFST_OL_TYPE:
case HFST_OLW_TYPE:
- //HFST_THROW_MESSAGE(FunctionNotImplementedException,
- // "HfstTransducer::operator= for type HFST_OL(W)_TYPE");
delete implementation.hfst_ol;
break;
/* Add here your implementation. */
@@ -5698,35 +5599,35 @@ HfstTransducer &HfstTransducer::operator=(const HfstTransducer &another)
{
#if HAVE_FOMA
case FOMA_TYPE:
- implementation.foma =
+ implementation.foma =
foma_interface.copy(another_1.implementation.foma);
break;
#endif
#if HAVE_SFST
case SFST_TYPE:
- implementation.sfst =
+ implementation.sfst =
sfst_interface.copy(another_1.implementation.sfst);
break;
#endif
#if HAVE_OPENFST
case TROPICAL_OPENFST_TYPE:
- implementation.tropical_ofst =
+ implementation.tropical_ofst =
tropical_ofst_interface.copy(another_1.implementation.tropical_ofst);
break;
#if HAVE_OPENFST_LOG
case LOG_OPENFST_TYPE:
- implementation.log_ofst =
+ implementation.log_ofst =
log_ofst_interface.copy(another_1.implementation.log_ofst);
break;
#endif
#endif
case HFST_OL_TYPE:
- implementation.hfst_ol
+ implementation.hfst_ol
= another_1.implementation.hfst_ol->
copy(another_1.implementation.hfst_ol, false);
break;
case HFST_OLW_TYPE:
- implementation.hfst_ol
+ implementation.hfst_ol
= another_1.implementation.hfst_ol->
copy(another_1.implementation.hfst_ol, true);
break;
@@ -5739,11 +5640,11 @@ HfstTransducer &HfstTransducer::operator=(const HfstTransducer &another)
}
-HfstTokenizer HfstTransducer::create_tokenizer()
+HfstTokenizer HfstTransducer::create_tokenizer()
{
HfstTokenizer tok;
- if (this->type == SFST_TYPE)
+ if (this->type == SFST_TYPE)
{
StringPairSet sps = this->get_symbol_pairs();
for (StringPairSet::const_iterator it = sps.begin();
@@ -5755,7 +5656,7 @@ HfstTokenizer HfstTransducer::create_tokenizer()
tok.add_multichar_symbol(it->second);
}
}
- else
+ else
{
hfst::implementations::HfstBasicTransducer t(*this);
t.prune_alphabet();
@@ -5789,7 +5690,7 @@ HfstTransducer * HfstTransducer::read_lexc_ptr(const std::string &filename,
(void)filename;
if (! is_implementation_type_available(type))
- HFST_THROW(ImplementationTypeNotAvailableException);
+ throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
HfstTransducer * retval = new HfstTransducer();
@@ -5818,25 +5719,6 @@ HfstTransducer * HfstTransducer::read_lexc_ptr(const std::string &filename,
compiler.setVerbosity(verbose);
compiler.parse(filename.c_str());
retval = compiler.compileLexical();
-
- /*
- std::map<std::string,hfst::HfstTransducer> stringTriesPrint_;
- stringTriesPrint_ = compiler.getStringTries();
-
- cout << "\n size: " << stringTriesPrint_.size() << endl;
-
-
- std::map<std::string,hfst::HfstTransducer>:: iterator it;
-
- for ( std::map<std::string,hfst::HfstTransducer>::iterator it=stringTriesPrint_.begin(); it!=stringTriesPrint_.end(); ++it )
- {
- std::cout << "string" << it->first << " => tr:" << '\n';
- it->second.minimize().write_in_att_format(stdout, 1);
- }
- */
-
-
-
return retval;
break;
}
@@ -6339,12 +6221,12 @@ int main(int argc, char * argv[])
{
std::cout << "Unit tests for " __FILE__ ":" << std::endl;
- ImplementationType types[] = {SFST_TYPE,
+ ImplementationType types[] = {SFST_TYPE,
TROPICAL_OPENFST_TYPE,
FOMA_TYPE};
unsigned int NUMBER_OF_TYPES=3;
- for (unsigned int i=0; i < NUMBER_OF_TYPES; i++)
+ for (unsigned int i=0; i < NUMBER_OF_TYPES; i++)
{
if (! HfstTransducer::is_implementation_type_available(types[i]))
continue;
@@ -6361,11 +6243,11 @@ int main(int argc, char * argv[])
assert(alpha.find("b") != alpha.end());
assert(alpha.find("c") != alpha.end());
// TODO: which is correct?
- //assert(alpha.find("a") != alpha.end());
+ //assert(alpha.find("a") != alpha.end());
HfstTransducer t1("a", "b", types[i]);
HfstTransducer t2("a", "c", types[i]);
- t1.substitute(StringPair("a", "b"), t2);
+ t1.substitute(StringPair("a", "b"), t2);
alpha = t1.get_alphabet();
assert(alpha.find("a") != alpha.end());
@@ -6435,7 +6317,7 @@ int main(int argc, char * argv[])
// Flag diacritic harmonization test
HfstTokenizer flag_tokenizer;
flag_tokenizer.add_multichar_symbol("@P.Char.ON@");
- flag_tokenizer.add_multichar_symbol("@R.Char.ON@");
+ flag_tokenizer.add_multichar_symbol("@R.Char.ON@");
HfstTransducer any_a("A",types[i]);
HfstTransducer any_b("B",types[i]);
@@ -6458,14 +6340,14 @@ int main(int argc, char * argv[])
a_paths_copy.convert(HFST_OLW_TYPE);
- HfstOneLevelPaths * results =
+ HfstOneLevelPaths * results =
a_paths_copy.lookup_fd(flag_tokenizer.tokenize_one_level("ABCBA"));
assert(results->size() == 1);
assert(hfst::symbols::remove_flags(results->begin()->second) ==
flag_tokenizer.tokenize_one_level("ABCBA"));
delete results;
- results =
+ results =
a_paths_copy.lookup_fd(flag_tokenizer.tokenize_one_level("ABCAA"));
assert(results->size() == 1);
assert(hfst::symbols::remove_flags(results->begin()->second) ==
@@ -6473,13 +6355,13 @@ int main(int argc, char * argv[])
delete results;
HfstTransducer b_paths(any_symbol);
- HfstTransducer b_paths_("B"
+ HfstTransducer b_paths_("B"
"@P.Char.ON@",
flag_tokenizer,
types[i]);
b_paths.concatenate(b_paths_);
b_paths.concatenate(any);
- HfstTransducer b_end("@R.Char.ON@" "B",
+ HfstTransducer b_end("@R.Char.ON@" "B",
flag_tokenizer,
types[i]);
b_end.concatenate(any_symbol);
@@ -6488,14 +6370,14 @@ int main(int argc, char * argv[])
HfstTransducer b_paths_copy(b_paths);
b_paths_copy.convert(HFST_OLW_TYPE);
- results =
+ results =
b_paths_copy.lookup_fd(flag_tokenizer.tokenize_one_level("ABCBA"));
assert(results->size() == 1);
assert(hfst::symbols::remove_flags(results->begin()->second) ==
flag_tokenizer.tokenize_one_level("ABCBA"));
delete results;
- results =
+ results =
b_paths_copy.lookup_fd(flag_tokenizer.tokenize_one_level("ABCBB"));
assert(results->size() == 1);
assert(hfst::symbols::remove_flags(results->begin()->second) ==
@@ -6508,24 +6390,24 @@ int main(int argc, char * argv[])
a_paths.convert(HFST_OLW_TYPE);
- HfstOneLevelPaths * one_result =
+ HfstOneLevelPaths * one_result =
a_paths.lookup_fd(flag_tokenizer.tokenize_one_level("ABCBA"));
assert(one_result->size() == 1);
assert(hfst::symbols::remove_flags(one_result->begin()->second) ==
flag_tokenizer.tokenize_one_level("ABCBA"));
delete one_result;
- HfstOneLevelPaths * no_results =
+ HfstOneLevelPaths * no_results =
a_paths.lookup_fd(flag_tokenizer.tokenize_one_level("ABCBB"));
assert(no_results->size() == 0);
delete no_results;
- no_results =
+ no_results =
a_paths.lookup_fd(flag_tokenizer.tokenize_one_level("ABCAA"));
assert(no_results->size() == 0);
delete no_results;
- no_results =
+ no_results =
a_paths.lookup_fd(flag_tokenizer.tokenize_one_level("ABCCC"));
assert(no_results->size() == 0);
delete no_results;
diff --git a/libhfst/src/HfstTransducer.h b/libhfst/src/HfstTransducer.h
index 2060856..fbf254d 100644
--- a/libhfst/src/HfstTransducer.h
+++ b/libhfst/src/HfstTransducer.h
@@ -18,7 +18,6 @@
#include "HfstDataTypes.h"
#include "HfstSymbolDefs.h"
-#include "implementations/HfstTransitionGraph.h"
#include "parsers/LexcCompiler.h"
#if HAVE_SFST
@@ -38,7 +37,7 @@
#include "implementations/XfsmTransducer.h"
#endif
-/* Include here the header file of the files that take care
+/* Include here the header file of the files that take care
of the interaction between HFST and your transducer library. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//#include "implementations/MyTransducerLibraryTransducer.h"
@@ -52,8 +51,7 @@
#include "HfstOutputStream.h"
#include <string>
-#include <cassert>
-#include <iostream>
+#include <iosfwd>
#include <vector>
#include <map>
#include <set>
@@ -61,20 +59,17 @@
#include "hfstdll.h"
/** @file HfstTransducer.h
- \brief Declarations of HFST API functions and datatypes.
+ \brief Declarations of HFST API functions and datatypes.
This file must be included to a program that uses the HFST API. */
/** \brief A namespace for HFST functions and datatypes. */
namespace hfst
{
- namespace implementations {
- template <class T> class HfstTransitionGraph;
- class HfstTropicalTransducerTransitionData;
- typedef HfstTransitionGraph<HfstTropicalTransducerTransitionData>
- HfstBasicTransducer;
+ namespace implementations {
+ class HfstBasicTransducer;
}
- class HfstCompiler;
+ class SfstCompiler;
class HfstTransducer;
namespace xfst {
class XfstCompiler;
@@ -93,8 +88,6 @@ namespace hfst
#if HAVE_OPENFST
using hfst::implementations::TropicalWeightTransducer;
- using hfst::implementations::TropicalWeightState;
- using hfst::implementations::TropicalWeightStateIterator;
#if HAVE_OPENFST_LOG
using hfst::implementations::LogWeightTransducer;
#endif // #if HAVE_OPENFST_LOG
@@ -126,36 +119,45 @@ namespace hfst
#endif
enum MinimizationAlgorithm { HOPCROFT, BRZOZOWSKI };
- /* Which minimization algorithm is used.
- In foma, Hopcroft is always used.
+ /* Which minimization algorithm is used.
+ In foma, Hopcroft is always used.
In OpenFst and SFST, the default algorithm is Hopcroft. */
HFSTDLL void set_minimization_algorithm(MinimizationAlgorithm);
- HFSTDLL MinimizationAlgorithm get_minimization_algorithm();
+ HFSTDLL MinimizationAlgorithm get_minimization_algorithm();
+ /* Whether weights are encoded as part of transition in weighted minimization. Defaults to false. */
HFSTDLL void set_encode_weights(bool);
HFSTDLL bool get_encode_weights();
+ /* Whether minimization is carried out even it the transducer is known to be minimal. Defaults to false. */
HFSTDLL void set_minimize_even_if_already_minimal(bool);
HFSTDLL bool get_minimize_even_if_already_minimal();
+ /* Allow full minimization of intermediary results instead of only removing epsilons and determinizing. Defaults to true. */
+ HFSTDLL void set_minimization(bool);
+ HFSTDLL bool get_minimization();
+
+ /* Whether flag diacritics are matched with unknowns and identities in composition. Defaults to false. */
HFSTDLL void set_xerox_composition(bool);
HFSTDLL bool get_xerox_composition();
+ /* Whether flag diacritics are treated as epsilons in composition. Defaults to false. */
HFSTDLL void set_flag_is_epsilon_in_composition(bool);
HFSTDLL bool get_flag_is_epsilon_in_composition();
/* Whether in harmonization the smaller transducer is always harmonized
- according to the bigger transducer. */
+ according to the bigger transducer. Defaults to true. */
HFSTDLL void set_harmonize_smaller(bool);
HFSTDLL bool get_harmonize_smaller();
/* Whether unknown and identity symbols are used. By default, they are used.
- These symbols are always reserved for use and included in alphabets
- of transducers, but code optimization is possible if it is known
+ These symbols are always reserved for use and included in alphabets
+ of transducers, but code optimization is possible if it is known
that they do not appear in transducer transitions. */
HFSTDLL void set_unknown_symbols_in_use(bool);
HFSTDLL bool get_unknown_symbols_in_use();
+ /* Where warnings from TropicalWeightTransducer are printed. */
void set_warning_stream(std::ostream * os);
std::ostream * get_warning_stream();
@@ -166,51 +168,51 @@ namespace hfst
\section argument_handling Argument handling
- Transducer functions modify their calling object and return
- a reference to the calling object after modification,
+ Transducer functions modify their calling object and return
+ a reference to the calling object after modification,
unless otherwise mentioned.
Transducer arguments are usually not modified.
\verbatim
// transducer is reversed
transducer.reverse();
// transducer2 is not modified, but a copy of it is disjuncted with
- // transducer1
- transducer1.disjunct(transducer2);
+ // transducer1
+ transducer1.disjunct(transducer2);
// a chain of functions is possible
- transducer.reverse().determinize().reverse().determinize();
+ transducer.reverse().determinize().reverse().determinize();
\endverbatim
\section implementation_types Implementation types
- Currently, an HfstTransducer has four implementation types as
+ Currently, an HfstTransducer has four implementation types as
defined by the enumeration ImplementationType.
- When an HfstTransducer is created, its type is defined with an
+ When an HfstTransducer is created, its type is defined with an
ImplementationType argument.
- For functions that take a transducer as an argument, the type of
+ For functions that take a transducer as an argument, the type of
the calling transducer
must be the same as the type of the argument transducer:
\verbatim
// this will cause an error
- log_transducer.disjunct(sfst_transducer);
+ log_transducer.disjunct(sfst_transducer);
// this works, but weights are lost in the conversion
- log_transducer.convert(SFST_TYPE).disjunct(sfst_transducer);
+ log_transducer.convert(SFST_TYPE).disjunct(sfst_transducer);
// this works, information is not lost
- log_transducer.disjunct(sfst_transducer.convert(LOG_OPENFST_TYPE));
+ log_transducer.disjunct(sfst_transducer.convert(LOG_OPENFST_TYPE));
\endverbatim
\section creating_transducers Creating transducers
- With HfstTransducer constructors it is possible to create empty,
+ With HfstTransducer constructors it is possible to create empty,
epsilon, one-transition and single-path transducers.
Transducers can also be created from scratch with HfstBasicTransducer
and converted to an HfstTransducer.
- More complex transducers can be combined from simple ones with various
+ More complex transducers can be combined from simple ones with various
functions.
- <a name="symbols"></a>
+ <a name="symbols"></a>
\section special_symbols Special symbols
- The HFST transducers support transitions with epsilon, unknown
+ The HFST transducers support transitions with epsilon, unknown
and identity symbols.
The special symbols are explained in documentation of datatype #String.
@@ -218,7 +220,7 @@ An example:
\verbatim
// In the xerox formalism used here, "?" means the unknown symbol
- // and "?:?" the identity pair
+ // and "?:?" the identity pair
HfstBasicTransducer tr1;
tr1.add_state(1);
@@ -233,7 +235,7 @@ An example:
tr2.add_state(2);
tr2.set_final_weight(2, 0);
tr2.add_transition
- (0, HfstBasicTransition(1, "@_IDENTITY_SYMBOL_@",
+ (0, HfstBasicTransition(1, "@_IDENTITY_SYMBOL_@",
"@_IDENTITY_SYMBOL_@", 0) );
tr2.add_transition
(1, HfstBasicTransition(2, "bar", "bar", 0) );
@@ -258,7 +260,7 @@ An example:
protected:
/* The backend implementation type of the transducer */
- ImplementationType type;
+ ImplementationType type;
bool anonymous; // currently not used
bool is_trie; // currently not used
@@ -290,9 +292,6 @@ An example:
hfst_ol::Transducer * hfst_ol;
-#if HAVE_OPENFST // is this needed?
- hfst::implementations::StdVectorFst * internal;
-#endif
};
/* The backend implementation */
@@ -303,7 +302,7 @@ An example:
static hfst::implementations::SfstTransducer sfst_interface;
#endif
#if HAVE_OPENFST
- static hfst::implementations::TropicalWeightTransducer
+ static hfst::implementations::TropicalWeightTransducer
tropical_ofst_interface;
#if HAVE_OPENFST_LOG
static hfst::implementations::LogWeightTransducer log_ofst_interface;
@@ -325,7 +324,7 @@ An example:
//#endif
/* The same as harmonize but does not change \a another but
- returns a harmonized version of that. If this and \a another
+ returns a harmonized version of that. If this and \a another
have type FOMA_TYPE, NULL is returned. */
HfstTransducer * harmonize_(const HfstTransducer &another);
@@ -333,7 +332,7 @@ An example:
/* Check if transducer \a another has in its alphabet flag diacritics
that are not found in the alphabet of this transducer and insert
- all missing flag diacritics to \a missing_flags.
+ all missing flag diacritics to \a missing_flags.
\a return_on_first_miss defines whether the function returns as soon as
a missing flag is found so that only that flag is inserted to
\a missing flags. */
@@ -345,10 +344,10 @@ An example:
a trie.
Currently not implemented, TODO */
HfstTransducer &disjunct_as_tries(HfstTransducer &another,
- ImplementationType type);
+ ImplementationType type);
- /* Remove paths where @..._2@ transitions immediately preceede
+ /* Remove paths where @..._2@ transitions immediately preceede
@..._1@ transitions, i.e. transitions resulting from incorrect
ordering of flag diacritics. */
HfstTransducer &remove_illegal_flag_paths(void);
@@ -359,12 +358,12 @@ An example:
(FILE *ifile, ImplementationType type, const std::string &epsilon_symbol);
public:
- /* Whether the conversion requested can be done without losing
+ /* Whether the conversion requested can be done without losing
any information */
static bool is_safe_conversion(ImplementationType original,
ImplementationType conversion);
- /* whether HFST is linked to the transducer library
+ /* whether HFST is linked to the transducer library
needed by implementation type \a type. */
HFSTDLL static bool is_implementation_type_available(ImplementationType type);
@@ -374,7 +373,7 @@ An example:
HFSTDLL void twosided_flag_diacritics();
protected:
- /* For internal use: Create a tokenizer that recognizes all symbols
+ /* For internal use: Create a tokenizer that recognizes all symbols
that occur in the transducer. */
HfstTokenizer create_tokenizer();
@@ -385,9 +384,9 @@ An example:
protected:
/* Get the number used to represent the symbol \a symbol. */
- unsigned int get_symbol_number(const std::string &symbol);
+ //unsigned int get_symbol_number(const std::string &symbol);
- /* For internal use, implemented only for SFST_TYPE. */
+ /* For internal use, implemented only for SFST_TYPE. */
std::vector<HfstTransducer*> extract_path_transducers();
/* For internal use:
@@ -406,28 +405,28 @@ An example:
/* For internal use:
Create a backend implementation of the same type that this transducer
- has and that is equivalent to \a t and delete \a t. Assign the
+ has and that is equivalent to \a t and delete \a t. Assign the
backend implementation as the value of the implementation of this
transducer. */
HfstTransducer &convert_to_hfst_transducer
(implementations::HfstBasicTransducer *t);
- /* \brief For internal use: Create a transducer of type \a type as
+ /* \brief For internal use: Create a transducer of type \a type as
defined in AT&T format in file named \a filename.
\a epsilon_symbol defines how epsilons are represented.
NOTE: If the transition symbols contain space characters
they must be represented as "@_SPACE_@" because
- whitespace characters are used as field separators
+ whitespace characters are used as field separators
in AT&T format.
@pre The file exists, otherwise an exception is thrown.
@see HfstTransducer(FILE, ImplementationType, const std::string&)
- @throws StreamNotReadableException
- @throws NotValidAttFormatException
+ @throws StreamNotReadableException
+ @throws NotValidAttFormatException
*/
static HfstTransducer &read_in_att_format
- (const std::string &filename, ImplementationType type,
+ (const std::string &filename, ImplementationType type,
const std::string &epsilon_symbol);
public:
@@ -453,7 +452,7 @@ An example:
// ----- Constructors, destructor, assignment -----
// ------------------------------------------------
- /** \brief Create an uninitialized transducer (use with care).
+ /** \brief Create an uninitialized transducer (use with care).
@note This constructor leaves the backend implementation variable
uninitialized. An uninitialized transducer is likely to cause a
@@ -461,21 +460,21 @@ An example:
a value at some point. */
HFSTDLL HfstTransducer();
- /** \brief Create an empty transducer, i.e. a transducer that does not
+ /** \brief Create an empty transducer, i.e. a transducer that does not
recognize any string. The type of the transducer is defined by \a type.
- @note Use HfstTransducer("@_EPSILON_SYMBOL_@")
+ @note Use HfstTransducer("@_EPSILON_SYMBOL_@")
to create an epsilon transducer.
**/
HFSTDLL HfstTransducer(ImplementationType type);
/** \brief Create a transducer by tokenizing the utf8 string \a utf8_string
with tokenizer \a multichar_symbol_tokenizer.
- The type of the transducer is defined by \a type.
+ The type of the transducer is defined by \a type.
- \a utf8_str is read one token at a time and for each token
+ \a utf8_str is read one token at a time and for each token
a new transition is created in the resulting
- transducer. The input and output symbols of that transition are
+ transducer. The input and output symbols of that transition are
the same as the token read.
An example:
@@ -487,20 +486,20 @@ An example:
\endverbatim
@see HfstTokenizer **/
- HFSTDLL HfstTransducer(const std::string& utf8_str,
+ HFSTDLL HfstTransducer(const std::string& utf8_str,
const HfstTokenizer &multichar_symbol_tokenizer,
ImplementationType type);
- /** \brief Create a transducer by tokenizing
+ /** \brief Create a transducer by tokenizing
the utf8 input string \a input_utf8_string
and output string \a output_utf8_string
with tokenizer \a multichar_symbol_tokenizer.
- The type of the transducer is defined by \a type.
+ The type of the transducer is defined by \a type.
\a input_utf8_str and \a output_utf8_str are read one token at a time
- and for each token a new transition
+ and for each token a new transition
is created in the resulting transducer. The input and output symbols
- of that transition are the same as
+ of that transition are the same as
the input and output tokens read. If either string contains less tokens
than another, epsilons are used
as transition symbols for the shorter string.
@@ -520,11 +519,11 @@ An example:
const HfstTokenizer &multichar_symbol_tokenizer,
ImplementationType type);
- /* @brief Create a transducer that recognizes the union of string pairs in
+ /* @brief Create a transducer that recognizes the union of string pairs in
\a sps. The type of the transducer is defined by \a type. \a cyclic
defines whether the transducer recognizes any number (from zero to
infinity, inclusive) of consecutive string pairs in \s sps. */
- HFSTDLL HfstTransducer(const StringPairSet & sps, ImplementationType type,
+ HFSTDLL HfstTransducer(const StringPairSet & sps, ImplementationType type,
bool cyclic=false);
/* \brief Create a transducer that recognizes the concatenation of
@@ -534,28 +533,28 @@ An example:
HFSTDLL HfstTransducer(const StringVector & sv, ImplementationType type);
- /* \brief Create a transducer that recognizes the concatenation of the
+ /* \brief Create a transducer that recognizes the concatenation of the
unions of string pairs in string pair sets in \a spsv. The type of
the transducer is defined by \a type. */
- HFSTDLL HfstTransducer(const std::vector<StringPairSet> & spsv,
+ HFSTDLL HfstTransducer(const std::vector<StringPairSet> & spsv,
ImplementationType type);
- /** \brief Read a binary transducer from transducer stream \a in.
+ /** \brief Read a binary transducer from transducer stream \a in.
The stream can contain HFST tranducers or OpenFst, foma or SFST
transducers without an HFST header. If the backend implementations
are used as such, they are converted into HFST transducers.
For more information on transducer conversions and the HFST header
- structure, see
+ structure, see
<a href="https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstTransducerHeader">here</a>.
@pre ( in.is_eof() == in.is_bad() == false && in.is_fst() ).
Otherwise, an exception is thrown.
- @throws NotTransducerStreamException
+ @throws NotTransducerStreamException
@throws StreamNotReadableException
- @throws StreamIsClosedException
+ @throws StreamIsClosedException
@throws TransducerTypeMismatchException
@throws MissingOpenFstInputSymbolTableException
@@ -565,33 +564,33 @@ An example:
/** \brief Create a deep copy of transducer \a another. **/
HFSTDLL HfstTransducer(const HfstTransducer &another);
- /** \brief Create an HFST transducer equivalent to
+ /** \brief Create an HFST transducer equivalent to
HFST basic transducer \a t. The type of the created transducer
is defined by \a type. **/
- HFSTDLL HfstTransducer(const hfst::implementations::HfstBasicTransducer &t,
+ HFSTDLL HfstTransducer(const hfst::implementations::HfstBasicTransducer &t,
ImplementationType type);
/** \brief Create a transducer that recognizes the string pair
- <"symbol","symbol">, i.e. [symbol:symbol].
- The type of the transducer is defined by \a type.
+ <"symbol","symbol">, i.e. [symbol:symbol].
+ The type of the transducer is defined by \a type.
@see String **/
HFSTDLL HfstTransducer(const std::string &symbol, ImplementationType type);
- /** \brief Create a transducer that recognizes the string pair
- <"isymbol","osymbol">, i.e [isymbol:osymbol].
- The type of the transducer is defined by \a type.
+ /** \brief Create a transducer that recognizes the string pair
+ <"isymbol","osymbol">, i.e [isymbol:osymbol].
+ The type of the transducer is defined by \a type.
@see String **/
- HFSTDLL HfstTransducer(const std::string &isymbol, const std::string &osymbol,
+ HFSTDLL HfstTransducer(const std::string &isymbol, const std::string &osymbol,
ImplementationType type);
- /** \brief Create a transducer of type \a type as defined in AT&T format
- in FILE \a ifile. \a epsilon_symbol defines how epsilons
+ /** \brief Create a transducer of type \a type as defined in AT&T format
+ in FILE \a ifile. \a epsilon_symbol defines how epsilons
are represented.
In AT&T format, the transition lines are of the form:
-\verbatim
- [0-9]+[\w]+[0-9]+[\w]+[^\w]+[\w]+[^\w]([\w]+(-)[0-9]+(\.[0-9]+))
+\verbatim
+ [0-9]+[\w]+[0-9]+[\w]+[^\w]+[\w]+[^\w]([\w]+(-)[0-9]+(\.[0-9]+))
\endverbatim
and final state lines:
@@ -600,10 +599,10 @@ An example:
[0-9]+[\w]+([\w]+(-)[0-9]+(\.[0-9]+))
\endverbatim
- If several transducers are listed in the same file,
- they are separated by lines of
- two consecutive hyphens "--". If the weight
- (<tt>([\\w]+(-)[0-9]+(\.[0-9]+))</tt>)
+ If several transducers are listed in the same file,
+ they are separated by lines of
+ two consecutive hyphens "--". If the weight
+ (<tt>([\\w]+(-)[0-9]+(\.[0-9]+))</tt>)
is missing, the transition or final state is given a zero weight.
NOTE: If transition symbols contains spaces, they must be escaped
@@ -624,12 +623,12 @@ An example:
0 0 a <eps> 0.2
\endverbatim
- The example lists four transducers in AT&T format:
+ The example lists four transducers in AT&T format:
one transducer accepting the string pair <"foo","bar">, one
- epsilon transducer, one empty transducer and one transducer
+ epsilon transducer, one empty transducer and one transducer
that accepts any number of 'a's and produces an empty string
in all cases. The transducers
- can be read with the following commands (from a file named
+ can be read with the following commands (from a file named
"testfile.att"):
\verbatim
std::vector<HfstTransducer> transducers;
@@ -648,19 +647,19 @@ fprintf(stderr, "Read %i transducers in total.\n", (int)transducers.size());
\endverbatim
Epsilon will be represented as "@_EPSILON_SYMBOL_@" in the resulting transducer.
-The argument \a epsilon_symbol only denotes how epsilons are represented
+The argument \a epsilon_symbol only denotes how epsilons are represented
in \a ifile.
- at throws NotValidAttFormatException
+ at throws NotValidAttFormatException
@throws StreamNotReadableException
@throws StreamIsClosedException
- at see #write_in_att_format(FILE*,bool)const
+ at see #write_in_att_format(FILE*,bool)const
@see String
**/
- HFSTDLL HfstTransducer(FILE * ifile, ImplementationType type,
+ HFSTDLL HfstTransducer(FILE * ifile, ImplementationType type,
const std::string &epsilon_symbol, unsigned int & linecount);
- HFSTDLL HfstTransducer(FILE * ifile, ImplementationType type,
+ HFSTDLL HfstTransducer(FILE * ifile, ImplementationType type,
const std::string &epsilon_symbol);
@@ -677,11 +676,11 @@ in \a ifile.
// ----------- Properties, comparison, conversion -------------
// ------------------------------------------------------------
- /** \brief Rename the transducer \a name.
+ /** \brief Rename the transducer \a name.
@see get_name */
HFSTDLL void set_name(const std::string &name);
- /** \brief Get the name of the transducer.
+ /** \brief Get the name of the transducer.
@see set_name */
HFSTDLL std::string get_name() const;
@@ -696,7 +695,7 @@ in \a ifile.
*/
HFSTDLL void set_property(const std::string& property, const std::string& value);
- /**
+ /**
* @brief Get arbitrary string propert @a property.
* get_property("name") works like get_name.
*/
@@ -705,33 +704,33 @@ in \a ifile.
* @brief Get all properties form transducer.
*/
HFSTDLL const std::map<std::string,std::string>& get_properties() const;
- /** \brief Get the alphabet of the transducer.
+ /** \brief Get the alphabet of the transducer.
- The alphabet is defined as the set of symbols known
+ The alphabet is defined as the set of symbols known
to the transducer. */
HFSTDLL StringSet get_alphabet() const;
- /** \brief Get first input level symbols of strings recognized
+ /** \brief Get first input level symbols of strings recognized
(or rejected, if they end in a non-final state) by the transducer. */
HFSTDLL StringSet get_first_input_symbols() const;
/** \brief Harmonize transducers this and another.
- @note In harmonization,
+ @note In harmonization,
the symbol-to-number correspondencies of this transducer are recoded
so that they are equivalent to the ones used in transducer another.
Then the unknown and identity symbols are expanded
- in both transducers. If this and \a another have type FOMA_TYPE,
+ in both transducers. If this and \a another have type FOMA_TYPE,
nothing is done, since foma takes care of harmonization. */
- HFSTDLL void harmonize(HfstTransducer &another);
+ HFSTDLL void harmonize(HfstTransducer &another, bool force=false);
- /** \brief Explicitly insert \a symbol to the alphabet
- of the transducer.
+ /** \brief Explicitly insert \a symbol to the alphabet
+ of the transducer.
@note Usually this function is not needed since new symbols are
added to the alphabet by default. */
- HFSTDLL void insert_to_alphabet(const std::string &symbol);
- HFSTDLL void insert_to_alphabet(const std::set<std::string> &symbols);
+ HFSTDLL void insert_to_alphabet(const std::string &symbol);
+ HFSTDLL void insert_to_alphabet(const std::set<std::string> &symbols);
/** \brief Remove \a symbol from the alphabet of the transducer.
CURRENTLY NOT IMPLEMENTED.
@@ -772,20 +771,20 @@ in \a ifile.
Two transducers are equivalent iff they accept the same input/output
string pairs with the same weights
- and the same alignments.
+ and the same alignments.
*/
HFSTDLL bool compare(const HfstTransducer &another, bool harmonize=true) const;
- /** \brief Convert the transducer into an equivalent transducer
- in format \a type.
+ /** \brief Convert the transducer into an equivalent transducer
+ in format \a type.
- If a weighted transducer is converted into an unweighted one,
- all weights are lost.
- In the reverse case, all weights are initialized to the
- semiring's one.
+ If a weighted transducer is converted into an unweighted one,
+ all weights are lost.
+ In the reverse case, all weights are initialized to the
+ semiring's one.
A transducer of type #SFST_TYPE, #TROPICAL_OPENFST_TYPE,
- #LOG_OPENFST_TYPE or #FOMA_TYPE can be converted into an
+ #LOG_OPENFST_TYPE or #FOMA_TYPE can be converted into an
#HFST_OL_TYPE or #HFST_OLW_TYPE transducer, but an #HFST_OL_TYPE
or #HFST_OLW_TYPE transducer cannot be converted to any other type.
@@ -799,21 +798,21 @@ in \a ifile.
// --- String lookup and conversion to/from AT&T format ---
// --------------------------------------------------------
- /** \brief Write the transducer in AT&T format to FILE \a ofile.
+ /** \brief Write the transducer in AT&T format to FILE \a ofile.
\a write_weights defines whether weights are written.
- The fields in the resulting AT&T format are separated
+ The fields in the resulting AT&T format are separated
by tabulator characters.
NOTE: If the transition symbols contain space characters,
the spaces are printed as "@_SPACE_@" because
- whitespace characters are used as field separators
+ whitespace characters are used as field separators
in AT&T format. Epsilon symbols are printed as "@0@".
- If several transducers are written in the same file, they must
+ If several transducers are written in the same file, they must
be separated by a line of two consecutive hyphens "--", so that
- they will be read correctly by
+ they will be read correctly by
HfstTransducer(FILE*, ImplementationType, const std::string&).
An example:
@@ -848,7 +847,7 @@ This will yield a file "testfile.att" that looks as follows:
0 0 a a 0.0
\endverbatim
- @throws StreamCannotBeWrittenException
+ @throws StreamCannotBeWrittenException
@throws StreamIsClosedException
@see operator<<(std::ostream &out, const HfstTransducer &t)
@@ -869,43 +868,43 @@ This will yield a file "testfile.att" that looks as follows:
(FILE * ofile, bool write_weights=true) const;
- /** \brief \brief Write the transducer in AT&T format to FILE
+ /** \brief \brief Write the transducer in AT&T format to FILE
named \a filename. \a write_weights
defines whether weights are written.
- If the file exists, it is overwritten.
- If the file does not exist, it is created.
+ If the file exists, it is overwritten.
+ If the file does not exist, it is created.
@see #write_in_att_format */
- HFSTDLL void write_in_att_format(const std::string &filename,
+ HFSTDLL void write_in_att_format(const std::string &filename,
bool write_weights=true) const;
public:
- /* \brief Call \a callback with some or all string pairs recognized
+ /* \brief Call \a callback with some or all string pairs recognized
by the transducer?
- If the callback returns false the search will be terminated.
+ If the callback returns false the search will be terminated.
The \a cycles parameter
indicates how many times a cycle will be followed, with negative numbers
- indicating unlimited. Note that if the transducer is cyclic and
+ indicating unlimited. Note that if the transducer is cyclic and
cycles aren't capped,
the search will not end until the callback returns false. */
HFSTDLL void extract_paths(ExtractStringsCb& callback, int cycles=-1) const;
- /** \brief Extract a maximum of \a max_num paths that are
+ /** \brief Extract a maximum of \a max_num paths that are
recognized by the transducer following a maximum of \a cycles cycles
and store the paths into \a results.
- @param results The extracted paths are inserted here.
- @param max_num The total number of resulting strings is capped at
- \a max_num, with 0 or negative indicating unlimited.
+ @param results The extracted paths are inserted here.
+ @param max_num The total number of resulting strings is capped at
+ \a max_num, with 0 or negative indicating unlimited.
@param cycles Indicates how many times a cycle will be followed, with
negative numbers indicating unlimited.
- This is a version of extract_paths that handles flag diacritics
+ This is a version of extract_paths that handles flag diacritics
as ordinary symbols and does not validate the sequences prior to
- outputting as opposed to
+ outputting as opposed to
#extract_paths_fd(HfstTwoLevelPaths &, int, int, bool) const.
If this function is called on a cyclic transducer with unlimited
@@ -937,10 +936,10 @@ This will yield a file "testfile.att" that looks as follows:
ostring.append(IT->second);
}
// Print input and output strings of each path
- std::cerr << istring << ":" << ostring;
+ std::cerr << istring << ":" << ostring;
// and optionally the weight of the path.
//std::cerr << "\t" << it->first;
- std::cerr << std::endl;
+ std::cerr << std::endl;
}
\endverbatim
@@ -971,7 +970,7 @@ ccc : ddd
@bug Does not work for HFST_OL_TYPE or HFST_OLW_TYPE?
@throws TransducerIsCyclicException
- @see #n_best
+ @see #n_best
@see hfst::HfstTransducer::extract_paths_fd(hfst::HfstTwoLevelPaths&, int, int, bool) const
*/
HFSTDLL void extract_paths
@@ -983,7 +982,7 @@ ccc : ddd
HFSTDLL void extract_random_paths_fd
(HfstTwoLevelPaths &results, int max_num, bool filter_fd) const;
- /* \brief Call \a callback with extracted strings that are not
+ /* \brief Call \a callback with extracted strings that are not
invalidated by flag diacritic rules.
@see extract_paths(HfstTwoLevelPaths&, int, int) */
@@ -1001,15 +1000,15 @@ ccc : ddd
HFSTDLL int longest_path_size(bool obey_flags=true) const;
public:
- /** \brief Extract a maximum of \a max_num paths that are
- recognized by the transducer and are not invalidated by flag
+ /** \brief Extract a maximum of \a max_num paths that are
+ recognized by the transducer and are not invalidated by flag
diacritic rules following a maximum of \a cycles cycles
and store the paths into \a results. \a filter_fd defines whether
the flag diacritics themselves are filtered out of the result strings.
- @param results The extracted paths are inserted here.
- @param max_num The total number of resulting strings is capped at
- \a max_num, with 0 or negative indicating unlimited.
+ @param results The extracted paths are inserted here.
+ @param max_num The total number of resulting strings is capped at
+ \a max_num, with 0 or negative indicating unlimited.
@param cycles Indicates how many times a cycle will be followed, with
negative numbers indicating unlimited.
@param filter_fd Whether the flag diacritics are filtered out of the
@@ -1018,9 +1017,9 @@ ccc : ddd
If this function is called on a cyclic transducer with unlimited
values for both \a max_num and \a cycles, an exception will be thrown.
- Flag diacritics are of the form @[PNDRCU][.][A-Z]+([.][A-Z]+)?@.
+ Flag diacritics are of the form @[PNDRCU][.][A-Z]+([.][A-Z]+)?@.
- For example the transducer
+ For example the transducer
\verbatim
[[@P.FEATURE.FOO@ foo] | [@P.FEATURE.BAR@ bar]] | [[foo @U.FEATURE.FOO@] | [bar @U.FEATURE.BAR@]]
@@ -1035,12 +1034,12 @@ ccc : ddd
@throws TransducerIsCyclicException
@see extract_paths(HfstTwoLevelPaths&, int, int) const */
HFSTDLL void extract_paths_fd
- (HfstTwoLevelPaths &results, int max_num=-1, int cycles=-1,
+ (HfstTwoLevelPaths &results, int max_num=-1, int cycles=-1,
bool filter_fd=true) const;
//! @brief Lookup or apply a single tokenized string \a s and
//! return a maximum of \a limit results.
- //!
+ //!
//! This is a version of lookup that handles flag diacritics as ordinary
//! symbols and does not validate the sequences prior to outputting.
//! Currently, this function calls lookup_fd.
@@ -1054,13 +1053,20 @@ ccc : ddd
//! @brief Lookup or apply a single string \a s and
//! return a maximum of \a limit results.
- //!
+ //!
//! This is an overloaded lookup function that leaves tokenizing to the
//! transducer.
HFSTDLL HfstOneLevelPaths * lookup(const std::string & s,
ssize_t limit = -1,
double time_cutoff = 0.0) const;
+ //! @brief Lookup or apply a single string \a s and
+ //! return a maximum of \a limit results in string pair format.
+ //!
+ HFSTDLL HfstTwoLevelPaths * lookup_pairs(const std::string & s,
+ ssize_t limit = -1,
+ double time_cutoff = 0.0) const;
+
//! @brief Lookup or apply a single string \a s minding flag diacritics
//! properly and store a maximum of \a limit results to \a results.
//!
@@ -1074,18 +1080,18 @@ ccc : ddd
//! Epsilons on the second level are represented by empty strings
//! in \a results. For an example of flag diacritics, see
//! #hfst::HfstTransducer::extract_paths_fd(hfst::HfstTwoLevelPaths&, int, int, bool) const
- //!
+ //!
//!
//! @pre The transducer must be of type #HFST_OL_TYPE or #HFST_OLW_TYPE.
//! This function is not implemented for other transducer types.
//!
//! @param s String to look up. The weight is ignored.
- //! @param limit (Currently ignored.) Number of strings to look up.
- //! -1 tries to look up all and may get stuck
+ //! @param limit (Currently ignored.) Number of strings to look up.
+ //! -1 tries to look up all and may get stuck
//! if infinitely ambiguous.
//! @param time_cutoff Number of seconds that can pass before lookup is stopped.
//! \return{A pointer to a HfstOneLevelPaths container allocated by callee}
- //!
+ //!
//! @see HfstTokenizer::tokenize_one_level
//! @see is_lookup_infinitely_ambiguous(const StringVector&) const
//!
@@ -1106,8 +1112,8 @@ ccc : ddd
//! speed can therefore be *extremely* slow.}
//!
//! @param s String to look up. The weight is ignored.
- //! @param limit (Currently ignored.) Number of strings to look up.
- //! -1 tries to look up all and may get stuck
+ //! @param limit (Currently ignored.) Number of strings to look up.
+ //! -1 tries to look up all and may get stuck
//! if infinitely ambiguous.
//! @param time_cutoff Number of seconds that can pass before lookup is stopped.
//! \return{A pointer to a HfstOneLevelPaths container allocated by callee}
@@ -1118,11 +1124,11 @@ ccc : ddd
ssize_t limit = -1,
double time_cutoff = 0.0) const;
- //! @brief Lookup or apply a single string \a s and store a maximum of
+ //! @brief Lookup or apply a single string \a s and store a maximum of
//! \a limit results to \a results. \a tok defined how \a s is tokenized.
//!
//!
- //! This function is the same as
+ //! This function is the same as
//! #lookup(const StringVector&, ssize_t, double) const
//! but lookup is not done using a string and a tokenizer instead of
//! a StringVector.
@@ -1131,12 +1137,12 @@ ccc : ddd
ssize_t limit = -1,
double time_cutoff = 0.0) const;
- //! @brief Lookup or apply a single string \a s minding flag diacritics
- //! properly and store a maximum of \a limit results to \a results.
+ //! @brief Lookup or apply a single string \a s minding flag diacritics
+ //! properly and store a maximum of \a limit results to \a results.
//! \a tok defines how s is tokenized.
//!
- //! The same as
- //! #lookup_fd(const StringVector&, ssize_t, double) const
+ //! The same as
+ //! #lookup_fd(const StringVector&, ssize_t, double) const
//! but uses a tokenizer and a string instead of a StringVector.
//!
HFSTDLL HfstOneLevelPaths * lookup_fd(
@@ -1144,7 +1150,7 @@ ccc : ddd
const std::string &s, ssize_t limit = -1,
double time_cutoff = 0.0) const;
- //! @brief (Not implemented) Lookdown a single string \a s and return
+ //! @brief (Not implemented) Lookdown a single string \a s and return
//! a maximum of \a limit results.
//!
//! Traverse all paths on logical second level of the transducer to produce
@@ -1164,9 +1170,9 @@ ccc : ddd
HFSTDLL HfstOneLevelPaths * lookdown(const std::string& s,
ssize_t limit = -1) const;
- //! @brief (Not implemented) Lookdown a single string minding
+ //! @brief (Not implemented) Lookdown a single string minding
//! flag diacritics properly.
- //!
+ //!
//! This is a version of lookdown that handles flag diacritics as epsilons
//! and validates the sequences prior to outputting.
//!
@@ -1203,7 +1209,7 @@ ccc : ddd
HFSTDLL HfstTransducer &eliminate_flags();
HFSTDLL HfstTransducer &eliminate_flag(const std::string &flag);
- /** \brief Remove all <i>epsilon:epsilon</i> transitions
+ /** \brief Remove all <i>epsilon:epsilon</i> transitions
from the transducer so that the transducer remains equivalent. */
HFSTDLL HfstTransducer &remove_epsilons();
@@ -1219,25 +1225,31 @@ ccc : ddd
/** \brief Minimize the transducer.
- Minimizing a transducer yields an equivalent transducer with
- the smallest number of states.
+ Minimizing a transducer yields an equivalent transducer with
+ the smallest number of states.
- @bug OpenFst's minimization algorithm seems to add epsilon
+ @bug OpenFst's minimization algorithm seems to add epsilon
transitions to weighted transducers? */
HFSTDLL HfstTransducer &minimize();
- /** \brief Extract \a n best paths of the transducer.
+ /** \brief Minimize or determinize the transducer.
- In the case of a weighted transducer (#TROPICAL_OPENFST_TYPE or
- #LOG_OPENFST_TYPE), best paths are defined as paths with
+ If hfst::set_minimization(false) has been called, determinize the transducer.
+ Else, minimize it. */
+ HFSTDLL HfstTransducer &optimize();
+
+ /** \brief Extract \a n best paths of the transducer.
+
+ In the case of a weighted transducer (#TROPICAL_OPENFST_TYPE or
+ #LOG_OPENFST_TYPE), best paths are defined as paths with
the lowest weight.
- In the case of an unweighted transducer (#SFST_TYPE or #FOMA_TYPE),
+ In the case of an unweighted transducer (#SFST_TYPE or #FOMA_TYPE),
the function returns random paths.
This function is not implemented for #FOMA_TYPE or #SFST_TYPE.
- If this function is called by an HfstTransducer of type #FOMA_TYPE
+ If this function is called by an HfstTransducer of type #FOMA_TYPE
or #SFST_TYPE, it is converted to #TROPICAL_OPENFST_TYPE,
- paths are extracted and it is converted back to #FOMA_TYPE or
+ paths are extracted and it is converted back to #FOMA_TYPE or
#SFST_TYPE. If HFST is not linked to OpenFst library, an
ImplementationTypeNotAvailableException is thrown.
*/
@@ -1248,57 +1260,57 @@ ccc : ddd
// ------------- Algebraic operations -------------
// ------------------------------------------------
- /** \brief A concatenation of N transducers where N is any number
+ /** \brief A concatenation of N transducers where N is any number
from zero to infinity. */
HFSTDLL HfstTransducer &repeat_star();
- /** \brief A concatenation of N transducers where N is any number
+ /** \brief A concatenation of N transducers where N is any number
from one to infinity. */
HFSTDLL HfstTransducer &repeat_plus();
/** \brief A concatenation of \a n transducers. */
HFSTDLL HfstTransducer &repeat_n(unsigned int n);
- /** \brief A concatenation of N transducers where N is any number
+ /** \brief A concatenation of N transducers where N is any number
from zero to \a n, inclusive.*/
HFSTDLL HfstTransducer &repeat_n_minus(unsigned int n);
- /** \brief A concatenation of N transducers where N is any number
+ /** \brief A concatenation of N transducers where N is any number
from \a n to infinity, inclusive.*/
HFSTDLL HfstTransducer &repeat_n_plus(unsigned int n);
- /** \brief A concatenation of N transducers where N is any number
+ /** \brief A concatenation of N transducers where N is any number
from \a n to \a k, inclusive.*/
HFSTDLL HfstTransducer& repeat_n_to_k(unsigned int n, unsigned int k);
/** \brief Disjunct the transducer with an epsilon transducer. */
HFSTDLL HfstTransducer &optionalize();
- /** \brief Swap the input and output symbols of each transition
+ /** \brief Swap the input and output symbols of each transition
in the transducer. */
HFSTDLL HfstTransducer &invert();
- /** \brief Reverse the transducer.
+ /** \brief Reverse the transducer.
- A reverted transducer accepts the string "n(0) n(1) ... n(N)"
+ A reverted transducer accepts the string "n(0) n(1) ... n(N)"
iff the original
transducer accepts the string "n(N) n(N-1) ... n(0)" */
HFSTDLL HfstTransducer &reverse();
- /** \brief Extract the input language of the transducer.
+ /** \brief Extract the input language of the transducer.
- All transition symbol pairs <i>isymbol:osymbol</i> are changed
+ All transition symbol pairs <i>isymbol:osymbol</i> are changed
to <i>isymbol:isymbol</i>. */
HFSTDLL HfstTransducer &input_project();
/** \brief Extract the output language of the transducer.
- All transition symbol pairs <i>isymbol:osymbol</i> are changed
+ All transition symbol pairs <i>isymbol:osymbol</i> are changed
to <i>osymbol:osymbol</i>. */
HFSTDLL HfstTransducer &output_project();
/** \brief Compose this transducer with \a another. */
- HFSTDLL HfstTransducer &compose(const HfstTransducer &another,
+ HFSTDLL HfstTransducer &compose(const HfstTransducer &another,
bool harmonize=true);
HFSTDLL HfstTransducer &merge(const HfstTransducer &another, const std::map<std::string, std::set<std::string> > & list_symbols);
@@ -1309,9 +1321,9 @@ ccc : ddd
transducers in \a v. If \a invert is true, then compose the
intersection of the transducers in \a v with this transducer.
- The algorithm used by this function is faster than intersecting
- all transducers one by one and then composing this transducer
- with the intersection.
+ The algorithm used by this function is faster than intersecting
+ all transducers one by one and then composing this transducer
+ with the intersection.
@pre The transducers in \a v are deterministic and epsilon-free.
*/
@@ -1372,7 +1384,7 @@ ccc : ddd
* If transducer A accepts string "foo" and transducer B string "bar",
* the transducer that results from shuffling A and B accepts all strings
* [(f|b)(o|a)(o|r)].
- *
+ *
* @pre Both transducers must be automata, i.e. map strings onto themselves.
*
*/
@@ -1400,7 +1412,7 @@ ccc : ddd
- /* For HfstCompiler: Optimized disjunction function. */
+ /* For SfstCompiler: Optimized disjunction function. */
HFSTDLL HfstTransducer &disjunct(const StringPairVector &spv);
/** \brief Intersect this transducer with \a another. */
@@ -1414,10 +1426,10 @@ ccc : ddd
// ---------- Insertion and substitution ----------
// ------------------------------------------------
- /** \brief Freely insert symbol pair \a symbol_pair into the transducer.
+ /** \brief Freely insert symbol pair \a symbol_pair into the transducer.
- To each state in this transducer is added a transition that
- leads from that state to itself with input and output symbols
+ To each state in this transducer is added a transition that
+ leads from that state to itself with input and output symbols
defined by \a symbol_pair.
If harmonize is true, then identity and unknown symbols in the
@@ -1426,25 +1438,25 @@ ccc : ddd
*/
HFSTDLL HfstTransducer &insert_freely(const StringPair &symbol_pair, bool harmonize=true);
- /** \brief Freely insert a copy of \a tr into the transducer.
+ /** \brief Freely insert a copy of \a tr into the transducer.
- A copy of \a tr is attached with epsilon transitions
- to each state of this transducer. After the operation, for each
- state S in this transducer, there is an epsilon transition
- that leads from state S to the initial state of \a tr,
+ A copy of \a tr is attached with epsilon transitions
+ to each state of this transducer. After the operation, for each
+ state S in this transducer, there is an epsilon transition
+ that leads from state S to the initial state of \a tr,
and for each final state of \a tr, there is an epsilon transition
that leads from that final state to state S in this transducer.
- The weights of the final states in \a tr are copied to the
+ The weights of the final states in \a tr are copied to the
epsilon transitions leading to state S.
- Implemented only for implementations::HfstBasicTransducer.
+ Implemented only for implementations::HfstBasicTransducer.
Conversion is carried out for an HfstTransducer, if this function
is called.
*/
HFSTDLL HfstTransducer &insert_freely(const HfstTransducer &tr, bool harmonize=true);
- /** \brief Substitute all transition \a sp with transitions \a sps
- as defined by function \a func.
+ /** \brief Substitute all transition \a sp with transitions \a sps
+ as defined by function \a func.
@param func A pointer to a function that takes as its argument
a StringPair sp and inserts to StringPairSet sps all StringPairs
@@ -1454,7 +1466,7 @@ ccc : ddd
An example:
\verbatim
-bool function(const StringPair &sp, StringPairSet &sps)
+bool function(const StringPair &sp, StringPairSet &sps)
{
if (sp.second.compare(sp.first) != 0)
return false;
@@ -1476,20 +1488,20 @@ bool function(const StringPair &sp, StringPairSet &sps)
...
-// For all transitions in transducer t whose input and output wovels
+// For all transitions in transducer t whose input and output wovels
// are equivalent, substitute the output wovel with a symbol that defines
// whether the wovel in question is a front or back wovel.
t.substitute(&function);
-\endverbatim
+\endverbatim
@see String
*/
HFSTDLL HfstTransducer &substitute
(bool (*func)(const StringPair &sp, StringPairSet &sps));
- /** \brief Substitute all transition symbols equal to \a old_symbol
+ /** \brief Substitute all transition symbols equal to \a old_symbol
with symbol \a new_symbol.
- \a input_side and \a output_side define whether
+ \a input_side and \a output_side define whether
the substitution is made on input and output sides.
@param old_symbol Symbol to be substituted.
@@ -1499,7 +1511,7 @@ t.substitute(&function);
@param output_side Whether the substitution is made on the output side
of a transition.
- The transition weights remain the same.
+ The transition weights remain the same.
@see String */
HFSTDLL HfstTransducer &substitute(const std::string &old_symbol,
@@ -1507,15 +1519,15 @@ t.substitute(&function);
bool input_side=true,
bool output_side=true);
- /** \brief Substitute all transition symbol pairs equal to
- \a old_symbol_pair with \a new_symbol_pair.
+ /** \brief Substitute all transition symbol pairs equal to
+ \a old_symbol_pair with \a new_symbol_pair.
The transition weights remain the same.
Implemented only for #TROPICAL_OPENFST_TYPE and #LOG_OPENFST_TYPE.
- If this function is called by an unweighted HfstTransducer,
+ If this function is called by an unweighted HfstTransducer,
it is converted to a weighted one,
- substitution is made and the transducer is converted back
+ substitution is made and the transducer is converted back
to the original format.
@see String
@@ -1523,13 +1535,13 @@ t.substitute(&function);
HFSTDLL HfstTransducer &substitute(const StringPair &old_symbol_pair,
const StringPair &new_symbol_pair);
- /** \brief Substitute all transitions equal to \a old_symbol_pair
- with a set of transitions equal to \a new_symbol_pair_set.
+ /** \brief Substitute all transitions equal to \a old_symbol_pair
+ with a set of transitions equal to \a new_symbol_pair_set.
The weight of the original transition is copied to all new transitions.
Implemented only for #TROPICAL_OPENFST_TYPE and #LOG_OPENFST_TYPE.
- If this function is called by an unweighted HfstTransducer
+ If this function is called by an unweighted HfstTransducer
(#SFST_TYPE or #FOMA_TYPE), it is converted to #TROPICAL_OPENFST_TYPE,
substitution is done and it is converted back to the original format.
@@ -1546,8 +1558,8 @@ t.substitute(&function);
/** \brief Substitute all transition symbols as defined in \a substitutions.
- Each symbol old_symbol is substituted with symbol new_symbol, iff
- substitutions.find(old_symbol) == new_symbol != substitutions.end().
+ Each symbol old_symbol is substituted with symbol new_symbol, iff
+ substitutions.find(old_symbol) == new_symbol != substitutions.end().
Otherwise, old_symbol remains the same.
This function performs all substitutions at the same time, so it is
@@ -1560,8 +1572,8 @@ t.substitute(&function);
/** \brief Substitute all transition symbol pairs as defined in \a substitutions.
Each symbol pair old_isymbol:old_osymbol is substituted with symbol pair
- new_isymbol:new_osymbol, iff substitutions.find(old_isymbol:old_osymbol) ==
- new_isymbol:new_osymbol != substitutions.end().
+ new_isymbol:new_osymbol, iff substitutions.find(old_isymbol:old_osymbol) ==
+ new_isymbol:new_osymbol != substitutions.end().
Otherwise, old_isymbol:old_osymbol remains the same.
This function performs all substitutions at the same time, so it is
@@ -1571,17 +1583,17 @@ t.substitute(&function);
HFSTDLL HfstTransducer &substitute_symbol_pairs(const HfstSymbolPairSubstitutions &substitutions);
- /** \brief Substitute all transitions equal to \a symbol_pair
- with a copy of transducer \a transducer.
+ /** \brief Substitute all transitions equal to \a symbol_pair
+ with a copy of transducer \a transducer.
- A copy of \a transducer is attached (using epsilon transitions)
- between the source and target states of the transition
+ A copy of \a transducer is attached (using epsilon transitions)
+ between the source and target states of the transition
to be substituted.
- The weight of the original transition is copied to
+ The weight of the original transition is copied to
the epsilon transition leaving from the source state.
Implemented only for #TROPICAL_OPENFST_TYPE and #LOG_OPENFST_TYPE.
- If this function is called by an unweighted HfstTransducer
+ If this function is called by an unweighted HfstTransducer
(#SFST_TYPE or #FOMA_TYPE), it is converted to #TROPICAL_OPENFST_TYPE,
substitution is done and it is converted back to the original format.
@@ -1591,20 +1603,20 @@ t.substitute(&function);
HfstTransducer &transducer, bool harmonize=true);
// -----------------------------------------------
- // --------------- Weight handling ---------------
+ // --------------- Weight handling ---------------
// -----------------------------------------------
/** \brief Set the weights of all final states to \a weight.
\a increment defines whether the old weight is incremented by \a weight
or overwritten.
- If the HfstTransducer is of unweighted type
+ If the HfstTransducer is of unweighted type
(#SFST_TYPE or #FOMA_TYPE), nothing is done.
*/
HFSTDLL HfstTransducer &set_final_weights(float weight, bool increment=false);
- /** \brief Transform all transition and state weights as defined
- in \a func.
+ /** \brief Transform all transition and state weights as defined
+ in \a func.
@param func A pointer to a function that takes a weight as its
argument and returns a weight that will be the new
@@ -1612,25 +1624,25 @@ t.substitute(&function);
An example:
\verbatim
-float func(float f) {
- return 2*f + 0.5;
+float func(float f) {
+ return 2*f + 0.5;
}
...
// All transition and final weights are multiplied by two and summed with 0.5.
transducer.transform_weights(&func);
-\endverbatim
+\endverbatim
- If the HfstTransducer is of unweighted type
+ If the HfstTransducer is of unweighted type
(#SFST_TYPE or #FOMA_TYPE), nothing is done.
*/
HFSTDLL HfstTransducer &transform_weights(float (*func)(float));
- /** \brief Push weights towards initial or final state(s)
+ /** \brief Push weights towards initial or final state(s)
as defined by \a type.
- If the HfstTransducer is of unweighted type
+ If the HfstTransducer is of unweighted type
(#SFST_TYPE or #FOMA_TYPE), nothing is done.
@see PushType
*/
@@ -1658,9 +1670,9 @@ transducer.transform_weights(&func);
/*
If both \a this and \a another contain flag diacritics, replace flag
diacritic @X.Y.(.Z)@ by @X.Y_1(.Z)@ in \a this and replace it by
- @X.Y_2(.Z)@ in \a another.
+ @X.Y_2(.Z)@ in \a another.
- If \a insert_renamed_flags is true, then the flags from \a this are
+ If \a insert_renamed_flags is true, then the flags from \a this are
inserted freely in \a another and vice versa after replacing.
*/
HFSTDLL void harmonize_flag_diacritics(HfstTransducer &another,
@@ -1688,7 +1700,7 @@ transducer.transform_weights(&func);
friend class HfstInputStream;
friend class HfstOutputStream;
friend class hfst::implementations::HfstTransitionGraph<class C>;
- friend class HfstCompiler;
+ friend class SfstCompiler;
friend class hfst::implementations::ConversionFunctions;
friend class HfstGrammar;
friend class xfst::XfstCompiler;
@@ -1698,14 +1710,14 @@ transducer.transform_weights(&func);
/** \brief Write transducer \a t in AT&T format to ostream \a out.
- The same as
- #hfst::HfstTransducer::write_in_att_format(FILE*, bool) const
+ The same as
+ #hfst::HfstTransducer::write_in_att_format(FILE*, bool) const
with ostreams. Weights are written if the type of \a t is weighted. */
HFSTDLL std::ostream &operator<<(std::ostream &out,const HfstTransducer &t);
HFSTDLL std::ostream &redirect(std::ostream &out,const HfstTransducer &t);
- /** \brief A namespace for functions that create two-level, replace,
+ /** \brief A namespace for functions that create two-level, replace,
restriction and coercion rule transducers. */
namespace rules
{
@@ -1720,30 +1732,30 @@ transducer.transform_weights(&func);
(const HfstTransducer &t, const StringPairSet &alphabet);
HFSTDLL HfstTransducer replace
- (HfstTransducer &t, ReplaceType repl_type, bool optional,
+ (HfstTransducer &t, ReplaceType repl_type, bool optional,
StringPairSet &alphabet);
HFSTDLL HfstTransducer replace_transducer
- (HfstTransducer &t, std::string lm, std::string rm,
+ (HfstTransducer &t, std::string lm, std::string rm,
ReplaceType repl_type, StringPairSet &alphabet);
HFSTDLL HfstTransducer replace_context
- (HfstTransducer &t, std::string m1, std::string m2,
+ (HfstTransducer &t, std::string m1, std::string m2,
StringPairSet &alphabet);
HFSTDLL HfstTransducer replace_in_context
- (HfstTransducerPair &context, ReplaceType repl_type,
+ (HfstTransducerPair &context, ReplaceType repl_type,
HfstTransducer &t, bool optional, StringPairSet &alphabet);
/* Used by hfst-calculate. */
HFSTDLL HfstTransducer restriction
- (HfstTransducerPairVector &contexts, HfstTransducer &mapping,
- StringPairSet &alphabet, TwolType twol_type, int direction );
+ (HfstTransducerPairVector &contexts, HfstTransducer &mapping,
+ StringPairSet &alphabet, TwolType twol_type, int direction );
// ***** THE PUBLIC INTERFACE *****
- /** \brief A transducer that obligatorily performs the mappings
+ /** \brief A transducer that obligatorily performs the mappings
defined by \a mappings in the context \a context
- when the alphabet is \a alphabet.
+ when the alphabet is \a alphabet.
@param context A pair of transducers where the first transducer
defines the left context and the second transducer
@@ -1759,7 +1771,7 @@ context = pair( [c|d], [e] )
mappings = set(a:b)
alphabet = set(a, a:b, b, c, d, e, ...)
\endverbatim
- obligatorily maps the symbol a to b if c or d precedes
+ obligatorily maps the symbol a to b if c or d precedes
and e follows. (Elsewhere,
the mapping of a to b is optional)
This expression is identical to ![.* [c|d] [a:. & !a:b] [e] .*]
@@ -1769,51 +1781,51 @@ alphabet = set(a, a:b, b, c, d, e, ...)
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>
*/
- HFSTDLL HfstTransducer two_level_if(HfstTransducerPair &context,
- StringPairSet &mappings,
+ HFSTDLL HfstTransducer two_level_if(HfstTransducerPair &context,
+ StringPairSet &mappings,
StringPairSet &alphabet);
- /** \brief A transducer that allows the mappings defined by
+ /** \brief A transducer that allows the mappings defined by
\a mappings only in the context \a context,
- when the alphabet is \a alphabet.
+ when the alphabet is \a alphabet.
- If called with the same arguments as in the example of
+ If called with the same arguments as in the example of
#two_level_if, the transducer
- allows the mapping of symbol a to b only if c or d precedes
+ allows the mapping of symbol a to b only if c or d precedes
and e follows. The
- mapping of a to b is optional in this context but cannot occur
+ mapping of a to b is optional in this context but cannot occur
in any other context.
- The expression is equivalent to
+ The expression is equivalent to
![ [ ![.* [c|d]] a:b .* ] | [ .* a:b ![[e] .*] ] ]
@see #two_level_if
*/
- HFSTDLL HfstTransducer two_level_only_if(HfstTransducerPair &context,
- StringPairSet &mappings,
+ HFSTDLL HfstTransducer two_level_only_if(HfstTransducerPair &context,
+ StringPairSet &mappings,
StringPairSet &alphabet);
- /** \brief A transducer that always performs the mappings defined
+ /** \brief A transducer that always performs the mappings defined
by \a mappings in the context \a context
- and only in that context, when the alphabet is \a alphabet.
+ and only in that context, when the alphabet is \a alphabet.
- If called with the same arguments as in the example of
+ If called with the same arguments as in the example of
#two_level_if, the transducer
maps symbol a to b only and only if c or d precedes and e follows.
- The mapping of a to b is obligatory in this context and cannot
+ The mapping of a to b is obligatory in this context and cannot
occur in any other context.
The expression is equivalent to ![.* [c|d] [a:. & !a:b] [e] .*] &
![ [ ![.* [c|d]] a:b .* ] | [ .* a:b ![[e] .*] ] ]
@see #two_level_if
*/
- HFSTDLL HfstTransducer two_level_if_and_only_if(HfstTransducerPair &context,
- StringPairSet &mappings,
+ HFSTDLL HfstTransducer two_level_if_and_only_if(HfstTransducerPair &context,
+ StringPairSet &mappings,
StringPairSet &alphabet);
- /** \brief A transducer that performs an upward mapping \a mapping
+ /** \brief A transducer that performs an upward mapping \a mapping
in the context \a context when the alphabet is \a alphabet.
- \a optional defines whether the mapping is optional.
+ \a optional defines whether the mapping is optional.
@param context A pair of transducers where the first transducer
defines the left context and the second transducer
@@ -1826,15 +1838,15 @@ alphabet = set(a, a:b, b, c, d, e, ...)
(see the explanation below).
Each substring s of the input string which is in the input language
- of the transducer \a mapping and whose left context is matched
+ of the transducer \a mapping and whose left context is matched
by the expression
- [.* l] (where l is the first element of \a context) and
- whose right context is matched by [r .*]
- (where r is the second element in the context) is mapped
+ [.* l] (where l is the first element of \a context) and
+ whose right context is matched by [r .*]
+ (where r is the second element in the context) is mapped
to the respective
- surface strings defined by transducer \a mapping. Any other
+ surface strings defined by transducer \a mapping. Any other
character is mapped to
- the characters specified in \a alphabet. The left and right
+ the characters specified in \a alphabet. The left and right
contexts must
be automata (i.e. transducers which map strings onto themselves).
@@ -1846,12 +1858,12 @@ alphabet = set(a, b, c)
\endverbatim
would map the string "caacac" to "cbbcac".
- Note that the alphabet must contain the characters a and b,
+ Note that the alphabet must contain the characters a and b,
but not the pair
- a:b (unless this replacement is to be allowed everywhere
+ a:b (unless this replacement is to be allowed everywhere
in the context).
- Note that replace operations (unlike the two-level rules)
+ Note that replace operations (unlike the two-level rules)
have to be combined by composition
rather than intersection.
@@ -1861,66 +1873,66 @@ alphabet = set(a, b, c)
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>
*/
- HFSTDLL HfstTransducer replace_up(HfstTransducerPair &context,
- HfstTransducer &mapping,
- bool optional,
+ HFSTDLL HfstTransducer replace_up(HfstTransducerPair &context,
+ HfstTransducer &mapping,
+ bool optional,
StringPairSet &alphabet);
- /** \brief The same as replace_up, but matching is done on
- the output side of \a mapping
+ /** \brief The same as replace_up, but matching is done on
+ the output side of \a mapping
- @see replace_up
+ @see replace_up
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HFSTDLL HfstTransducer replace_down(HfstTransducerPair &context,
- HfstTransducer &mapping,
- bool optional,
+ HFSTDLL HfstTransducer replace_down(HfstTransducerPair &context,
+ HfstTransducer &mapping,
+ bool optional,
StringPairSet &alphabet);
- HFSTDLL HfstTransducer replace_down_karttunen(HfstTransducerPair &context,
- HfstTransducer &mapping,
- bool optional,
+ HFSTDLL HfstTransducer replace_down_karttunen(HfstTransducerPair &context,
+ HfstTransducer &mapping,
+ bool optional,
StringPairSet &alphabet);
- /** \brief The same as replace_up, but left context matching is
+ /** \brief The same as replace_up, but left context matching is
done on the output side of \a mapping
- and right context on the input side of \a mapping
+ and right context on the input side of \a mapping
- @see replace_up
+ @see replace_up
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HFSTDLL HfstTransducer replace_right(HfstTransducerPair &context,
- HfstTransducer &mapping,
- bool optional,
+ HFSTDLL HfstTransducer replace_right(HfstTransducerPair &context,
+ HfstTransducer &mapping,
+ bool optional,
StringPairSet &alphabet);
- /** \brief The same as replace_up, but left context matching is
+ /** \brief The same as replace_up, but left context matching is
done on the input side of \a mapping
- and right context on the output side of \a mapping.
+ and right context on the output side of \a mapping.
- @see replace_up
+ @see replace_up
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HFSTDLL HfstTransducer replace_left(HfstTransducerPair &context,
- HfstTransducer &mapping,
- bool optional,
+ HFSTDLL HfstTransducer replace_left(HfstTransducerPair &context,
+ HfstTransducer &mapping,
+ bool optional,
StringPairSet &alphabet);
- /** \brief The same as replace_up but \a mapping is performed
- in every context.
+ /** \brief The same as replace_up but \a mapping is performed
+ in every context.
@see replace_up */
- HFSTDLL HfstTransducer replace_up(HfstTransducer &mapping,
- bool optional,
+ HFSTDLL HfstTransducer replace_up(HfstTransducer &mapping,
+ bool optional,
StringPairSet &alphabet);
/** \brief The same as replace_down(HfstTransducerPair&, HfstTransducer&, bool, StringPairSet&)
but \a mapping is performed in every context.
@see replace_up */
- HFSTDLL HfstTransducer replace_down(HfstTransducer &mapping,
- bool optional,
+ HFSTDLL HfstTransducer replace_down(HfstTransducer &mapping,
+ bool optional,
StringPairSet &alphabet);
/** \brief Inversion of the replace_up and the result needs to be composed on the upper side
@@ -1987,55 +1999,55 @@ SFST manual</a>. */
- /** \brief A transducer that allows any (substring) mapping defined
+ /** \brief A transducer that allows any (substring) mapping defined
by \a mapping
- only if it occurs in any of the contexts in \a contexts.
+ only if it occurs in any of the contexts in \a contexts.
Symbols outside of the matching
- substrings are mapped to any symbol allowed by \a alphabet.
+ substrings are mapped to any symbol allowed by \a alphabet.
@throws EmptySetOfContextsException
@see
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HFSTDLL HfstTransducer restriction(HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
+ HFSTDLL HfstTransducer restriction(HfstTransducerPairVector &contexts,
+ HfstTransducer &mapping,
StringPairSet &alphabet);
- /** \brief A transducer that requires that one of the mappings
+ /** \brief A transducer that requires that one of the mappings
defined by \a mapping
- must occur in each context in \a contexts. Symbols outside of
+ must occur in each context in \a contexts. Symbols outside of
the matching
substrings are mapped to any symbol allowed by \a alphabet.
@see
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HFSTDLL HfstTransducer coercion(HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
+ HFSTDLL HfstTransducer coercion(HfstTransducerPairVector &contexts,
+ HfstTransducer &mapping,
StringPairSet &alphabet);
- /** \brief A transducer that is equivalent to the intersection
+ /** \brief A transducer that is equivalent to the intersection
of restriction and coercion
- and requires that the mappings defined by \a mapping occur
+ and requires that the mappings defined by \a mapping occur
always and only in the
given contexts in \a contexts. Symbols outside of the matching
substrings are mapped to any symbol allowed by \a alphabet.
@see
- restriction(HfstTransducerPairVector&, HfstTransducer&, StringPairSet&)
- #coercion
+ restriction(HfstTransducerPairVector&, HfstTransducer&, StringPairSet&)
+ #coercion
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a> */
HFSTDLL HfstTransducer restriction_and_coercion(HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
+ HfstTransducer &mapping,
StringPairSet &alphabet);
- /** \brief A transducer that specifies that a string from
+ /** \brief A transducer that specifies that a string from
the input language of the
- transducer \a mapping may only be mapped to one of its
+ transducer \a mapping may only be mapped to one of its
output strings (according
- to transducer \a mapping) if it appears in any of the contexts
+ to transducer \a mapping) if it appears in any of the contexts
in \a contexts.
Symbols outside of the matching substrings are mapped
to any symbol allowed by \a alphabet.
@@ -2043,76 +2055,76 @@ SFST manual</a>. */
@see
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HFSTDLL HfstTransducer surface_restriction(HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
+ HFSTDLL HfstTransducer surface_restriction(HfstTransducerPairVector &contexts,
+ HfstTransducer &mapping,
StringPairSet &alphabet);
- /** \brief A transducer that specifies that a string from
+ /** \brief A transducer that specifies that a string from
the input language of the transducer
- \a mapping always has to the mapped to one of its output
+ \a mapping always has to the mapped to one of its output
strings according to
- transducer \a mapping if it appears in any of the contexts
+ transducer \a mapping if it appears in any of the contexts
in \a contexts.
- Symbols outside of the matching substrings are mapped to
+ Symbols outside of the matching substrings are mapped to
any symbol allowed by \a alphabet.
@see
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HFSTDLL HfstTransducer surface_coercion(HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
+ HFSTDLL HfstTransducer surface_coercion(HfstTransducerPairVector &contexts,
+ HfstTransducer &mapping,
StringPairSet &alphabet);
- /** \brief A transducer that is equivalent to the intersection of
+ /** \brief A transducer that is equivalent to the intersection of
surface_restriction and surface_coercion.
- @see #surface_restriction #surface_coercion
+ @see #surface_restriction #surface_coercion
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
HFSTDLL HfstTransducer surface_restriction_and_coercion
- (HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
+ (HfstTransducerPairVector &contexts,
+ HfstTransducer &mapping,
StringPairSet &alphabet);
- /** \brief A transducer that specifies that a string from
+ /** \brief A transducer that specifies that a string from
the output language of the transducer
- \a mapping may only be mapped to one of its input strings
+ \a mapping may only be mapped to one of its input strings
(according to transducer \a mappings)
if it appears in any of the contexts in \a contexts.
- Symbols outside of the matching substrings are mapped
+ Symbols outside of the matching substrings are mapped
to any symbol allowed by \a alphabet.
@see
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HFSTDLL HfstTransducer deep_restriction(HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
+ HFSTDLL HfstTransducer deep_restriction(HfstTransducerPairVector &contexts,
+ HfstTransducer &mapping,
StringPairSet &alphabet);
- /** \brief A transducer that specifies that a string from
+ /** \brief A transducer that specifies that a string from
the output language of the transducer
- \a mapping always has to be mapped to one of its input strings
+ \a mapping always has to be mapped to one of its input strings
(according to transducer \a mappings)
if it appears in any of the contexts in \a contexts.
- Symbols outside of the matching substrings are mapped
+ Symbols outside of the matching substrings are mapped
to any symbol allowed by \a alphabet.
@see
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HFSTDLL HfstTransducer deep_coercion(HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
+ HFSTDLL HfstTransducer deep_coercion(HfstTransducerPairVector &contexts,
+ HfstTransducer &mapping,
StringPairSet &alphabet);
- /** \brief A transducer that is equivalent to the intersection
+ /** \brief A transducer that is equivalent to the intersection
of deep_restriction and deep_coercion.
- @see #deep_restriction #deep_coercion
+ @see #deep_restriction #deep_coercion
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
HFSTDLL HfstTransducer deep_restriction_and_coercion
- (HfstTransducerPairVector &contexts,
- HfstTransducer &mapping,
+ (HfstTransducerPairVector &contexts,
+ HfstTransducer &mapping,
StringPairSet &alphabet);
}
diff --git a/libhfst/src/HfstXeroxRules.cc b/libhfst/src/HfstXeroxRules.cc
index e818911..790ce46 100644
--- a/libhfst/src/HfstXeroxRules.cc
+++ b/libhfst/src/HfstXeroxRules.cc
@@ -1,3136 +1,3151 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
-// information.
-
-#include "HfstXeroxRules.h"
-
-#ifndef MAIN_TEST
-
-namespace hfst
-{
- namespace xeroxRules
- {
- Rule::Rule ( const HfstTransducerPairVector &mappingPairVector )
- {
- HfstTokenizer TOK;
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
-
- ImplementationType type = mappingPairVector[0].first.get_type();
- // Check if all transducer types are the same
- for ( unsigned int i = 0; i < mappingPairVector.size(); i++ )
- {
- if ( mappingPairVector[i].first.get_type() != type
- || mappingPairVector[i].second.get_type() != type)
- {
- HFST_THROW_MESSAGE(TransducerTypeMismatchException, "Rule mapping");
- }
- }
-
- HfstTransducerPair contextPair(HfstTransducer("@_EPSILON_SYMBOL_@", TOK, type),
- HfstTransducer("@_EPSILON_SYMBOL_@", TOK, type));
- HfstTransducerPairVector epsilonContext;
- epsilonContext.push_back(contextPair);
-
- mapping = mappingPairVector;
-
- // HfstTransducerPairVector tmpV = mappingPairVector;
- // tmpV[0].first = encodeFlagDiacritics(tmpV[0].first);
-
- //mapping = tmpV;
- context = epsilonContext;
- replType = REPL_UP;
-
- }
- Rule::Rule ( const HfstTransducerPairVector &mappingPairVector,
- const HfstTransducerPairVector &a_context,
- ReplaceType a_replType )
- {
- // Check if all transducer types are the same
- ImplementationType type = mappingPairVector[0].first.get_type();
- for ( unsigned int i = 0; i < mappingPairVector.size(); i++ )
- {
- if ( mappingPairVector[i].first.get_type() != type
- || mappingPairVector[i].second.get_type() != type)
- {
- HFST_THROW_MESSAGE(TransducerTypeMismatchException, "Rule mapping");
- }
- }
- for ( unsigned int j = 0; j < a_context.size(); j++ )
- {
- if ( a_context[j].first.get_type() != type
- || a_context[j].second.get_type() != type )
- {
- HFST_THROW_MESSAGE(TransducerTypeMismatchException, "Rule context");
- }
- }
-
-
-
- //HfstTransducerPairVector tmpV = mappingPairVector;
- //tmpV[0].first = encodeFlagDiacritics(tmpV[0].first);
-
- mapping = mappingPairVector;
- // mapping = tmpV ;
- context = a_context;
- replType = a_replType;
- }
-
-
- Rule::Rule ( const Rule &a_rule )
- {
- mapping = a_rule.get_mapping();
- context = a_rule.get_context();
- replType = a_rule.get_replType();
-
- }
-
- HfstTransducerPairVector Rule::get_mapping() const
- {
- return mapping;
- }
-
- HfstTransducerPairVector Rule::get_context() const
- {
- return context;
- }
- ReplaceType Rule::get_replType() const
- {
- return replType;
- }
- void Rule::encodeFlags()
- {
- //cerr << "\n Encode flags " << endl;
- HfstTransducerPairVector tmpM = this->mapping;
-
- //cerr << "\n mapping size: " << tmpM.size() << endl;
- for ( unsigned int i = 0; i < tmpM.size(); i++ )
- {
- tmpM[i].first = encodeFlagDiacritics(tmpM[i].first);
- tmpM[i].second = encodeFlagDiacritics(tmpM[i].second);
- }
-
-
- HfstTransducerPairVector tmpC = this->context;
-
- //cerr << "\n context size: " << tmpC.size() << endl;
- for ( unsigned int i = 0; i < tmpC.size(); i++ )
- {
- tmpC[i].first = encodeFlagDiacritics(tmpC[i].first);
- tmpC[i].second = encodeFlagDiacritics(tmpC[i].second);
- }
-
- this->mapping = tmpM;
- this->context = tmpC;
- }
-
- std::ostream & operator<<(std::ostream &out, const Rule & r)
- {
- out << "hfst::xeroxRules::Rule:" << std::endl;
- out << "replType: ";
- switch(r.replType)
- {
- case REPL_UP:
- out << "REPL_UP";
- break;
- case REPL_DOWN:
- out << "REPL_DOWN";
- break;
- case REPL_RIGHT:
- out << "REPL_RIGHT";
- break;
- case REPL_LEFT:
- out << "REPL_LEFT";
- break;
- }
- out << std::endl;
-
- out << "mapping:" << std::endl;
- unsigned int index = 1;
- for (HfstTransducerPairVector::const_iterator it = r.mapping.begin();
- it != r.mapping.end(); it++)
- {
- out << "#" << index << " (right side):" << std::endl;
- out << it->first;
- out << "#" << index << " (left side):" << std::endl;
- out << it->second;
- index++;
- }
-
- out << "context:" << std::endl;
- index = 1;
- for (HfstTransducerPairVector::const_iterator it = r.context.begin();
- it != r.context.end(); it++)
- {
- out << "#" << index << " (right side):" << std::endl;
- out << it->first;
- out << "#" << index << " (left side):" << std::endl;
- out << it->second;
- index++;
- }
-
- return out;
- }
-
-
- //////////////////////////////////////
- // In the transducer tr, change all flag diacritics to "non-special" multichar symbols
- // It means that @ sign will be changed to $ sign
- // ie. @P.FOO.BAR@ will be changed into $P.FOO.BAR$
- HfstTransducer encodeFlagDiacritics( const HfstTransducer &tr )
- {
- //std::cerr << "\n encodeFlagDiacritics " << endl;
- HfstSymbolSubstitutions realFlagstoFakeFlags;
- StringSet removeFromAlphabet;
- //printf("alphabet lexicons: \n");
- StringSet transducerAlphabet = tr.get_alphabet();
- for (StringSet::const_iterator s = transducerAlphabet.begin();
- s != transducerAlphabet.end();
- ++s)
- {
-
- String alph = *s;
- String alphFirst3 = alph.substr(0,3);
- // String alphLast = alph.back();
-
- //@operator.feature.value@ and @operator.feature@
-
- //std::cout << "debug: " << alphFirst3 << endl;
- if ( alphFirst3 == "@P." || alphFirst3 == "@R."
- || alphFirst3 == "@U." || alphFirst3 == "@D."
- || alphFirst3 == "@C." || alphFirst3 == "@N."
- ||alphFirst3 == "@p." || alphFirst3 == "@r."
- || alphFirst3 == "@u." || alphFirst3 == "@d."
- || alphFirst3 == "@c." || alphFirst3 == "@n.")
- {
-
- replace(alph.begin(), alph.end(), '@', '$');
- //std::cout << alph << '\n';
- realFlagstoFakeFlags.insert(StringPair(*s, alph));
- removeFromAlphabet.insert(*s);
- }
- }
-
- HfstTransducer retval(tr);
- retval.substitute(realFlagstoFakeFlags);
-
- //std::cerr << "\n flag substitute done" << endl;
- retval.remove_from_alphabet(removeFromAlphabet);
- //std::cerr << "\n alph substitute done" << endl;
- return retval;
-
- }
-
- HfstTransducer decodeFlagDiacritics( const HfstTransducer &tr )
- {
- //std::cerr << "\n decodeFlagDiacritics " << endl;
- HfstSymbolSubstitutions fakeFlagsToRealFlags;
-
- //printf("alphabet lexicons: \n");
- StringSet transducerAlphabet = tr.get_alphabet();
- StringSet removeFromAlphabet;
- for (StringSet::const_iterator s = transducerAlphabet.begin();
- s != transducerAlphabet.end();
- ++s)
- {
-
- String alph = *s;
- String alphFirst3 = alph.substr(0,3);
- // String alphLast = alph.back();
-
- //@operator.feature.value@ and @operator.feature@
-
- //std::cout << "debug: " << alphFirst3 << endl;
- if ( alphFirst3 == "$P." || alphFirst3 == "$R."
- || alphFirst3 == "$U." || alphFirst3 == "$D."
- || alphFirst3 == "$C." || alphFirst3 == "$N."
- || alphFirst3 == "$p." || alphFirst3 == "$r."
- || alphFirst3 == "$u." || alphFirst3 == "$d."
- || alphFirst3 == "$c." || alphFirst3 == "$n.")
- {
-
- replace(alph.begin(), alph.end(), '$', '@');
- //std::cout << alph << '\n';
- fakeFlagsToRealFlags.insert(StringPair(*s, alph));
- removeFromAlphabet.insert(*s);
- }
- }
-
- HfstTransducer retval(tr);
- retval.substitute(fakeFlagsToRealFlags);
- retval.remove_from_alphabet(removeFromAlphabet);
- return retval;
-
- }
-
-
-
- HfstTransducer disjunctVectorMembers( const HfstTransducerVector &trVector )
- {
- HfstTransducer retval( trVector[0] );
- for ( unsigned int i = 1; i < trVector.size(); i++ )
- {
- retval.disjunct(trVector[i]).minimize();
- }
- return retval;
- }
-
-
-
- //////////////////////////////////////
-
-
-
- HfstTransducer removeMarkers( const HfstTransducer &tr )
- {
- HfstTransducer retval(tr);
-
-
- String leftMarker("@LM@");
- String rightMarker("@RM@");
-
- retval.substitute(StringPair(leftMarker, leftMarker), StringPair("@_EPSILON_SYMBOL_@", "@_EPSILON_SYMBOL_@")).minimize();
- retval.substitute(StringPair(rightMarker, rightMarker), StringPair("@_EPSILON_SYMBOL_@", "@_EPSILON_SYMBOL_@")).minimize();
-
-
- retval.remove_from_alphabet(leftMarker);
- retval.remove_from_alphabet(rightMarker);
-
- retval.minimize();
-
-
-
- //printf("tr without markers: \n");
- //retval.write_in_att_format(stdout, 1);
-
- retval = decodeFlagDiacritics(retval);
-
- return retval;
- }
-
- float zero_weight(float f) {
- return 0;
- }
-
- /*
- * Generalized Lenient Composition, described in Anssi Yli-Jyrä 2008b
- */
- // tmp = t.1 .o. Constr .o. t.1
- // (t.1 - tmp.2) .o. t
- HfstTransducer constraintComposition( const HfstTransducer &t, const HfstTransducer &Constraint )
- {
-
-// printf("---- \n");
-// printf("orig: \n");
-// t.write_in_att_format(stdout, 1);
-// printf("---- \n");
-
- HfstTransducer retval(t);
- retval.transform_weights(&zero_weight);
-
- retval.input_project().minimize();
-
- HfstTransducer tmp(retval);
- tmp.compose(Constraint).minimize();
- //printf("tmp: \n");
- //tmp.write_in_att_format(stdout, 1);
-
- tmp.compose(retval).minimize();
- //printf("tmp 2: \n");
- //tmp.write_in_att_format(stdout, 1);
- tmp.output_project().minimize();
- retval.subtract(tmp).minimize();
-
- //transform weights to zero
- retval.transform_weights(&zero_weight);
- retval.compose(t).minimize();
-
- return retval;
- }
-
- void insertFreelyAllTheBrackets( HfstTransducer &t, bool optional )
- {
- HfstTokenizer TOK;
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
- String leftMarker("@LM@");
- String rightMarker("@RM@");
- String leftMarker2("@LM2@");
- String rightMarker2("@RM2@");
-
- TOK.add_multichar_symbol(leftMarker);
- TOK.add_multichar_symbol(rightMarker);
- TOK.add_multichar_symbol(leftMarker2);
- TOK.add_multichar_symbol(rightMarker2);
-
- ImplementationType type = t.get_type();
-
- HfstTransducer leftBracket(leftMarker, TOK, type);
- HfstTransducer rightBracket(rightMarker, TOK, type);
-
- t.insert_freely(leftBracket, false).minimize();
- t.insert_freely(rightBracket, false).minimize();
-
- if ( !optional )
- {
- HfstTransducer leftBracket2(leftMarker2, TOK, type);
- HfstTransducer rightBracket2(rightMarker2, TOK, type);
-
- t.insert_freely(leftBracket2, false).minimize();
- t.insert_freely(rightBracket2, false).minimize();
- }
- }
-
- HfstTransducer expandContextsWithMapping (const HfstTransducerPairVector &ContextVector,
- const HfstTransducer &mappingWithBracketsAndTmpBoundary,
- const HfstTransducer &identityExpanded,
- ReplaceType replType,
- bool optional)
- {
- ImplementationType type = identityExpanded.get_type();
-
- HfstTransducer unionContextReplace(type);
-
- HfstTokenizer TOK;
- // TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
- // HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
-
-
- for ( unsigned int i = 0; i < ContextVector.size(); i++ )
- {
-
- // Expand context with mapping
- // Cr' = (Rc .*) << Markers (<,>,|) .o. [I:I | <a:b>]*
- // Cr = Cr|Cr'
- // (same for left context)
-
- // Lc = (*. Lc) << {<,>}
-
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- HfstTransducer identityStar(identityPair);
- identityStar.repeat_star();
-
- HfstTransducer firstContext( identityStar);
- firstContext.concatenate(ContextVector[i].first);
- firstContext.transform_weights(&zero_weight);
- firstContext.minimize();
-
- insertFreelyAllTheBrackets( firstContext, optional );
-
- // Rc = (Rc .*) << {<,>}
- HfstTransducer secondContext(ContextVector[i].second);
- secondContext.concatenate(identityStar);
- secondContext.transform_weights(&zero_weight);
- secondContext.minimize();
- insertFreelyAllTheBrackets( secondContext, optional);
-
- /* RULE: LC: RC:
- * up up up
- * left up down
- * right down up
- * down down down
- */
-
- HfstTransducer leftContextExpanded(type);
- HfstTransducer rightContextExpanded(type);
-
- // both contexts are in upper language
- if ( replType == REPL_UP)
- {
- // compose them with [I:I | <a:b>]*
- leftContextExpanded = firstContext;
- rightContextExpanded = secondContext;
-
- leftContextExpanded.compose(identityExpanded);
- rightContextExpanded.compose(identityExpanded);
- }
- // left context is in lower language, right in upper ( // )
- if ( replType == REPL_RIGHT )
- {
- // compose them with [I:I | <a:b>]*
-
- // left compose opposite way
- leftContextExpanded = identityExpanded;
- rightContextExpanded = secondContext;
-
- leftContextExpanded.compose(firstContext);
- rightContextExpanded.compose(identityExpanded);
- }
- // right context is in lower language, left in upper ( \\ )
- if ( replType == REPL_LEFT )
- {
- // compose them with [I:I | <a:b>]*
- leftContextExpanded = firstContext;
- rightContextExpanded = identityExpanded;
-
- leftContextExpanded.compose(identityExpanded);
- rightContextExpanded.compose(secondContext);
- }
- if ( replType == REPL_DOWN )
- {
- // compose them with [I:I | <a:b>]*
- leftContextExpanded = identityExpanded;
- rightContextExpanded = identityExpanded;
-
- leftContextExpanded.compose(firstContext);
- rightContextExpanded.compose(secondContext);
- }
-
-
- leftContextExpanded.transform_weights(&zero_weight);
- rightContextExpanded.transform_weights(&zero_weight);
- leftContextExpanded.minimize();
- rightContextExpanded.minimize();
-
- firstContext.disjunct(leftContextExpanded);
- firstContext.minimize();
-
- secondContext.disjunct(rightContextExpanded);
- secondContext.minimize();
-
- // add boundary symbol before/after contexts
- String boundaryMarker(".#.");
- TOK.add_multichar_symbol(boundaryMarker);
- HfstTransducer boundary(boundaryMarker, TOK, type);
-
- identityStar.insert_to_alphabet(boundaryMarker);
-
- // to firstContext
- StringSet firstContextAlphabet = firstContext.get_alphabet();
- bool hasBoundary = false;
- for (StringSet::const_iterator s = firstContextAlphabet.begin();
- s != firstContextAlphabet.end();
- ++s)
- {
- if( boundaryMarker.compare(s->c_str()) == 0 )
- hasBoundary = true;
- }
-
- if ( hasBoundary == false )
- {
- //printf("\n - Left context boundary false! - \n");
- firstContext.insert_to_alphabet(boundaryMarker);
- HfstTransducer tmp(boundary);
- tmp.concatenate(identityStar).minimize();
- //printf("tmp \n");
- //tmp.write_in_att_format(stdout, 1);
- tmp.concatenate(firstContext);
-
- //printf("tmp2 \n");
- //tmp.write_in_att_format(stdout, 1);
- firstContext = tmp;
-
- }
-
- // to secondContext
- StringSet secondContextAlphabet = secondContext.get_alphabet();
- hasBoundary = false;
- for (StringSet::const_iterator s = secondContextAlphabet.begin();
- s != secondContextAlphabet.end();
- ++s)
- {
- if( boundaryMarker.compare(s->c_str()) == 0 )
- hasBoundary = true;
- }
-
- if ( hasBoundary == false )
- {
- // printf("\n - Right context boundary false! - \n");
- secondContext.insert_to_alphabet(boundaryMarker);
- secondContext.concatenate(identityStar).concatenate(boundary).minimize();
- }
-
- // put mapping between (expanded) contexts
- HfstTransducer oneContextReplace(firstContext);
- oneContextReplace.concatenate(mappingWithBracketsAndTmpBoundary).
- concatenate(secondContext);
-
-
- oneContextReplace.transform_weights(&zero_weight);
- unionContextReplace.disjunct(oneContextReplace);
- unionContextReplace.minimize();
-
- }
- return unionContextReplace;
- }
-
-
-
- /*
- * unconditional replace, in multiple contexts
- * first: (.* T<a:b>T .*) - [( .* L1 T<a:b>T R1 .*) u (.* L2 T<a:b>T R2 .*)...],
- * where .* = [I:I (+ {tmpMarker (T), <,>} in alphabet) | <a:b>]*
- * then: remove tmpMarker from transducer and alphabet, and do negation:
- * .* - result from upper operations
- *
- * Also note that context is expanded in this way:
- * Cr' = (Rc .*) << Markers (<,>,|) .o. [I:I | <a:b>]*
- * Cr = Cr | Cr'
- * (same for left context, (.* Cl))
- */
-
- HfstTransducer bracketedReplace(const Rule &rule, bool optional)
- {
- //printf("bracketedReplace function..... \n");
-
- HfstTokenizer TOK;
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
- TOK.add_multichar_symbol("@_UNKNOWN_SYMBOL_@");
- String leftMarker("@LM@");
- String rightMarker("@RM@");
- String tmpMarker("@TMPM@");
- String leftMarker2("@LM2@");
- String rightMarker2("@RM2@");
- String newEpsilon("$Epsilon$");
-
- TOK.add_multichar_symbol(leftMarker);
- TOK.add_multichar_symbol(rightMarker);
- TOK.add_multichar_symbol(leftMarker2);
- TOK.add_multichar_symbol(rightMarker2);
- TOK.add_multichar_symbol(tmpMarker);
- TOK.add_multichar_symbol(newEpsilon);
- TOK.add_multichar_symbol( ".#.");
-
- //first, encode all flag diacritics
- Rule ruletmp(rule);
- ruletmp.encodeFlags();
-
- HfstTransducerPairVector mappingPairVector( ruletmp.get_mapping() );
- HfstTransducerPairVector ContextVector( ruletmp.get_context() );
- ReplaceType replType( ruletmp.get_replType() );
-
- ImplementationType type = mappingPairVector[0].first.get_type();
-
- // Identity (normal)
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- HfstTransducer identity (identityPair);
- identity.repeat_star().minimize();
-
-
- HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
- HfstTransducer mapping(type);
- for ( unsigned int i = 0; i < mappingPairVector.size(); i++ )
- {
- HfstTransducer oneMappingPair(mappingPairVector[i].first);
-
- //markup rules are already cross product in mappingPairVector[i].first (second is empty)
- //so the cross product should not be done for markup rules
- if ( mappingPairVector[i].first.get_property("isMarkup") != "yes" )
- {
- oneMappingPair.cross_product(mappingPairVector[i].second);
- }
- // printf("aftrer cross product \n");
- // oneMappingPair.write_in_att_format(stdout, 1);
-
-
- // for removing .#. from the center
- HfstTransducer identityWithoutBoundary(identity);
- identityWithoutBoundary.insert_to_alphabet(".#.");
- HfstTransducer removeHash(identityWithoutBoundary);
- HfstTransducer boundary(".#.", TOK, type);
- removeHash.concatenate(boundary).concatenate(identityWithoutBoundary).minimize();
- //printf("removeHash \n");
- //removeHash.write_in_att_format(stdout, 1);
-
-
- // printf("oneMappingPair kkkk\n");
- // oneMappingPair.write_in_att_format(stdout, 1);
-
-
- if ( i == 0 )
- {
- // remove .#. from the center
- // center - (?* .#. ?*)
- oneMappingPair.subtract(removeHash, false).minimize();
- oneMappingPair.remove_from_alphabet(".#.");
- mapping = oneMappingPair;
- }
- else
- {
- oneMappingPair.subtract(removeHash, false).minimize();
- oneMappingPair.remove_from_alphabet(".#.");
- mapping.disjunct(oneMappingPair).minimize();
- }
- }
- //printf("mapping all after cross product \n");
- // mapping.write_in_att_format(stdout, 1);
-
-
- // In case of ? -> x replacement
- // If left side is empty, return identity transducer
- // If right side is empty, return identity transducer
- // with alphabet from the left side
- HfstTransducer empty(type);
- if ( mapping.compare(empty) )
- {
- mapping = identity;
- if (mappingPairVector[0].second.compare(empty))
- {
- //printf("alphabet: \n");
- StringSet transducerAlphabet = mappingPairVector[0].first.get_alphabet();
- for (StringSet::const_iterator s = transducerAlphabet.begin();
- s != transducerAlphabet.end();
- ++s)
- {
- //printf("%s \n", s->c_str());
- mapping.insert_to_alphabet(s->c_str());
- }
- //printf("------------------ \n");
- }
- }
-
- mapping.insert_to_alphabet(leftMarker);
- mapping.insert_to_alphabet(rightMarker);
- mapping.insert_to_alphabet(tmpMarker);
-
- HfstTransducer leftBracket(leftMarker, TOK, type);
- HfstTransducer rightBracket(rightMarker, TOK, type);
- HfstTransducer leftBracket2(leftMarker2, TOK, type);
- HfstTransducer rightBracket2(rightMarker2, TOK, type);
- HfstTransducer tmpBracket(tmpMarker, TOK, type);
-
- // Surround mapping with brackets
- HfstTransducer tmpMapping(leftBracket);
- tmpMapping.concatenate(mapping).concatenate(rightBracket).minimize();
-
- HfstTransducer mappingWithBrackets(tmpMapping);
-
- //printf("mappingWithBrackets: \n");
- //mappingWithBrackets.minimize().write_in_att_format(stdout, 1);
-
- // Identity pair
- // HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- // for non-optional replacements
- if ( optional != true )
- {
- // non - optional
- // mapping = <a:b> u <2a:a>2
-
- HfstTransducer mappingWithBrackets2(leftBracket2);
- HfstTransducer leftMappingUnion(type);
- leftMappingUnion = mappingPairVector[0].first;
- for ( unsigned int i = 1; i < mappingPairVector.size(); i++ )
- {
- leftMappingUnion.disjunct(mappingPairVector[i].first).minimize();
- }
- // needed in case of ? -> x replacement
- leftMappingUnion.insert_to_alphabet(leftMarker2);
- leftMappingUnion.insert_to_alphabet(rightMarker2);
- leftMappingUnion.insert_to_alphabet(leftMarker);
- leftMappingUnion.insert_to_alphabet(rightMarker);
- leftMappingUnion.insert_to_alphabet(tmpMarker);
-
- //printf("leftMappingUnion: \n");
- //leftMappingUnion.minimize().write_in_att_format(stdout, 1);
-
-
- mappingWithBrackets2.concatenate(leftMappingUnion).concatenate(rightBracket2).minimize();
-
- //printf("mappingWithBrackets2: \n");
- //mappingWithBrackets2.minimize().write_in_att_format(stdout, 1);
-
- // mappingWithBrackets...... expanded
- mappingWithBrackets.insert_to_alphabet(leftMarker2);
- mappingWithBrackets.insert_to_alphabet(rightMarker2);
- //mappingWithBrackets.insert_to_alphabet(leftMarker);
- //mappingWithBrackets.insert_to_alphabet(rightMarker);
- mappingWithBrackets.disjunct(mappingWithBrackets2).minimize();
- }
-
- //printf("mappingWithBrackets: \n");
- //mappingWithBrackets.minimize().write_in_att_format(stdout, 1);
-
-
- // Identity with bracketed mapping and marker symbols and TmpMarker in alphabet
- // [I:I | <a:b>]* (+ tmpMarker in alphabet)
- HfstTransducer identityExpanded (identityPair);
-
- identityExpanded.insert_to_alphabet(leftMarker);
- identityExpanded.insert_to_alphabet(rightMarker);
- identityExpanded.insert_to_alphabet(tmpMarker);
-
- if ( optional != true )
- {
- identityExpanded.insert_to_alphabet(leftMarker2);
- identityExpanded.insert_to_alphabet(rightMarker2);
- }
-
- identityExpanded.disjunct(mappingWithBrackets).minimize();
- identityExpanded.repeat_star().minimize();
-
- // when there aren't any contexts, result is identityExpanded
- if ( ContextVector.size() == 1 )
- {
- HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
- if ( ContextVector[0].first.compare(epsilon) && ContextVector[0].second.compare(epsilon) )
- {
- identityExpanded.remove_from_alphabet(tmpMarker);
- //printf("identityExpanded: \n");
- //identityExpanded.minimize().write_in_att_format(stdout, 1);
- return identityExpanded;
- }
- }
-
- // Surround mapping with tmp boudaries
- HfstTransducer mappingWithBracketsAndTmpBoundary(tmpBracket);
- mappingWithBracketsAndTmpBoundary.concatenate(mappingWithBrackets).concatenate(tmpBracket).minimize();
- //printf("mappingWithBracketsAndTmpBoundary: \n");
- //mappingWithBracketsAndTmpBoundary.write_in_att_format(stdout, 1);
-
-
- // .* |<a:b>| :*
- HfstTransducer bracketedReplace(identityExpanded);
- bracketedReplace.concatenate(mappingWithBracketsAndTmpBoundary).concatenate(identityExpanded).minimize();
-
- //printf("mappingWithBracketsAndTmpBoundary: \n");
- //mappingWithBracketsAndTmpBoundary.write_in_att_format(stdout, 1);
-
- // printf("identityExpanded: \n");
- //identityExpanded.write_in_att_format(stdout, 1);
-
- // Expand all contexts with mapping taking in consideration replace type
- // result is their union
- HfstTransducer unionContextReplace(type);
- unionContextReplace = expandContextsWithMapping (ContextVector,
- mappingWithBracketsAndTmpBoundary,
- identityExpanded,
- replType,
- optional);
-
- //printf("unionContextReplace \n");
- //unionContextReplace.write_in_att_format(stdout, 1);
-
- // subtract all mappings in contexts from replace without contexts
- HfstTransducer replaceWithoutContexts(bracketedReplace);
- replaceWithoutContexts.subtract(unionContextReplace).minimize();
-
- //printf("replaceWithoutContexts \n");
- //replaceWithoutContexts.write_in_att_format(stdout, 1);
-
- // remove tmpMaprker
- replaceWithoutContexts.substitute(StringPair(tmpMarker, tmpMarker),
- StringPair("@_EPSILON_SYMBOL_@", "@_EPSILON_SYMBOL_@")).minimize();
- replaceWithoutContexts.remove_from_alphabet(tmpMarker);
- replaceWithoutContexts.minimize();
-
- identityExpanded.remove_from_alphabet(tmpMarker);
-
- // final negation
- HfstTransducer uncondidtionalTr(identityExpanded);
- uncondidtionalTr.subtract(replaceWithoutContexts).minimize();
-
- return uncondidtionalTr;
-
- }
-
-
- // Return the string "@N@" where N is the string representation of i.
- static std::string getMarkerString(unsigned int i)
- {
- std::ostringstream oss;
- oss << i;
- return std::string("@") + oss.str() + std::string("@");
- }
-
- // Return the number representation of N in string "@N@".
- static unsigned int getMarkerNumber(const std::string & str)
- {
- std::string number_str = str.substr(1, str.size()-2);
- std::istringstream iss; // should be iss(number_str); i guess, but cannot be fixed, because some HfstXeroxRules tests will fail...
- unsigned int retval;
- iss >> retval;
- //return retval;
- return 100000;
- }
-
- // Bracketed replace for parallel rules.
- HfstTransducer parallelBracketedReplace
- (const std::vector<Rule> &ruleVector, bool optional)
- {
- // For each parallel rule, we need to concatenate a special marker symbol
- // to its output side. This is needed so that overlapping mappings with
- // different weights and contexts are kept separate. If we have N rules,
- // we need marker symbols "@1@", "@2@", ... , "@N@" ("@0@" is reserved
- // for epsilon symbol). At the end, we must substitute any marker symbols
- // with epsilons.
- // For an example, consider the following case:
- // a -> b::1 || c _ c ,, a -> b::2 || d _ d
- // If no markers are added, input 'dad' will yield 'dbd' with weight 1
- // because the transition <a:b::2> will be eliminated by the transition
- // <a:b::1> as it has a lower weight.
-
- StringSet marker_symbols; // "@1@", "@2@", ... , "@N@"
- HfstSymbolSubstitutions marker_substitutions; // any-marker-to-epsilon
- for (unsigned int i=0; i < ruleVector.size(); i++)
- {
- std::string marker_string = getMarkerString(i+1);
- marker_symbols.insert(marker_string);
- marker_substitutions[marker_string] = hfst::internal_epsilon;
- }
-
- HfstTokenizer TOK;
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
- String leftMarker("@LM@");
- String rightMarker("@RM@");
-
- String leftMarker2("@LM2@");
- String rightMarker2("@RM2@");
-
- String tmpMarker("@TMPM@");
- TOK.add_multichar_symbol(leftMarker);
- TOK.add_multichar_symbol(rightMarker);
- TOK.add_multichar_symbol(leftMarker2);
- TOK.add_multichar_symbol(rightMarker2);
- TOK.add_multichar_symbol(tmpMarker);
- TOK.add_multichar_symbol( ".#.");
-
- ImplementationType type
- = ruleVector[0].get_mapping()[0].first.get_type();
-
- HfstTransducer leftBracket(leftMarker, TOK, type);
- HfstTransducer rightBracket(rightMarker, TOK, type);
- HfstTransducer leftBracket2(leftMarker2, TOK, type);
- HfstTransducer rightBracket2(rightMarker2, TOK, type);
- HfstTransducer tmpBracket(tmpMarker, TOK, type);
-
-
- // Identity pair (unknowns/identities must not be expanded to marker
- // symbols)
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- identityPair.insert_to_alphabet(marker_symbols);
-
- HfstTransducer identity (identityPair);
- // unknowns/identities must not be expanded to marker symbols
- identity.insert_to_alphabet(marker_symbols);
- identity.repeat_star().minimize();
-
- HfstTransducer identityExpanded(identityPair);
- identityExpanded.insert_to_alphabet(leftMarker);
- identityExpanded.insert_to_alphabet(rightMarker);
- identityExpanded.insert_to_alphabet(leftMarker2);
- identityExpanded.insert_to_alphabet(rightMarker2);
- identityExpanded.insert_to_alphabet(tmpMarker);
- identityExpanded.insert_to_alphabet(marker_symbols);
- // will be expanded with mappings
-
- // for removing .#. from the center
- HfstTransducer identityWithoutBoundary(identity);
- identityWithoutBoundary.insert_to_alphabet(".#.");
- // (must not be expanded to marker symbols)
- identityWithoutBoundary.insert_to_alphabet(marker_symbols);
- HfstTransducer removeHash(identityWithoutBoundary);
- HfstTransducer boundary(".#.", TOK, type);
- removeHash.concatenate(boundary)
- .concatenate(identityWithoutBoundary).minimize();
- //printf("removeHash \n");
- //removeHash.write_in_att_format(stdout, 1);
-
- HfstTransducerVector mappingWithBracketsVector;
- bool noContexts = true;
-
- // go through vector and do everything for each rule
- for ( unsigned int i = 0; i < ruleVector.size(); i++ )
- {
- Rule ruletmp(ruleVector[i]);
- ruletmp.encodeFlags();
-
- HfstTransducerPairVector mappingPairVector
- = ruletmp.get_mapping();
- HfstTransducer mapping(type);
- for ( unsigned int j = 0; j < mappingPairVector.size(); j++ )
- {
- // i+1 because @0@ is epsilon..
- std::string marker_string = getMarkerString(i+1);
- HfstTransducer marker(marker_string, type);
- HfstTransducer oneMappingPair(mappingPairVector[j].first);
- // unknowns/identities must not be expanded to marker symbols
- oneMappingPair.insert_to_alphabet(marker_symbols);
- HfstTransducer foo(mappingPairVector[j].second);
- foo.insert_to_alphabet(marker_symbols);
- oneMappingPair.cross_product
- (foo.concatenate(marker));
- //oneMappingPair.cross_product(mappingPairVector[j].second);
-
- //printf("oneMappingPair \n");
- //oneMappingPair.write_in_att_format(stdout, 1);
-
-
- if ( j == 0 )
- {
- // remove .#. from the center
- // center - (?* .#. ?*)
- oneMappingPair.subtract(removeHash, false).minimize();
- oneMappingPair.remove_from_alphabet(".#.");
- mapping = oneMappingPair;
- }
- else
- {
- oneMappingPair.subtract(removeHash, false).minimize();
- oneMappingPair.remove_from_alphabet(".#.");
- mapping.disjunct(oneMappingPair).minimize();
- }
- }
-
-
- HfstTransducerPairVector contextVector = ruletmp.get_context();
- //ReplaceType replaceType = ruletmp.get_replType();
-
- // when there aren't any contexts, result is identityExpanded
- if ( contextVector.size() == 1 )
- {
- HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
- if ( !(contextVector[0].first.compare(epsilon) &&
- contextVector[0].second.compare(epsilon)) )
- {
- noContexts = false;
- }
- }
-
- //////////////////////////////////////////////////////////////////
- // In case of ? -> x replacement
- // If left side is empty, return identity transducer
- // If right side is empty, return identity transducer
- // with alphabet from the left side
- HfstTransducer empty(type);
-
- if ( mapping.compare(empty) )
- {
- mapping = identity;
- if (mappingPairVector[0].second.compare(empty))
- {
- //printf("alphabet: \n");
- StringSet transducerAlphabet
- = mappingPairVector[0].first.get_alphabet();
- for (StringSet::const_iterator s
- = transducerAlphabet.begin();
- s != transducerAlphabet.end();
- ++s)
- {
- //printf("%s \n", s->c_str());
- mapping.insert_to_alphabet(s->c_str());
- }
- //printf("------------------ \n");
- }
- }
- //////////////////////////////////////////////////////////////////
-
-
- mapping.insert_to_alphabet(leftMarker);
- mapping.insert_to_alphabet(rightMarker);
- mapping.insert_to_alphabet(tmpMarker);
-
- // Surround mapping with brackets
- HfstTransducer mappingWithBrackets(leftBracket);
- mappingWithBrackets.concatenate(mapping)
- .concatenate(rightBracket).minimize();
-
-
- // non - optional
- // mapping = <a:b> u <2a:a>2
- if ( optional != true )
- {
- // needed in case of ? -> x replacement
- mapping.insert_to_alphabet(leftMarker2);
- mapping.insert_to_alphabet(rightMarker2);
- mappingWithBrackets.insert_to_alphabet(leftMarker2);
- mappingWithBrackets.insert_to_alphabet(rightMarker2);
-
-
- HfstTransducer mappingProject(mapping);
- mappingProject.input_project().minimize();
-
- HfstTransducer mappingWithBracketsNonOptional(leftBracket2);
- // needed in case of ? -> x replacement
- //mappingWithBracketsNonOptional.insert_to_alphabet(leftMarker2);
- //mappingWithBracketsNonOptional.
- //insert_to_alphabet(rightMarker2);
-
- mappingWithBracketsNonOptional.concatenate(mappingProject).
- concatenate(rightBracket2).
- minimize();
- // mappingWithBrackets...... expanded
- mappingWithBrackets.disjunct(mappingWithBracketsNonOptional)
- .minimize();
-
- }
-
- identityExpanded.disjunct(mappingWithBrackets).minimize();
- mappingWithBracketsVector.push_back(mappingWithBrackets);
- }
-
- identityExpanded.repeat_star().minimize();
-
- // if none of the rules have contexts, return identityExpanded
- if ( noContexts )
- {
- identityExpanded.remove_from_alphabet(tmpMarker);
- // substitute markers with epsilons
- identityExpanded.substitute(marker_substitutions);
- identityExpanded.remove_from_alphabet(marker_symbols);
- return identityExpanded;
- }
-
- // if they have contexts, process them
- if ( ruleVector.size() != mappingWithBracketsVector.size() )
- {
- HFST_THROW_MESSAGE(TransducerTypeMismatchException,
- "Vector sizes don't match");
- }
-
- std::map<std::string,hfst::HfstBasicTransducer> contextReplaceMap;
-
-
- HfstTransducer unionContextReplace(type);
- HfstTransducer bracketedReplace(type);
- //THIS is for disjuncting labels first, and then substitute them with
- // transducers
- //HfstTransducer unionContextReplace_labels(type);
- for ( unsigned int i = 0; i < ruleVector.size(); i++ )
- {
- Rule ruletmp(ruleVector[i]);
- ruletmp.encodeFlags();
-
- // Surround mapping with brackets with tmp boudaries
- HfstTransducer mappingWithBracketsAndTmpBoundary(tmpBracket);
- mappingWithBracketsAndTmpBoundary
- .concatenate(mappingWithBracketsVector[i])
- .concatenate(tmpBracket)
- .minimize();
- // .* |<a:b>| :*
- HfstTransducer bracketedReplaceTmp(identityExpanded);
- bracketedReplaceTmp.concatenate(mappingWithBracketsAndTmpBoundary)
- .concatenate(identityExpanded)
- .minimize();
-
- bracketedReplaceTmp.transform_weights(&zero_weight);
- bracketedReplace.disjunct(bracketedReplaceTmp).minimize();
-
- //Create context part
- HfstTransducer unionContextReplaceTmp(type);
-
- // For each context that uses the output side (REPL_DOWN,
- // REPL_LEFT, REPL_RIGHT) we must freely allow all markers that can
- // be generated by other rules. Consider e.g. the following case:
- //
- // C -> d::1 || \/ b _ b ,, a::1 -> b || c _ C , C _ c
- //
- // where the second rule yields caCac -> cbCbc and the first one
- // again cbCbc -> cbdbc.
- HfstTransducerPairVector cont = ruletmp.get_context();
-
- if (ruletmp.get_replType() != REPL_UP)
- {
- for (HfstTransducerPairVector::iterator cont_it = cont.begin();
- cont_it != cont.end(); cont_it++)
- {
- for (StringSet::const_iterator sit
- = marker_symbols.begin();
- sit != marker_symbols.end(); sit++)
- {
- if (getMarkerNumber(*sit) != i)
- {
- StringPair marker_pair(*sit, *sit);
- // 'false' makes sure harmonization is not done
- cont_it->first.insert_freely(marker_pair, false);
- cont_it->second.insert_freely(marker_pair, false);
- }
- }
- }
- }
-
- unionContextReplaceTmp
- = expandContextsWithMapping ( cont,
- mappingWithBracketsAndTmpBoundary,
- identityExpanded,
- ruletmp.get_replType(),
- optional);
-
- unionContextReplaceTmp.transform_weights(&zero_weight);
-
- unionContextReplace.disjunct(unionContextReplaceTmp).minimize();
- /*
- //THIS part is for disjuncting labels first, and then substitute
- // them with transducers
- //create map with context keys and basic transducers
- stringstream tmpString;
- tmpString << "$_contextRepl." << i << "_$";
- string contx_key = tmpString.str();
- TOK.add_multichar_symbol(contx_key);
- // cout << "lexicon " << regex_key << "\n";
-
- if (contextReplaceMap.find(contx_key) == contextReplaceMap.end())
- {
- contextReplaceMap.insert(pair<string,HfstBasicTransducer>(contx_key,
- HfstBasicTransducer(unionContextReplaceTmp)));
- }
-
- //disjunct all keys
- unionContextReplace_labels.disjunct(HfstTransducer(contx_key,
- TOK, type)).minimize();
-
- // printf("unionContextReplace_labels: \n");
- // unionContextReplace_labels.write_in_att_format(stdout, 1);
-
- */
- }
-
- //printf("unionContextReplace_labels: \n");
- //unionContextReplace.write_in_att_format(stdout, 1);
-
-
-
- /*
- //THIS part is for disjuncting labels first, and then substitute them
- with transducers
- printf("substitute labels with real tr: \n");
- //substitute labels with real tr
- HfstBasicTransducer btr(unionContextReplace_labels);
- btr.substitute(contextReplaceMap, true);
- btr.prune_alphabet();
-
- printf("Substitute done. \n");
- HfstTransducer unionContextReplace(btr, type);
- printf("Created tr \n");
-
-
- //weight can be zero, because it will be subtracted
- bracketedReplace.transform_weights(&zero_weight);
- unionContextReplace.transform_weights(&zero_weight);
-
- printf("min \n");
- unionContextReplace.minimize();
- printf("min done. \n");
-
- */
-
- /////////////////////}
- //printf("subtract all mappings in contexts from replace without
- //contexts: \n");
- // subtract all mappings in contexts from replace without contexts
- HfstTransducer replaceWithoutContexts(bracketedReplace);
- replaceWithoutContexts.subtract(unionContextReplace).minimize();
-
- //printf("remove bla bla: \n");
- // remove tmpMaprker
- replaceWithoutContexts.substitute
- (StringPair(tmpMarker, tmpMarker),
- StringPair("@_EPSILON_SYMBOL_@", "@_EPSILON_SYMBOL_@")).minimize();
- replaceWithoutContexts.remove_from_alphabet(tmpMarker);
- replaceWithoutContexts.minimize();
-
- identityExpanded.remove_from_alphabet(tmpMarker);
-
- //printf("before final negation: \n");
- //replaceWithoutContexts.write_in_att_format(stdout, 1);
-
- //printf("final subtract: \n");
- // final negation
- HfstTransducer uncondidtionalTr(identityExpanded);
- uncondidtionalTr.subtract(replaceWithoutContexts).minimize();
-
- //printf("uncondidtionalTr: \n");
- //uncondidtionalTr.write_in_att_format(stdout, 1);
-
- // substitute markers with epsilons
- uncondidtionalTr.substitute(marker_substitutions);
- uncondidtionalTr.remove_from_alphabet(marker_symbols);
-
- return uncondidtionalTr;
-
- }
-
-
- //---------------------------------
- // CONSTRAINTS
- //---------------------------------
-
- // (help function)
- // returns: [ B:0 | 0:B | ?-B ]*
- // which is used in some constraints
- HfstTransducer constraintsRightPart( ImplementationType type )
- {
- HfstTokenizer TOK;
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
-
- String leftMarker("@LM@");
- String rightMarker("@RM@");
- TOK.add_multichar_symbol(leftMarker);
- TOK.add_multichar_symbol(rightMarker);
-
- // Identity pair (normal)
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
-
-
- HfstTransducer leftBracket(leftMarker, TOK, type);
- HfstTransducer rightBracket(rightMarker, TOK, type);
-
- // Create Right Part
- HfstTransducer B(leftBracket);
- B.disjunct(rightBracket).minimize();
-
- HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
- HfstTransducer epsilonToLeftMark("@_EPSILON_SYMBOL_@", leftMarker, TOK, type);
- HfstTransducer LeftMarkToEpsilon(leftMarker, "@_EPSILON_SYMBOL_@", TOK, type);
-
- HfstTransducer epsilonToBrackets(epsilon);
- epsilonToBrackets.cross_product(B);
-
- HfstTransducer bracketsToEpsilon(B);
- bracketsToEpsilon.cross_product(epsilon);
-
- HfstTransducer identityPairMinusBrackets(identityPair);
- identityPairMinusBrackets.subtract(B).minimize();//.repeat_plus().minimize();
-
- HfstTransducer rightPart(epsilonToBrackets);
- rightPart.disjunct(bracketsToEpsilon).disjunct(identityPairMinusBrackets).minimize().repeat_star().minimize();
-
- return rightPart;
- }
-
- // .#. ?* <:0 0:> ?* .#.
- // filters out empty string
- HfstTransducer oneBetterthanNoneConstraint( const HfstTransducer &uncondidtionalTr )
- {
- ImplementationType type = uncondidtionalTr.get_type();
- HfstTokenizer TOK;
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
- TOK.add_multichar_symbol(".#.");
-
- String leftMarker("@LM@");
- String rightMarker("@RM@");
- TOK.add_multichar_symbol(leftMarker);
- TOK.add_multichar_symbol(rightMarker);
-
- // Identity (normal)
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- HfstTransducer identity (identityPair);
- identity.repeat_star().minimize();
-
- HfstTransducer leftBracketToZero(leftMarker, "@_EPSILON_SYMBOL_@", TOK, type);
- HfstTransducer rightBracketToZero(rightMarker, "@_EPSILON_SYMBOL_@", TOK, type);
-
-
- HfstTransducer boundary(".#.", TOK, type);
- HfstTransducer Constraint(boundary);
- Constraint.concatenate(identity);
- Constraint.concatenate(leftBracketToZero)
- .concatenate(rightBracketToZero)
- .concatenate(boundary)
- .concatenate(identity)
- .minimize();
-
-// printf("Constraint: \n");
-// Constraint.write_in_att_format(stdout, 1);
-
- //// Compose with unconditional replace transducer
- // tmp = t.1 .o. Constr .o. t.1
- // (t.1 - tmp.2) .o. t
-
-
- HfstTransducer retval(type);
- retval = constraintComposition(uncondidtionalTr, Constraint);
-
-
- return retval;
- }
-
-
-
- // .#. ?* <:0 [B:0]* [I-B] [ B:0 | 0:B | ?-B ]* .#.
- HfstTransducer leftMostConstraint( const HfstTransducer &uncondidtionalTr )
- {
- HfstTokenizer TOK;
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
- TOK.add_multichar_symbol("@_UNKNOWN_SYMBOL_@");
- TOK.add_multichar_symbol(".#.");
-
- String leftMarker("@LM@");
- String rightMarker("@RM@");
- TOK.add_multichar_symbol(leftMarker);
- TOK.add_multichar_symbol(rightMarker);
-
- ImplementationType type = uncondidtionalTr.get_type();
-
- HfstTransducer leftBracket(leftMarker, TOK, type);
- HfstTransducer rightBracket(rightMarker, TOK, type);
-
-
- // Identity (normal)
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- HfstTransducer identity (identityPair);
- identity.repeat_star().minimize();
-
-
- // Create Right Part: [ B:0 | 0:B | ?-B ]*
- HfstTransducer rightPart(type);
- rightPart = constraintsRightPart(type);
-
-
- // epsilon
- HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
- // B
- HfstTransducer B(leftBracket);
- B.disjunct(rightBracket).minimize();
- // (B:0)*
-
-
- HfstTransducer bracketsToEpsilonStar(B);
- bracketsToEpsilonStar.cross_product(epsilon).minimize().repeat_star().minimize();
-
- // (I-B) and (I-B)+
- HfstTransducer identityPairMinusBrackets(identityPair);
- identityPairMinusBrackets.subtract(B).minimize();
-
- HfstTransducer identityPairMinusBracketsPlus(identityPairMinusBrackets);
- identityPairMinusBracketsPlus.repeat_plus().minimize();
-
- /*
- HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
- HfstTransducer identityPairMinusBracketsOrEpsilon(identityPairMinusBrackets);
- identityPairMinusBracketsOrEpsilon.disjunct(epsilon).minimize();
- */
- HfstTransducer LeftBracketToEpsilon(leftMarker, "@_EPSILON_SYMBOL_@", TOK, type);
-
-
- HfstTransducer boundary(".#.", TOK, type);
-
- HfstTransducer Constraint(boundary);
- Constraint.concatenate(identity);
- //HfstTransducer Constraint(identity);
-
- // ?* <:0 [B:0]* [I-B] [ B:0 | 0:B | ?-B ]*
- Constraint.concatenate(LeftBracketToEpsilon).
- concatenate(bracketsToEpsilonStar).
- concatenate(identityPairMinusBrackets).
- concatenate(rightPart).
- minimize();
-
- Constraint.concatenate(boundary).minimize();
-
- // printf("Constraint: \n");
- // Constraint.write_in_att_format(stdout, 1);
-
- //// Compose with unconditional replace transducer
- // tmp = t.1 .o. Constr .o. t.1
- // (t.1 - tmp.2) .o. t
-
- HfstTransducer retval(type);
- retval = constraintComposition(uncondidtionalTr, Constraint);
-
- //printf("Constraint: \n");
- //Constraint.write_in_att_format(stdout, 1);
- return retval;
-
- }
-
- // [ B:0 | 0:B | ?-B ]* [I-B]+ >:0 [ ?-B ]*
- HfstTransducer rightMostConstraint( const HfstTransducer &uncondidtionalTr )
- {
- HfstTokenizer TOK;
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
- TOK.add_multichar_symbol("@_UNKNOWN_SYMBOL_@");
-
- String leftMarker("@LM@");
- String rightMarker("@RM@");
- TOK.add_multichar_symbol(leftMarker);
- TOK.add_multichar_symbol(rightMarker);
-
- ImplementationType type = uncondidtionalTr.get_type();
-
- HfstTransducer leftBracket(leftMarker, TOK, type);
- HfstTransducer rightBracket(rightMarker, TOK, type);
-
-
- // Identity (normal)
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- HfstTransducer identity (identityPair);
- identity.repeat_star().minimize();
-
-
- // Create Right Part: [ B:0 | 0:B | ?-B ]*
- HfstTransducer rightPart(type);
- rightPart = constraintsRightPart(type);
-
-
- // epsilon
- HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
- // B
- HfstTransducer B(leftBracket);
- B.disjunct(rightBracket).minimize();
- // (B:0)*
- HfstTransducer bracketsToEpsilonStar(B);
- bracketsToEpsilonStar.cross_product(epsilon).minimize().repeat_star().minimize();
-
- // (I-B) and (I-B)+
- HfstTransducer identityPairMinusBrackets(identityPair);
- identityPairMinusBrackets.subtract(B).minimize();
-
- HfstTransducer identityPairMinusBracketsPlus(identityPairMinusBrackets);
- identityPairMinusBracketsPlus.repeat_plus().minimize();
-
- HfstTransducer identityPairMinusBracketsStar(identityPairMinusBrackets);
- identityPairMinusBracketsStar.repeat_star().minimize();
-
-
- HfstTransducer RightBracketToEpsilon(rightMarker, "@_EPSILON_SYMBOL_@", TOK, type);
-
- HfstTransducer Constraint(rightPart);
- // [ B:0 | 0:B | ?-B ]* [I-B]+ >:0 [ ?-B ]*
-
- Constraint.concatenate(identityPairMinusBracketsPlus).
- concatenate(RightBracketToEpsilon).
- concatenate(identity).
- minimize();
-
- //// Compose with unconditional replace transducer
- // tmp = t.1 .o. Constr .o. t.1
- // (t.1 - tmp.2) .o. t
-
- HfstTransducer retval(type);
- retval = constraintComposition(uncondidtionalTr, Constraint);
-
-
- //printf("Constraint: \n");
- //Constraint.write_in_att_format(stdout, 1);
- return retval;
-
- }
-
-
- // Longest match
- // it should be composed to left most transducer........
- // ?* < [?-B]+ 0:> [ ? | 0:< | <:0 | 0:> | B ] [ B:0 | 0:B | ?-B ]*
- HfstTransducer longestMatchLeftMostConstraint( const HfstTransducer &uncondidtionalTr )
- {
-
- HfstTokenizer TOK;
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
-
- String leftMarker("@LM@");
- String rightMarker("@RM@");
- TOK.add_multichar_symbol(leftMarker);
- TOK.add_multichar_symbol(rightMarker);
-
- ImplementationType type = uncondidtionalTr.get_type();
-
- HfstTransducer leftBracket(leftMarker, TOK, type);
- HfstTransducer rightBracket(rightMarker, TOK, type);
-
- // Identity
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- HfstTransducer identity(identityPair);
- identity.repeat_star().minimize();
-
- // epsilon
- HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
- // B
- HfstTransducer B(leftBracket);
- B.disjunct(rightBracket).minimize();
- // (B:0)*
- HfstTransducer bracketsToEpsilonStar(B);
- bracketsToEpsilonStar.cross_product(epsilon).minimize().repeat_star().minimize();
-
- // (I-B) and (I-B)+
- HfstTransducer identityPairMinusBrackets(identityPair);
- identityPairMinusBrackets.subtract(B).minimize();
-
- HfstTransducer identityPairMinusBracketsPlus(identityPairMinusBrackets);
- identityPairMinusBracketsPlus.repeat_plus().minimize();
-
-
-
- // Create Right Part: [ B:0 | 0:B | ?-B ]*
- HfstTransducer rightPart(type);
- rightPart = constraintsRightPart(type);
-
- HfstTransducer RightBracketToEpsilon(rightMarker, "@_EPSILON_SYMBOL_@", TOK, type);
- HfstTransducer epsilonToRightBracket("@_EPSILON_SYMBOL_@", rightMarker, TOK, type);
- HfstTransducer LeftBracketToEpsilon(leftMarker, "@_EPSILON_SYMBOL_@", TOK, type);
- HfstTransducer epsilonToLeftBracket("@_EPSILON_SYMBOL_@", leftMarker, TOK, type);
-
- //[ ? | 0:< | <:0 | 0:> | B ]
- // HfstTransducer nonClosingBracketInsertion(identityPair);
- HfstTransducer nonClosingBracketInsertion(epsilonToLeftBracket);
- nonClosingBracketInsertion.
- //disjunct(epsilonToLeftBracket).
- disjunct(LeftBracketToEpsilon).
- disjunct(epsilonToRightBracket).
- disjunct(B).
- minimize();
- // printf("nonClosingBracketInsertion: \n");
- // nonClosingBracketInsertion.write_in_att_format(stdout, 1);
-
-
- nonClosingBracketInsertion.concatenate(identityPairMinusBracketsPlus).minimize();
-
- HfstTransducer middlePart(identityPairMinusBrackets);
- middlePart.disjunct(nonClosingBracketInsertion).minimize();
-
-
- // ?* < [?-B]+ 0:> [ ? | 0:< | <:0 | 0:> | B ] [?-B]+ [ B:0 | 0:B | ?-B ]*
- HfstTransducer Constraint(identity);
- Constraint.concatenate(leftBracket).
- concatenate(identityPairMinusBracketsPlus).
- concatenate(epsilonToRightBracket).
- // concatenate(nonClosingBracketInsertion).
- // concatenate(identityPairMinusBracketsPlus).
- concatenate(middlePart).
- concatenate(rightPart).
- minimize();
- //printf("Constraint Longest Match: \n");
- //Constraint.write_in_att_format(stdout, 1);
-
-
- //uncondidtionalTr should be left most for the left most longest match
- HfstTransducer retval(type);
- retval = constraintComposition(uncondidtionalTr, Constraint);
-
-
- return retval;
-
- }
-
- // Longest match RIGHT most
- HfstTransducer longestMatchRightMostConstraint(const HfstTransducer &uncondidtionalTr )
- {
- HfstTokenizer TOK;
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
-
- String leftMarker("@LM@");
- String rightMarker("@RM@");
- TOK.add_multichar_symbol(leftMarker);
- TOK.add_multichar_symbol(rightMarker);
-
- ImplementationType type = uncondidtionalTr.get_type();
-
- HfstTransducer leftBracket(leftMarker, TOK, type);
- HfstTransducer rightBracket(rightMarker, TOK, type);
-
- // Identity
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- HfstTransducer identity(identityPair);
- identity.repeat_star().minimize();
-
- // epsilon
- HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
- // B
- HfstTransducer B(leftBracket);
- B.disjunct(rightBracket).minimize();
- // (B:0)*
- HfstTransducer bracketsToEpsilonStar(B);
- bracketsToEpsilonStar.cross_product(epsilon).minimize().repeat_star().minimize();
-
- // (I-B) and (I-B)+
- HfstTransducer identityPairMinusBrackets(identityPair);
- identityPairMinusBrackets.subtract(B).minimize();
-
- HfstTransducer identityPairMinusBracketsPlus(identityPairMinusBrackets);
- identityPairMinusBracketsPlus.repeat_plus().minimize();
-
-
-
- // Create Right Part: [ B:0 | 0:B | ?-B ]*
- HfstTransducer rightPart(type);
- rightPart = constraintsRightPart(type);
-
-
-
- HfstTransducer RightBracketToEpsilon(rightMarker, "@_EPSILON_SYMBOL_@", TOK, type);
-
- HfstTransducer epsilonToRightBracket("@_EPSILON_SYMBOL_@", rightMarker, TOK, type);
- HfstTransducer LeftBracketToEpsilon(leftMarker, "@_EPSILON_SYMBOL_@", TOK, type);
- HfstTransducer epsilonToLeftBracket("@_EPSILON_SYMBOL_@", leftMarker, TOK, type);
-
-
- //[ ? | 0:< | >:0 | 0:> | B ]
- HfstTransducer nonClosingBracketInsertion(identityPair);
- nonClosingBracketInsertion.disjunct(epsilonToLeftBracket).
- disjunct(RightBracketToEpsilon).
- disjunct(epsilonToRightBracket).
- disjunct(B).
- minimize();
-
-
- // [ B:0 | 0:B | ?-B ]* [?-B]+ [ ? | 0:< | <:0 | 0:> | B ] 0:< [?-B]+ > ?*
-
- HfstTransducer Constraint(rightPart);
- Constraint.concatenate(identityPairMinusBracketsPlus).
- concatenate(nonClosingBracketInsertion).minimize().
- concatenate(epsilonToLeftBracket).
- concatenate(identityPairMinusBracketsPlus).
- concatenate(rightBracket).
- concatenate(identity).
- minimize();
- //printf("Constraint Longest Match: \n");
- //Constraint.write_in_att_format(stdout, 1);
-
-
- //uncondidtionalTr should be left most for the left most longest match
- HfstTransducer retval(type);
- retval = constraintComposition(uncondidtionalTr, Constraint);
-
-
- return retval;
- }
-
- // Shortest match
- // it should be composed to left most transducer........
- // ?* < [?-B]+ >:0
- // [?-B] or [ ? | 0:< | <:0 | >:0 | B ][?-B]+
- // [ B:0 | 0:B | ?-B ]*
- HfstTransducer shortestMatchLeftMostConstraint( const HfstTransducer &uncondidtionalTr )
- {
-
- HfstTokenizer TOK;
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
-
- String leftMarker("@LM@");
- String rightMarker("@RM@");
- TOK.add_multichar_symbol(leftMarker);
- TOK.add_multichar_symbol(rightMarker);
-
- ImplementationType type = uncondidtionalTr.get_type();
-
- HfstTransducer leftBracket(leftMarker, TOK, type);
- HfstTransducer rightBracket(rightMarker, TOK, type);
-
- // Identity
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- HfstTransducer identity(identityPair);
- identity.repeat_star().minimize();
-
-
- // Create Right Part: [ B:0 | 0:B | ?-B ]*
- HfstTransducer rightPart(type);
- rightPart = constraintsRightPart(type);
-
- // [?-B] and [?-B]+
- HfstTransducer B(leftBracket);
- B.disjunct(rightBracket).minimize();
- HfstTransducer identityPairMinusBrackets(identityPair);
- identityPairMinusBrackets.subtract(B).minimize();
- HfstTransducer identityPairMinusBracketsPlus(identityPairMinusBrackets);
- identityPairMinusBracketsPlus.repeat_plus().minimize();
-
-
- HfstTransducer RightBracketToEpsilon(rightMarker, "@_EPSILON_SYMBOL_@", TOK, type);
- HfstTransducer epsilonToRightBracket("@_EPSILON_SYMBOL_@", rightMarker, TOK, type);
- HfstTransducer LeftBracketToEpsilon(leftMarker, "@_EPSILON_SYMBOL_@", TOK, type);
- HfstTransducer epsilonToLeftBracket("@_EPSILON_SYMBOL_@", leftMarker, TOK, type);
-
-
- // [ 0:< | <:0 | >:0 | B ][?-B]+
- HfstTransducer nonClosingBracketInsertion(epsilonToLeftBracket);
- nonClosingBracketInsertion.
- //disjunct(epsilonToLeftBracket).
- disjunct(LeftBracketToEpsilon).
- disjunct(RightBracketToEpsilon).
- disjunct(B).
- minimize();
-
- nonClosingBracketInsertion.concatenate(identityPairMinusBracketsPlus).minimize();
-
- HfstTransducer middlePart(identityPairMinusBrackets);
- middlePart.disjunct(nonClosingBracketInsertion).minimize();
-
- // printf("nonClosingBracketInsertion: \n");
- // nonClosingBracketInsertion.write_in_att_format(stdout, 1);
-
- // ?* < [?-B]+ >:0
- // [?-B] or [ ? | 0:< | <:0 | >:0 | B ][?-B]+
- //[ B:0 | 0:B | ?-B ]*
- HfstTransducer Constraint(identity);
- Constraint.concatenate(leftBracket).
- concatenate(identityPairMinusBracketsPlus).
- concatenate(RightBracketToEpsilon).
- concatenate(middlePart).minimize().
- concatenate(rightPart).
- minimize();
-
- //printf("Constraint Shortest Match: \n");
- //Constraint.write_in_att_format(stdout, 1);
-
-
- //uncondidtionalTr should be left most for the left most shortest match
- HfstTransducer retval(type);
- retval = constraintComposition(uncondidtionalTr, Constraint);
-
-
- return retval;
-
- }
-
- // Shortest match
- // it should be composed to left most transducer........
- //[ B:0 | 0:B | ?-B ]*
- // [?-B] or [?-B]+ [ ? | 0:> | >:0 | <:0 | B ]
- // <:0 [?-B]+ > ?*
- HfstTransducer shortestMatchRightMostConstraint( const HfstTransducer &uncondidtionalTr )
- {
-
- HfstTokenizer TOK;
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
-
- String leftMarker("@LM@");
- String rightMarker("@RM@");
- TOK.add_multichar_symbol(leftMarker);
- TOK.add_multichar_symbol(rightMarker);
-
- ImplementationType type = uncondidtionalTr.get_type();
-
- HfstTransducer leftBracket(leftMarker, TOK, type);
- HfstTransducer rightBracket(rightMarker, TOK, type);
-
- // Identity
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- HfstTransducer identity(identityPair);
- identity.repeat_star().minimize();
-
-
- // Create Right Part: [ B:0 | 0:B | ?-B ]*
- HfstTransducer rightPart(type);
- rightPart = constraintsRightPart(type);
-
- // [?-B] and [?-B]+
- HfstTransducer B(leftBracket);
- B.disjunct(rightBracket).minimize();
- HfstTransducer identityPairMinusBrackets(identityPair);
- identityPairMinusBrackets.subtract(B).minimize();
- HfstTransducer identityPairMinusBracketsPlus(identityPairMinusBrackets);
- identityPairMinusBracketsPlus.repeat_plus().minimize();
-
-
-
- HfstTransducer RightBracketToEpsilon(rightMarker, "@_EPSILON_SYMBOL_@", TOK, type);
- HfstTransducer epsilonToRightBracket("@_EPSILON_SYMBOL_@", rightMarker, TOK, type);
- HfstTransducer LeftBracketToEpsilon(leftMarker, "@_EPSILON_SYMBOL_@", TOK, type);
- HfstTransducer epsilonToLeftBracket("@_EPSILON_SYMBOL_@", leftMarker, TOK, type);
-
-
- // [?-B]+ [ 0:> | >:0 | <:0 | B ]
- HfstTransducer nonClosingBracketInsertionTmp(epsilonToRightBracket);
- nonClosingBracketInsertionTmp.
- disjunct(RightBracketToEpsilon).
- disjunct(LeftBracketToEpsilon).
- disjunct(B).minimize();
- HfstTransducer nonClosingBracketInsertion(identityPairMinusBracketsPlus);
- nonClosingBracketInsertion.concatenate(nonClosingBracketInsertionTmp).minimize();
-
-
- HfstTransducer middlePart(identityPairMinusBrackets);
- middlePart.disjunct(nonClosingBracketInsertion).minimize();
-
-
- //[ B:0 | 0:B | ?-B ]*
- // [?-B] or [?-B]+ [ ? | 0:> | >:0 | <:0 | B ]
- // <:0 [?-B]+ > ?*
-
- HfstTransducer Constraint(rightPart);
- Constraint.concatenate(middlePart).
- concatenate(LeftBracketToEpsilon).
- concatenate(identityPairMinusBracketsPlus).
- concatenate(rightBracket).
- concatenate(identity).
- minimize();
-
- //printf("Constraint Shortest Match: \n");
- //Constraint.write_in_att_format(stdout, 1);
-
- //uncondidtionalTr should be left most for the left most longest match
- HfstTransducer retval(type);
- retval = constraintComposition(uncondidtionalTr, Constraint);
-
- return retval;
-
- }
-
-
- // ?* [ BL:0 (?-B)+ BR:0 ?* ]+
- HfstTransducer mostBracketsPlusConstraint( const HfstTransducer &uncondidtionalTr )
- {
- HfstTokenizer TOK;
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
-
- String leftMarker("@LM@");
- String rightMarker("@RM@");
- String leftMarker2("@LM2@");
- String rightMarker2("@RM2@");
-
- TOK.add_multichar_symbol(leftMarker);
- TOK.add_multichar_symbol(rightMarker);
- TOK.add_multichar_symbol(leftMarker2);
- TOK.add_multichar_symbol(rightMarker2);
-
- ImplementationType type = uncondidtionalTr.get_type();
-
- HfstTransducer leftBracket(leftMarker, TOK, type);
- HfstTransducer rightBracket(rightMarker, TOK, type);
- HfstTransducer leftBracket2(leftMarker2, TOK, type);
- HfstTransducer rightBracket2(rightMarker2, TOK, type);
-
- // Identity (normal)
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- HfstTransducer identity (identityPair);
- identity.repeat_star().minimize();
-
- HfstTransducer identityPlus (identityPair);
- identityPlus.repeat_plus().minimize();
-
- HfstTransducer identityStar (identityPair);
- identityStar.repeat_star().minimize();
-
- // epsilon
- String epsilon("@_EPSILON_SYMBOL_@");
-
- // BL:0 ( <1 : 0, <2 : 0)
- HfstTransducer leftBracketToEpsilon(leftMarker, epsilon, TOK, type );
- HfstTransducer leftBracket2ToEpsilon(leftMarker2, epsilon, TOK, type );
- HfstTransducer allLeftBracketsToEpsilon(leftBracketToEpsilon);
- allLeftBracketsToEpsilon.disjunct(leftBracket2ToEpsilon).minimize();
-
-
- // printf("allLeftBracketsToEpsilon: \n");
- // allLeftBracketsToEpsilon.write_in_att_format(stdout, 1);
-
- // BR:0 ( >1 : 0, >2 : 0)
- HfstTransducer rightBracketToEpsilon(rightMarker, epsilon, TOK, type );
- HfstTransducer rightBracket2ToEpsilon(rightMarker2, epsilon, TOK, type );
- HfstTransducer allRightBracketsToEpsilon(rightBracketToEpsilon);
- allRightBracketsToEpsilon.disjunct(rightBracket2ToEpsilon).minimize();
-
- // B (B1 | B2)
- HfstTransducer B(leftBracket);
- B.disjunct(rightBracket).minimize();
- B.disjunct(leftBracket2).minimize();
- B.disjunct(rightBracket2).minimize();
-
-
-
- // (? - B)+
- HfstTransducer identityPairMinusBracketsPlus(identityPair);
- identityPairMinusBracketsPlus.subtract(B).minimize().repeat_plus().minimize();
-
- // repeatingPart ( BL:0 (?-B)+ BR:0 ?* )+
- HfstTransducer repeatingPart(allLeftBracketsToEpsilon);
- repeatingPart.concatenate(identityPairMinusBracketsPlus).minimize();
- repeatingPart.concatenate(allRightBracketsToEpsilon).minimize();
- repeatingPart.concatenate(identityStar).minimize();
- repeatingPart.repeat_plus().minimize();
- //printf("middlePart: \n");
- //middlePart.write_in_att_format(stdout, 1);
-
- HfstTransducer Constraint(identityStar);
- Constraint.concatenate(repeatingPart).minimize();
- //printf("Constraint: \n");
- //Constraint.write_in_att_format(stdout, 1);
-
-
- //// Compose with unconditional replace transducer
- // tmp = t.1 .o. Constr .o. t.1
- // (t.1 - tmp.2) .o. t
-
- HfstTransducer retval(type);
- retval = constraintComposition(uncondidtionalTr, Constraint);
-
- //printf("After composition: \n");
- //retval.write_in_att_format(stdout, 1);
-
- return retval;
- }
-
- // ?* [ BL:0 (?-B)* BR:0 ?* ]+
- HfstTransducer mostBracketsStarConstraint( const HfstTransducer &uncondidtionalTr )
- {
- HfstTokenizer TOK;
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
-
- String leftMarker("@LM@");
- String rightMarker("@RM@");
- String leftMarker2("@LM2@");
- String rightMarker2("@RM2@");
- TOK.add_multichar_symbol(leftMarker);
- TOK.add_multichar_symbol(rightMarker);
- TOK.add_multichar_symbol(leftMarker2);
- TOK.add_multichar_symbol(rightMarker2);
-
- ImplementationType type = uncondidtionalTr.get_type();
-
- HfstTransducer leftBracket(leftMarker, TOK, type);
- HfstTransducer rightBracket(rightMarker, TOK, type);
-
- HfstTransducer leftBracket2(leftMarker2, TOK, type);
- HfstTransducer rightBracket2(rightMarker2, TOK, type);
-
-
- // Identity (normal)
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- HfstTransducer identity (identityPair);
- identity.repeat_star().minimize();
-
- HfstTransducer identityPlus (identityPair);
- identityPlus.repeat_plus().minimize();
-
-
- HfstTransducer identityStar (identityPair);
- identityStar.repeat_star().minimize();
-
-
- // epsilon
- String epsilon("@_EPSILON_SYMBOL_@");
-
- // BL:0 ( <1 : 0, <2 : 0)
- HfstTransducer leftBracketToEpsilon(leftMarker, epsilon, TOK, type );
- HfstTransducer leftBracket2ToEpsilon(leftMarker2, epsilon, TOK, type );
- HfstTransducer allLeftBracketsToEpsilon(leftBracketToEpsilon);
- allLeftBracketsToEpsilon.disjunct(leftBracket2ToEpsilon).minimize();
-
- // printf("allLeftBracketsToEpsilon: \n");
- // allLeftBracketsToEpsilon.write_in_att_format(stdout, 1);
-
- // BR:0 ( >1 : 0, >2 : 0)
- HfstTransducer rightBracketToEpsilon(rightMarker, epsilon, TOK, type );
- HfstTransducer rightBracket2ToEpsilon(rightMarker2, epsilon, TOK, type );
- HfstTransducer allRightBracketsToEpsilon(rightBracketToEpsilon);
- allRightBracketsToEpsilon.disjunct(rightBracket2ToEpsilon).minimize();
-
- // B (B1 | B2)
- HfstTransducer B(leftBracket);
- B.disjunct(rightBracket).minimize();
- B.disjunct(leftBracket2).minimize();
- B.disjunct(rightBracket2).minimize();
-
- // (? - B)*
- HfstTransducer identityPairMinusBracketsStar(identityPair);
- identityPairMinusBracketsStar.subtract(B).minimize().repeat_star().minimize();
-
- // repeatingPart [ BL:0 (?-B)* BR:0 ?* ]+
- HfstTransducer repeatingPart(allLeftBracketsToEpsilon);
- repeatingPart.concatenate(identityPairMinusBracketsStar).minimize();
- repeatingPart.concatenate(allRightBracketsToEpsilon).minimize();
- repeatingPart.concatenate(identityStar).minimize();
- repeatingPart.repeat_plus().minimize();
- //printf("middlePart: \n");
- //repeatingPart.write_in_att_format(stdout, 1);
-
- HfstTransducer Constraint(identityStar);
- Constraint.concatenate(repeatingPart).minimize();
- //printf("Constraint: \n");
- //Constraint.write_in_att_format(stdout, 1);
-
- //// Compose with unconditional replace transducer
- // tmp = t.1 .o. Constr .o. t.1
- // (t.1 - tmp.2) .o. t
- HfstTransducer retval(type);
- retval = constraintComposition(uncondidtionalTr, Constraint);
-
- //printf("After composition: \n");
- //retval.write_in_att_format(stdout, 1);
- return retval;
-
- }
- // ?* B2 ?*
- HfstTransducer removeB2Constraint( const HfstTransducer &t )
- {
- HfstTokenizer TOK;
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
-
- String leftMarker2("@LM2@");
- String rightMarker2("@RM2@");
-
- TOK.add_multichar_symbol(leftMarker2);
- TOK.add_multichar_symbol(rightMarker2);
-
- ImplementationType type = t.get_type();
-
- HfstTransducer leftBracket2(leftMarker2, TOK, type);
- HfstTransducer rightBracket2(rightMarker2, TOK, type);
-
- // Identity (normal)
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- HfstTransducer identity (identityPair);
- identity.repeat_star().minimize();
-
- HfstTransducer identityStar (identityPair);
- identityStar.repeat_star().minimize();
-
- // B (B2)
- HfstTransducer B(leftBracket2);
- B.disjunct(rightBracket2).minimize();
-
-
- HfstTransducer Constraint(identityStar);
- Constraint.concatenate(B).minimize();
- Constraint.concatenate(identityStar).minimize();
-
- //// Compose with unconditional replace transducer
- // tmp = t.1 .o. Constr .o. t.1
- // (t.1 - tmp.2) .o. t
-
- HfstTransducer retval(type);
- retval = constraintComposition(t, Constraint);
-
- retval.remove_from_alphabet(leftMarker2);
- retval.remove_from_alphabet(rightMarker2);
-
- //printf("Remove B2 After composition: \n");
- //retval.write_in_att_format(stdout, 1);
-
- return retval;
-
- }
- // to avoid repetition in empty replace rule
- HfstTransducer noRepetitionConstraint( const HfstTransducer &t )
- {
- HfstTokenizer TOK;
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
-
- String leftMarker("@LM@");
- String rightMarker("@RM@");
- TOK.add_multichar_symbol(leftMarker);
- TOK.add_multichar_symbol(rightMarker);
-
- String leftMarker2("@LM2@");
- String rightMarker2("@RM2@");
-
- //if the transdcuer is optional, LM2 and RM2 are not there
- bool optional = true;
- StringSet transducerAlphabet = t.get_alphabet();
- for (StringSet::const_iterator s = transducerAlphabet.begin();
- s != transducerAlphabet.end();
- ++s)
- {
- String alph = *s;
- if ( alph == leftMarker2)
- {
- optional = false;
- break;
- }
- }
-
- TOK.add_multichar_symbol(leftMarker2);
- TOK.add_multichar_symbol(rightMarker2);
-
- ImplementationType type = t.get_type();
-
-
- HfstTransducer leftBracket(leftMarker, TOK, type);
- HfstTransducer rightBracket(rightMarker, TOK, type);
-
- HfstTransducer leftBracket2(leftMarker2, TOK, type);
- HfstTransducer rightBracket2(rightMarker2, TOK, type);
-
-
- HfstTransducer leftBrackets(leftBracket);
- if (!optional)
- {
- leftBrackets.disjunct(leftBracket2).minimize();
- }
-
- HfstTransducer rightBrackets(rightBracket);
- if (!optional)
- {
- rightBrackets.disjunct(rightBracket2).minimize();
- }
- // Identity (normal)
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- /*
- identityPair.insert_to_alphabet(leftMarker);
- identityPair.insert_to_alphabet(rightMarker);
- identityPair.insert_to_alphabet(leftMarker);
- identityPair.insert_to_alphabet(rightMarker2);
- */
-
- HfstTransducer identityStar (identityPair);
- identityStar.repeat_star().minimize();
-
-
- HfstTransducer Constraint(identityStar);
- Constraint.concatenate(leftBrackets).
- concatenate(rightBrackets).
- concatenate(leftBrackets).
- concatenate(rightBrackets).
- concatenate(identityStar).minimize();
-
-
- //// Compose with unconditional replace transducer
- // tmp = t.1 .o. Constr .o. t.1
- // (t.1 - tmp.2) .o. t
-
- //printf("...Constraint: \n");
- //Constraint.write_in_att_format(stdout, 1);
-
-
- HfstTransducer retval(type);
- retval = constraintComposition(t, Constraint);
-
- // retval = removeB2Constraint(retval);
-
- return retval;
-
- }
-
-
-
- // to apply boundary marker (.#.)
- /*
- * [0:.#. | ? - .#.]*
- * .o.
- * tr., ie. a -> b || .#. _ ;
- * .o.
- * .#. (? - .#.)* .#.
- * .o.
- * [.#.:0 | ? - .#.]*
- */
- HfstTransducer applyBoundaryMark( const HfstTransducer &t )
- {
- HfstTokenizer TOK;
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
- TOK.add_multichar_symbol("@_UNKNOWN_SYMBOL_@");
- TOK.add_multichar_symbol("@TMP_UNKNOWN@");
- ImplementationType type = t.get_type();
-
- String boundaryMarker(".#.");
- TOK.add_multichar_symbol(boundaryMarker);
- HfstTransducer boundary(boundaryMarker, TOK, type);
-
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- identityPair.insert_to_alphabet(boundaryMarker);
- // ? - .#.
- HfstTransducer identityMinusBoundary(identityPair);
- identityMinusBoundary.subtract(boundary).minimize();
-
- // (? - .#.)*
- HfstTransducer identityMinusBoundaryStar(identityMinusBoundary);
- identityMinusBoundaryStar.repeat_star().minimize();
-
- // .#. (? - .#.)* .#.
- HfstTransducer boundaryAnythingBoundary(boundary);
- boundaryAnythingBoundary.concatenate(identityMinusBoundaryStar)
- .concatenate(boundary)
- .minimize();
-
- // [0:.#. | ? - .#.]*
- HfstTransducer zeroToBoundary("@_EPSILON_SYMBOL_@", boundaryMarker, TOK, type);
- HfstTransducer retval(zeroToBoundary);
- retval.disjunct(identityMinusBoundary)
- .minimize()
- .repeat_star()
- .minimize();
-
- //printf("retval .o. t: \n");
- //retval.write_in_att_format(stdout, 1);
- // [.#.:0 | ? - .#.]*
- HfstTransducer boundaryToZero(boundaryMarker, "@_EPSILON_SYMBOL_@", TOK, type);
- HfstTransducer removeBoundary(boundaryToZero);
- removeBoundary.disjunct(identityMinusBoundary)
- .minimize()
- .repeat_star()
- .minimize();
-
- // apply boundary to the transducer
- // compose [0:.#. | ? - .#.]* .o. t
- HfstTransducer tr(t);
-
- //tr.insert_to_alphabet(boundaryMarker);
- // substitutute unknowns with tmp symbol
- // this is necessary because of first composition
- tr.substitute("@_UNKNOWN_SYMBOL_@", "@TMP_UNKNOWN@");
-
- //printf("----first: ----\n");
- //tr.write_in_att_format(stdout, 1);
-
- retval.compose(tr).minimize();
-
-
-// printf("first composition: \n");
-// retval.write_in_att_format(stdout, 1);
-
- // compose with .#. (? - .#.)* .#.
- retval.compose(boundaryAnythingBoundary)
- .minimize();
-
-// printf("2. composition: \n");
-// retval.write_in_att_format(stdout, 1);
-
- // compose with [.#.:0 | ? - .#.]*
- retval.compose(removeBoundary).minimize();
-
-// printf("3. composition: \n");
-// retval.write_in_att_format(stdout, 1);
-
- // bring back unknown symbols
- retval.substitute("@TMP_UNKNOWN@", "@_UNKNOWN_SYMBOL_@");
- retval.remove_from_alphabet("@TMP_UNKNOWN@");
-
- // remove boundary from alphabet
- retval.remove_from_alphabet(boundaryMarker);
- return retval;
- }
-
-
- //---------------------------------
- // INTERFACE HELPING FUNCTIONS
- //---------------------------------
-
- // used by hfst-regexp parser
- // creates markup crossproduct and sets property of the first transducer in the mapping to "isMarkup" = "yes"
- // the other transducer in the mapping is set to epsilon transducer
- HfstTransducerPair create_mapping_for_mark_up_replace( const HfstTransducerPair &mappingPair,
- const HfstTransducerPair &marks )
- {
- HfstTokenizer TOK;
- String epsilon = "@_EPSILON_SYMBOL_@";
- TOK.add_multichar_symbol(epsilon);
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
-
- ImplementationType type = mappingPair.first.get_type();
-
- HfstTransducer leftMark(marks.first);
- HfstTransducer rightMark(marks.second);
-
- HfstTransducer epsilonToLeftMark("@_EPSILON_SYMBOL_@", TOK, type);
- epsilonToLeftMark.cross_product(leftMark).minimize();
-
- HfstTransducer epsilonToRightMark(epsilon, TOK, type);
- epsilonToRightMark.cross_product(rightMark).minimize();
-
- //Go through left part of every mapping pair
- // and concatenate: epsilonToLeftMark.leftMapping.epsilonToRightMark
- //then put it into right part of the new transducerPairVector
- HfstTransducer mappingCrossProduct(epsilonToLeftMark);
- mappingCrossProduct.concatenate(mappingPair.first).
- concatenate(epsilonToRightMark).
- minimize();
-
- mappingCrossProduct.set_property("isMarkup", "yes");
-
- HfstTransducer epsilonTr(epsilon, TOK, type);
- HfstTransducerPair retval(mappingCrossProduct, epsilonTr);
-
- return retval;
- }
-
-
-
-
- //---------------------------------
- // REPLACE FUNCTIONS - INTERFACE
- //---------------------------------
-
-
- // replace up, left, right, down
- HfstTransducer replace( const Rule &rule, bool optional)
- {
- HfstTransducer retval( bracketedReplace(rule, optional) );
-
- //printf("---bracketed replace done---: \n");
- //retval.minimize().write_in_att_format(stdout, 1);
-
- // for epenthesis rules
- // it can't have more than one epsilon repetition in a row
-
- retval = noRepetitionConstraint( retval );
-
- //printf("-----noRepetitionConstraint-----: \n");
- //retval.write_in_att_format(stdout, 1);
-
- // deals with boundary symbol, must be before mostBracketsStarConstraint
- retval = applyBoundaryMark( retval );
-
- //printf("----after applyBoundaryMark: ----\n");
- //retval.write_in_att_format(stdout, 1);
- if ( !optional )
- {
-
- //printf(" ---------- mostBracketsStarConstraint --------------\n");
- // Epenthesis rules behave differently if used mostBracketsPlusConstraint
- //retval = mostBracketsPlusConstraint(retval);
- retval = mostBracketsStarConstraint(retval);
- //printf("after non optional: \n");
- //retval.write_in_att_format(stdout, 1);
- }
- retval = removeB2Constraint(retval);
- retval = removeMarkers( retval );
- //printf("after removeMarkers: \n");
- //retval.write_in_att_format(stdout, 1);
- return retval;
- }
-
- // for parallel rules
- HfstTransducer replace( const std::vector<Rule> &ruleVector, bool optional)
- {
-
- // std::cerr << "replace"<< std::endl;
-
- HfstTransducer retval;
- // If there is only one rule in the vector, it is not parallel
- if ( ruleVector.size() == 1 )
- {
- retval = bracketedReplace( ruleVector[0], optional) ;
- }
- else
- {
- retval = parallelBracketedReplace(ruleVector, optional);
- }
-
- //std::cerr << "after bracketed replace"<< std::endl;
- // printf("- bracketed replace -\n");
- // retval.write_in_att_format(stdout, 1);
-
- // for epenthesis rules
- // it can't have more than one epsilon repetition in a row
- retval = noRepetitionConstraint( retval );
-
-
- // printf("----after noRepetitionConstraint: ----\n");
- // retval.write_in_att_format(stdout, 1);
-
-
- // deals with boundary symbol
- retval = applyBoundaryMark( retval );
-
- //printf("----after applyBoundaryMark: ----\n");
- //retval.write_in_att_format(stdout, 1);
-
- if ( !optional )
- {
- // Epenthesis rules behave differently if used mostBracketsPlusConstraint
- // retval = mostBracketsPlusConstraint(retval);
- retval = mostBracketsStarConstraint(retval);
- }
-
- // printf("----after mostBracketsStarConstraint: ----\n");
- // retval.write_in_att_format(stdout, 1);
-
- retval = removeB2Constraint(retval);
-
- //printf("----after removeB2Constraint: ----\n");
- // retval.write_in_att_format(stdout, 1);
-
- retval = removeMarkers( retval );
-
- //printf("----after removeMarkers: ----\n");
- //retval.write_in_att_format(stdout, 1);
- return retval;
-
- }
- // replace left
- HfstTransducer replace_left( const Rule &rule, bool optional)
- {
- HfstTransducerPairVector mappingPairVector = rule.get_mapping();
- //HfstTransducer newMapping = rule.get_mapping();
- //newMapping.invert().minimize();
-
- HfstTransducerPairVector newMappingPairVector;
- for ( unsigned int i = 0; i < mappingPairVector.size(); i++ )
- {
- // in every mapping pair invert first and second
- //HfstTransducer newMapping = rule.get_mapping();
- HfstTransducer first = mappingPairVector[i].first;
- HfstTransducer second = mappingPairVector[i].second;
- newMappingPairVector.push_back(HfstTransducerPair(second, first));
-
- }
-
- Rule newRule ( newMappingPairVector, rule.get_context(), rule.get_replType());
- HfstTransducer retval (replace( newRule, optional));
-
- retval.invert().minimize();
- return retval;
- }
- // replace left parallel
- HfstTransducer replace_left( const std::vector<Rule> &ruleVector, bool optional)
- {
- std::vector<Rule> leftRuleVector;
-
- for ( unsigned int i = 0; i < ruleVector.size(); i++ )
- {
-
- HfstTransducerPairVector mappingPairVector = ruleVector[i].get_mapping();
- //HfstTransducer newMapping = rule.get_mapping();
- //newMapping.invert().minimize();
-
- HfstTransducerPairVector newMappingPairVector;
- for ( unsigned int j = 0; j < mappingPairVector.size(); j++ )
- {
- // in every mapping pair invert first and second
- //HfstTransducer newMapping = rule.get_mapping();
- HfstTransducer first = mappingPairVector[j].first;
- HfstTransducer second = mappingPairVector[j].second;
- newMappingPairVector.push_back(HfstTransducerPair(second, first));
-
- }
-
- Rule newRule(newMappingPairVector, ruleVector[i].get_context(), ruleVector[i].get_replType());
-
- leftRuleVector.push_back(newRule);
- }
-
- HfstTransducer retval(replace(leftRuleVector, optional));
- retval.invert().minimize();
-
- return retval;
- }
-
- // left to right
- HfstTransducer replace_leftmost_longest_match( const Rule &rule )
- {
-
- HfstTransducer uncondidtionalTr( bracketedReplace(rule, true) );
- //uncondidtionalTr = bracketedReplace(rule, true);
-
- //printf("LM uncondidtionalTr: \n");
- //uncondidtionalTr.write_in_att_format(stdout, 1);
-
- // for epenthesis rules
- // it can't have more than one epsilon repetition in a row
- // it should be before leftMostConstraint
- uncondidtionalTr = noRepetitionConstraint( uncondidtionalTr );
-
- HfstTransducer retval (leftMostConstraint(uncondidtionalTr));
-
- //to remove empty strings
- retval = oneBetterthanNoneConstraint(retval);
-
-
- // printf("leftMostConstraint: \n");
- // retval.write_in_att_format(stdout, 1);
- retval = longestMatchLeftMostConstraint( retval );
-
- //printf("longestMatchLeftMostConstraint: \n");
- //retval.write_in_att_format(stdout, 1);
-
-
- retval = removeB2Constraint(retval);
- retval = removeMarkers( retval );
-
- // deals with boundary symbol
- retval = applyBoundaryMark( retval );
-
- return retval;
- }
- // left to right
- HfstTransducer replace_leftmost_longest_match( const std::vector<Rule> &ruleVector )
- {
- //printf("\n replace_leftmost_longest_match \n");
-
- HfstTransducer uncondidtionalTr;
- if ( ruleVector.size() == 1 )
- {
- uncondidtionalTr = bracketedReplace( ruleVector[0], true) ;
- }
- else
- {
- uncondidtionalTr = parallelBracketedReplace(ruleVector, true);
- }
-
- //printf("retval unconditional 1 \n");
- // uncondidtionalTr.write_in_att_format(stdout, 1);
-
- // for epenthesis rules
- // it can't have more than one epsilon repetition in a row
- // it should be before leftMostConstraint
- uncondidtionalTr = noRepetitionConstraint( uncondidtionalTr );
- //printf("uncondidtionalTr epenthesis \n");
- //uncondidtionalTr.write_in_att_format(stdout, 1);
-
- HfstTransducer retval (leftMostConstraint(uncondidtionalTr));
-
- //to remove empty strings
- retval = oneBetterthanNoneConstraint(retval);
-
- retval = longestMatchLeftMostConstraint( retval );
- //printf("retval longestMatchLeftMostConstraint \n");
- //retval.write_in_att_format(stdout, 1);
-
- // remove LM2, RM2
- retval = removeB2Constraint(retval);
-
- //printf("retval removeB2Constraint \n");
- //retval.write_in_att_format(stdout, 1);
-
- retval = removeMarkers( retval );
-
-// printf("LM removeMarkers: \n");
-// retval.write_in_att_format(stdout, 1);
-
- // deals with boundary symbol
- retval = applyBoundaryMark( retval );
-
- // printf("LM applyBoundaryMark: \n");
- // retval.write_in_att_format(stdout, 1);
-
- return retval;
- }
-
-
- // right to left
- HfstTransducer replace_rightmost_longest_match( const Rule &rule )
- {
-
- HfstTransducer uncondidtionalTr(bracketedReplace(rule, true));
- //uncondidtionalTr = bracketedReplace(rule, true);
-
- HfstTransducer retval (rightMostConstraint(uncondidtionalTr));
- //retval = rightMostConstraint(uncondidtionalTr);
-
- //printf("rightMostConstraint: \n");
- //retval.write_in_att_format(stdout, 1);
-
- retval = longestMatchRightMostConstraint( retval );
-
- //printf("longestMatchLeftMostConstraint: \n");
- //retval.write_in_att_format(stdout, 1);
-
- // for epenthesis rules
- // it can't have more than one epsilon repetition in a row
- retval = noRepetitionConstraint( retval );
- // remove LM2, RM2
- retval = removeB2Constraint(retval);
-
- retval = removeMarkers( retval );
-
-
- // deals with boundary symbol
- retval = applyBoundaryMark( retval );
-
- return retval;
- }
-
-
- // right to left
- HfstTransducer replace_rightmost_longest_match( const std::vector<Rule> &ruleVector )
- {
- HfstTransducer uncondidtionalTr;
- if ( ruleVector.size() == 1 )
- {
- uncondidtionalTr = bracketedReplace( ruleVector[0], true) ;
- }
- else
- {
- uncondidtionalTr = parallelBracketedReplace(ruleVector, true);
- }
-
- HfstTransducer retval (rightMostConstraint(uncondidtionalTr));
- //retval = rightMostConstraint(uncondidtionalTr);
-
- //printf("rightMostConstraint: \n");
- //retval.write_in_att_format(stdout, 1);
-
- retval = longestMatchRightMostConstraint( retval );
-
- //printf("longestMatchLeftMostConstraint: \n");
- //retval.write_in_att_format(stdout, 1);
-
- // for epenthesis rules
- // it can't have more than one epsilon repetition in a row
- retval = noRepetitionConstraint( retval );
- // remove LM2, RM2
- retval = removeB2Constraint(retval);
-
- retval = removeMarkers( retval );
-
- // deals with boundary symbol
- retval = applyBoundaryMark( retval );
-
- return retval;
- }
-
- HfstTransducer replace_leftmost_shortest_match( const Rule &rule)
- {
-
- HfstTransducer uncondidtionalTr(bracketedReplace(rule, true));
- // uncondidtionalTr = bracketedReplace(rule, true);
-
- // for epenthesis rules
- // it can't have more than one epsilon repetition in a row
- //has to be before leftMostConstraint
- uncondidtionalTr = noRepetitionConstraint( uncondidtionalTr );
-
- HfstTransducer retval (leftMostConstraint(uncondidtionalTr));
- //to remove empty strings
- retval = oneBetterthanNoneConstraint(retval);
-
- retval = shortestMatchLeftMostConstraint( retval );
-
- //printf("sh tr: \n");
- //retval.write_in_att_format(stdout, 1);
-
-
- // remove LM2, RM2
- retval = removeB2Constraint(retval);
-
- retval = removeMarkers( retval );
-
- // deals with boundary symbol
- retval = applyBoundaryMark( retval );
-
- return retval;
- }
-
-
- HfstTransducer replace_leftmost_shortest_match(const std::vector<Rule> &ruleVector )
- {
- HfstTransducer uncondidtionalTr;
- if ( ruleVector.size() == 1 )
- {
- uncondidtionalTr = bracketedReplace( ruleVector[0], true) ;
- }
- else
- {
- uncondidtionalTr = parallelBracketedReplace(ruleVector, true);
- }
-
- // for epenthesis rules
- // it can't have more than one epsilon repetition in a row
- uncondidtionalTr = noRepetitionConstraint( uncondidtionalTr );
-
-
- HfstTransducer retval (leftMostConstraint(uncondidtionalTr));
-
- //to remove empty strings
- retval = oneBetterthanNoneConstraint(retval);
-
- retval = shortestMatchLeftMostConstraint( retval );
-
- //printf("sh tr: \n");
- //retval.write_in_att_format(stdout, 1);
-
- // remove LM2, RM2
- retval = removeB2Constraint(retval);
-
- retval = removeMarkers( retval );
-
- // deals with boundary symbol
- retval = applyBoundaryMark( retval );
-
- return retval;
- }
-
- HfstTransducer replace_rightmost_shortest_match( const Rule &rule )
- {
-
- HfstTransducer uncondidtionalTr(bracketedReplace( rule, true));
- //uncondidtionalTr = bracketedReplace( rule, true);
-
- HfstTransducer retval (rightMostConstraint(uncondidtionalTr));
- //retval = rightMostConstraint(uncondidtionalTr);
- retval = shortestMatchRightMostConstraint( retval );
-
- //printf("sh tr: \n");
- //retval.write_in_att_format(stdout, 1);
-
-
- // for epenthesis rules
- // it can't have more than one epsilon repetition in a row
- retval = noRepetitionConstraint( retval );
- // remove LM2, RM2
- retval = removeB2Constraint(retval);
-
- retval = removeMarkers( retval );
-
- // deals with boundary symbol
- retval = applyBoundaryMark( retval );
-
- return retval;
- }
-
- HfstTransducer replace_rightmost_shortest_match( const std::vector<Rule> &ruleVector )
- {
- HfstTransducer uncondidtionalTr;
- if ( ruleVector.size() == 1 )
- {
- uncondidtionalTr = bracketedReplace( ruleVector[0], true) ;
- }
- else
- {
- uncondidtionalTr = parallelBracketedReplace(ruleVector, true);
- }
- HfstTransducer retval (rightMostConstraint(uncondidtionalTr));
- //retval = rightMostConstraint(uncondidtionalTr);
- retval = shortestMatchRightMostConstraint( retval );
-
- //printf("sh tr: \n");
- //retval.write_in_att_format(stdout, 1);
-
-
- // for epenthesis rules
- // it can't have more than one epsilon repetition in a row
- retval = noRepetitionConstraint( retval );
- // remove LM2, RM2
- retval = removeB2Constraint(retval);
-
- retval = removeMarkers( retval );
-
- // deals with boundary symbol
- retval = applyBoundaryMark( retval );
-
- return retval;
- }
-
-
-
- // replace up, left, right, down
- HfstTransducer replace_epenthesis( const Rule &rule, bool optional)
- {
- return replace(rule, optional);
- }
-
- // replace up, left, right, down
- HfstTransducer replace_epenthesis( const std::vector<Rule> &ruleVector, bool optional)
- {
- return replace(ruleVector, optional);
- }
-
-
- //---------------------------------
- // RESTRICTION FUNCTIONS
- //---------------------------------
-
-
- /*
- define U [ ? - %<D%> ] ;
-
- define CENTER [ x y | x x y y ];
-
- define L1 [ a ] ;
- define R1 [ b ] ;
-
- define L2 [ x ] ;
- define R2 [ y ] ;
-
- define RES1 [ U* L1 %<D%> U* %<D%> R1 U* ] ;
- define RES2 [ U* L2 %<D%> U* %<D%> R2 U* ] ;
-
- define CEN1 [ U* %<D%> CENTER %<D%> U* ] ;
-
- define NODU [ U | 0:%<D%> ]* ;
- define NODL [ U | %<D%>:0 ]* ;
-
- regex U* - [ NODU .o. [ CEN1 - [ RES1 | RES2 ] ] .o. NODL ] ;
- */
- HfstTransducer restriction( const HfstTransducer &_center, const HfstTransducerPairVector &context)
- {
- //check if the center is automata
- HfstTransducer c_proj1(_center);
- c_proj1.input_project();
- HfstTransducer c_proj2(_center);
- c_proj2.output_project();
-
- if ( ! c_proj1.compare(_center) || ! c_proj2.compare(_center) )
- {
- HFST_THROW_MESSAGE(TransducersAreNotAutomataException, "HfstXeroxRules::restriction");
- }
-
- ImplementationType type = _center.get_type();
- String restrictionMark("@_D_@");
-
- HfstTokenizer TOK;
- TOK.add_multichar_symbol(restrictionMark);
- TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
-
- HfstTransducer mark(restrictionMark, TOK, type);
- HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
-
- // Identity
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- HfstTransducer identity (identityPair);
- identity.repeat_star().minimize();
-
- HfstTransducer universalWithoutD(identity);
- universalWithoutD.insert_to_alphabet(restrictionMark);
- HfstTransducer universalWithoutDStar(universalWithoutD);
- universalWithoutDStar.repeat_star().minimize();
-
- // NODU
- HfstTransducer noDUpper("@_EPSILON_SYMBOL_@", restrictionMark, TOK, type );
- noDUpper.disjunct(universalWithoutD).repeat_star().minimize();
-
- // NODL
- HfstTransducer noDLower(restrictionMark, "@_EPSILON_SYMBOL_@", TOK, type );
- noDLower.disjunct(universalWithoutD).repeat_star().minimize();
-
- // 1. Surround center with marks
- // [ U* %<D%> CENTER %<D%> U* ]
- HfstTransducer center(_center);
- center.insert_to_alphabet(restrictionMark);
-
- HfstTransducer centerMarked(universalWithoutDStar);
- centerMarked.concatenate(mark)
- .concatenate(center)
- .concatenate(mark)
- .concatenate(universalWithoutDStar)
- .minimize();
-
- // 2. Put mark in context
- // [ U* L1 %<D%> U* %<D%> R1 U* ]
- HfstTransducer contextMarked;
- for ( unsigned int i = 0; i < context.size(); i++ )
- {
- HfstTransducer lefContext(context[i].first);
- lefContext.insert_to_alphabet(restrictionMark);
-
- HfstTransducer rightContext(context[i].second);
- rightContext.insert_to_alphabet(restrictionMark);
-
- HfstTransducer RES(universalWithoutDStar);
- RES.concatenate(lefContext)
- .concatenate(mark)
- .concatenate(universalWithoutDStar)
- .concatenate(mark)
- .concatenate(rightContext)
- .concatenate(universalWithoutDStar)
- .minimize();
-
- if ( i == 0 )
- {
- contextMarked = RES;
- }
- else
- {
- contextMarked.disjunct(RES).minimize();
- }
- }
- HfstTransducer centerMinusCtx(centerMarked);
- centerMinusCtx.subtract(contextMarked).minimize();
-
- HfstTransducer tmp(noDUpper);
- tmp.compose(centerMinusCtx)
- .compose(noDLower)
- .minimize();
-
- HfstTransducer retval(universalWithoutDStar);
- retval.subtract(tmp).minimize();
-
- retval.remove_from_alphabet(restrictionMark);
-
- // deals with boundary symbol
- retval = applyBoundaryMark( retval );
-
- return retval;
- }
-
-
- // a < b
- HfstTransducer before( const HfstTransducer &left, const HfstTransducer &right)
- {
- //check if the center is automata
- HfstTransducer l_proj1(left);
- l_proj1.input_project();
- HfstTransducer l_proj2(left);
- l_proj2.output_project();
- HfstTransducer r_proj1(right);
- r_proj1.input_project();
- HfstTransducer r_proj2(right);
- r_proj2.output_project();
-
- if ( ! l_proj1.compare(left) || ! l_proj2.compare(left)
- || ! r_proj1.compare(right) || ! r_proj2.compare(right) )
- {
- HFST_THROW_MESSAGE(TransducersAreNotAutomataException, "HfstXeroxRules::restriction");
- }
-
- ImplementationType type = left.get_type();
-
- // Identity
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- HfstTransducer identity (identityPair);
- identity.repeat_star().minimize();
-
- HfstTransducer tmp(identity);
- tmp.concatenate(right)
- .concatenate(identity)
- .concatenate(left)
- .concatenate(identity)
- .minimize();
-
- HfstTransducer retval(identity);
- retval.subtract(tmp).minimize();
-
- return retval;
- }
- // a > b
- HfstTransducer after( const HfstTransducer &left, const HfstTransducer &right)
- {
- //check if the center is automata
- HfstTransducer l_proj1(left);
- l_proj1.input_project();
- HfstTransducer l_proj2(left);
- l_proj2.output_project();
- HfstTransducer r_proj1(right);
- r_proj1.input_project();
- HfstTransducer r_proj2(right);
- r_proj2.output_project();
-
- if ( ! l_proj1.compare(left) || ! l_proj2.compare(left)
- || ! r_proj1.compare(right) || ! r_proj2.compare(right) )
- {
- HFST_THROW_MESSAGE(TransducersAreNotAutomataException, "HfstXeroxRules::restriction");
- }
-
- ImplementationType type = left.get_type();
-
- // Identity
- HfstTransducer identityPair = HfstTransducer::identity_pair( type );
- HfstTransducer identity (identityPair);
- identity.repeat_star().minimize();
-
- HfstTransducer tmp(identity);
- tmp.concatenate(left)
- .concatenate(identity)
- .concatenate(right)
- .concatenate(identity)
- .minimize();
-
- HfstTransducer retval(identity);
- retval.subtract(tmp).minimize();
-
- return retval;
- }
-
-
-
-
-
- }
-}
-#else
-
-#include "HfstXeroxRulesTest.cc"
-
-int main(int argc, char * argv[])
- {
- std::cout << "Unit tests for " __FILE__ ":" << std::endl;
-
-
- ImplementationType types[] = {SFST_TYPE, TROPICAL_OPENFST_TYPE, FOMA_TYPE};
- unsigned int NUMBER_OF_TYPES=3;
-
-
- for (unsigned int i=0; i < NUMBER_OF_TYPES; i++)
- {
- if (! HfstTransducer::is_implementation_type_available(types[i]))
- continue;
-
- // std::cout << "----- Type --------- " << i << std::endl;
-
-
-
- // a -> b || ? - a _
- //test8( types[i] );
-
- test1( types[i] );
-
- test1b( types[i]);
-
- // ? -> a
- test1c( types[i]);
-
-
- // a -> b || .#. _ ;
- test1d( types[i]);
-
- // a+ -> x a_a
- // also @-> and @>
- test2a( types[i] );
-
- // >@ ->@
- test2b( types[i] );
- test2c( types[i] );
-
- // ? @-> a ... b;
- // test2d( types[i] );
-
- // testing unconditional replace with and without contexts
-
- test3a( types[i] );
- test3b( types[i] );
- test3c( types[i] );
- test3d( types[i] );
-
- // left - right - down tests
- // b -> a || _a (r: bbaa)
- // b -> a \\ _a (r:aaaa)
- // input: bbba
- test4a( types[i] );
- test4b( types[i] );
- test4c( types[i] );
-
- // mark up rule - doesn't have api interface anymore
- // a | b -> %[...%] ;
- // test5( types[i] );
-
- // epenthesis rules
- test6a( types[i] );
- test6b( types[i] );
- // [. 0 .] -> b || _ a a ;
- test6c( types[i] );
-
- //parralel rules
- // a -> b , b -> c
- test7a( types[i] );
- // [. .] -> b , a -> c ;
- test7b( types[i] );
- // a @-> x , b @-> y
- // a @> x , b @> y
- // a -> x \/ m _ ,, b -> y || x _
- test7c( types[i] );
- // 0 .o. [ [. 0 .] -> a \/ _ b a , a b _ ,, [. 0 .] -> b \/ a _ a ]
- test7d( types[i] );
- // ? -> x , a -> b
- test7e( types[i] );
- // markup parallel rules
- //test7e( types[i] );
-
- // a -> b, b -> a
- test7f( types[i] );
-
- // a -> b b , a -> b
- test7g( types[i] );
- //[..] @-> a;
- test7h( types[i] );
-
- test9a( types[i] );
- test9b( types[i] );
-
- // rules with empty language
- test10a( types[i] );
- // empty -> non-empty
- test10b( types[i] );
-
-
- // flag diacritics
- //a -> "@P.FOO.BAR@" || a _ "@u.a.b@"
- //test11 (types[i] );
-
-
- //markup rules
- //test12 (types[i] );
-
- // restriction functions =>
-
- restriction_test1( types[i] );
- restriction_test1a( types[i] );
- restriction_test1b( types[i] );
- restriction_test2( types[i] );
- restriction_test3( types[i] );
- restriction_test3a( types[i] );
- restriction_test3b( types[i] );
- restriction_test3c( types[i] );
- restriction_test4( types[i] );
- restriction_test5( types[i] );
- restriction_test5a( types[i] );
- restriction_test6( types[i] );
- restriction_test7( types[i] );
- restriction_test8( types[i] );
-
- before_test1( types[i] );
- }
-
- std::cout << "ok" << std::endl;
- return 0;
- }
-#endif // MAIN_TEST
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 3 of the License, or (at your option) any later version.
+// See the file COPYING included with this distribution for more
+// information.
+
+#include "HfstXeroxRules.h"
+
+#ifndef MAIN_TEST
+
+namespace hfst
+{
+ namespace xeroxRules
+ {
+ Rule::Rule ( const HfstTransducerPairVector &mappingPairVector )
+ {
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+
+ ImplementationType type = mappingPairVector[0].first.get_type();
+ // Check if all transducer types are the same
+ for ( unsigned int i = 0; i < mappingPairVector.size(); i++ )
+ {
+ if ( mappingPairVector[i].first.get_type() != type
+ || mappingPairVector[i].second.get_type() != type)
+ {
+ HFST_THROW_MESSAGE(TransducerTypeMismatchException, "Rule mapping");
+ }
+ }
+
+ HfstTransducerPair contextPair(HfstTransducer("@_EPSILON_SYMBOL_@", TOK, type),
+ HfstTransducer("@_EPSILON_SYMBOL_@", TOK, type));
+ HfstTransducerPairVector epsilonContext;
+ epsilonContext.push_back(contextPair);
+
+ mapping = mappingPairVector;
+
+ // HfstTransducerPairVector tmpV = mappingPairVector;
+ // tmpV[0].first = encodeFlagDiacritics(tmpV[0].first);
+
+ //mapping = tmpV;
+ context = epsilonContext;
+ replType = REPL_UP;
+
+ }
+ Rule::Rule ( const HfstTransducerPairVector &mappingPairVector,
+ const HfstTransducerPairVector &a_context,
+ ReplaceType a_replType )
+ {
+ // Check if all transducer types are the same
+ ImplementationType type = mappingPairVector[0].first.get_type();
+ for ( unsigned int i = 0; i < mappingPairVector.size(); i++ )
+ {
+ if ( mappingPairVector[i].first.get_type() != type
+ || mappingPairVector[i].second.get_type() != type)
+ {
+ HFST_THROW_MESSAGE(TransducerTypeMismatchException, "Rule mapping");
+ }
+ }
+ for ( unsigned int j = 0; j < a_context.size(); j++ )
+ {
+ if ( a_context[j].first.get_type() != type
+ || a_context[j].second.get_type() != type )
+ {
+ HFST_THROW_MESSAGE(TransducerTypeMismatchException, "Rule context");
+ }
+ }
+
+
+
+ //HfstTransducerPairVector tmpV = mappingPairVector;
+ //tmpV[0].first = encodeFlagDiacritics(tmpV[0].first);
+
+ mapping = mappingPairVector;
+ // mapping = tmpV ;
+ context = a_context;
+ replType = a_replType;
+ }
+
+
+ Rule::Rule ( const Rule &a_rule )
+ {
+ mapping = a_rule.get_mapping();
+ context = a_rule.get_context();
+ replType = a_rule.get_replType();
+
+ }
+
+ // for SWIG
+ Rule::Rule()
+ {
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+ hfst::ImplementationType type = hfst::ImplementationType::TROPICAL_OPENFST_TYPE;
+ HfstTransducerPair contextPair(HfstTransducer("@_EPSILON_SYMBOL_@", TOK, type),
+ HfstTransducer("@_EPSILON_SYMBOL_@", TOK, type));
+ HfstTransducerPairVector epsilonContext;
+ epsilonContext.push_back(contextPair);
+ context = epsilonContext;
+ replType = REPL_UP;
+ }
+
+ HfstTransducerPairVector Rule::get_mapping() const
+ {
+ return mapping;
+ }
+
+ HfstTransducerPairVector Rule::get_context() const
+ {
+ return context;
+ }
+ ReplaceType Rule::get_replType() const
+ {
+ return replType;
+ }
+ void Rule::encodeFlags()
+ {
+ //cerr << "\n Encode flags " << endl;
+ HfstTransducerPairVector tmpM = this->mapping;
+
+ //cerr << "\n mapping size: " << tmpM.size() << endl;
+ for ( unsigned int i = 0; i < tmpM.size(); i++ )
+ {
+ tmpM[i].first = encodeFlagDiacritics(tmpM[i].first);
+ tmpM[i].second = encodeFlagDiacritics(tmpM[i].second);
+ }
+
+
+ HfstTransducerPairVector tmpC = this->context;
+
+ //cerr << "\n context size: " << tmpC.size() << endl;
+ for ( unsigned int i = 0; i < tmpC.size(); i++ )
+ {
+ tmpC[i].first = encodeFlagDiacritics(tmpC[i].first);
+ tmpC[i].second = encodeFlagDiacritics(tmpC[i].second);
+ }
+
+ this->mapping = tmpM;
+ this->context = tmpC;
+ }
+
+ std::ostream & operator<<(std::ostream &out, const Rule & r)
+ {
+ out << "hfst::xeroxRules::Rule:" << std::endl;
+ out << "replType: ";
+ switch(r.replType)
+ {
+ case REPL_UP:
+ out << "REPL_UP";
+ break;
+ case REPL_DOWN:
+ out << "REPL_DOWN";
+ break;
+ case REPL_RIGHT:
+ out << "REPL_RIGHT";
+ break;
+ case REPL_LEFT:
+ out << "REPL_LEFT";
+ break;
+ }
+ out << std::endl;
+
+ out << "mapping:" << std::endl;
+ unsigned int index = 1;
+ for (HfstTransducerPairVector::const_iterator it = r.mapping.begin();
+ it != r.mapping.end(); it++)
+ {
+ out << "#" << index << " (right side):" << std::endl;
+ out << it->first;
+ out << "#" << index << " (left side):" << std::endl;
+ out << it->second;
+ index++;
+ }
+
+ out << "context:" << std::endl;
+ index = 1;
+ for (HfstTransducerPairVector::const_iterator it = r.context.begin();
+ it != r.context.end(); it++)
+ {
+ out << "#" << index << " (right side):" << std::endl;
+ out << it->first;
+ out << "#" << index << " (left side):" << std::endl;
+ out << it->second;
+ index++;
+ }
+
+ return out;
+ }
+
+
+ //////////////////////////////////////
+ // In the transducer tr, change all flag diacritics to "non-special" multichar symbols
+ // It means that @ sign will be changed to $ sign
+ // ie. @P.FOO.BAR@ will be changed into $P.FOO.BAR$
+ HfstTransducer encodeFlagDiacritics( const HfstTransducer &tr )
+ {
+ //std::cerr << "\n encodeFlagDiacritics " << endl;
+ HfstSymbolSubstitutions realFlagstoFakeFlags;
+ StringSet removeFromAlphabet;
+ //printf("alphabet lexicons: \n");
+ StringSet transducerAlphabet = tr.get_alphabet();
+ for (StringSet::const_iterator s = transducerAlphabet.begin();
+ s != transducerAlphabet.end();
+ ++s)
+ {
+
+ String alph = *s;
+ String alphFirst3 = alph.substr(0,3);
+ // String alphLast = alph.back();
+
+ //@operator.feature.value@ and @operator.feature@
+
+ //std::cout << "debug: " << alphFirst3 << endl;
+ if ( alphFirst3 == "@P." || alphFirst3 == "@R."
+ || alphFirst3 == "@U." || alphFirst3 == "@D."
+ || alphFirst3 == "@C." || alphFirst3 == "@N."
+ ||alphFirst3 == "@p." || alphFirst3 == "@r."
+ || alphFirst3 == "@u." || alphFirst3 == "@d."
+ || alphFirst3 == "@c." || alphFirst3 == "@n.")
+ {
+
+ replace(alph.begin(), alph.end(), '@', '$');
+ //std::cout << alph << '\n';
+ realFlagstoFakeFlags.insert(StringPair(*s, alph));
+ removeFromAlphabet.insert(*s);
+ }
+ }
+
+ HfstTransducer retval(tr);
+ retval.substitute(realFlagstoFakeFlags);
+
+ //std::cerr << "\n flag substitute done" << endl;
+ retval.remove_from_alphabet(removeFromAlphabet);
+ //std::cerr << "\n alph substitute done" << endl;
+ return retval;
+
+ }
+
+ HfstTransducer decodeFlagDiacritics( const HfstTransducer &tr )
+ {
+ //std::cerr << "\n decodeFlagDiacritics " << endl;
+ HfstSymbolSubstitutions fakeFlagsToRealFlags;
+
+ //printf("alphabet lexicons: \n");
+ StringSet transducerAlphabet = tr.get_alphabet();
+ StringSet removeFromAlphabet;
+ for (StringSet::const_iterator s = transducerAlphabet.begin();
+ s != transducerAlphabet.end();
+ ++s)
+ {
+
+ String alph = *s;
+ String alphFirst3 = alph.substr(0,3);
+ // String alphLast = alph.back();
+
+ //@operator.feature.value@ and @operator.feature@
+
+ //std::cout << "debug: " << alphFirst3 << endl;
+ if ( alphFirst3 == "$P." || alphFirst3 == "$R."
+ || alphFirst3 == "$U." || alphFirst3 == "$D."
+ || alphFirst3 == "$C." || alphFirst3 == "$N."
+ || alphFirst3 == "$p." || alphFirst3 == "$r."
+ || alphFirst3 == "$u." || alphFirst3 == "$d."
+ || alphFirst3 == "$c." || alphFirst3 == "$n.")
+ {
+
+ replace(alph.begin(), alph.end(), '$', '@');
+ //std::cout << alph << '\n';
+ fakeFlagsToRealFlags.insert(StringPair(*s, alph));
+ removeFromAlphabet.insert(*s);
+ }
+ }
+
+ HfstTransducer retval(tr);
+ retval.substitute(fakeFlagsToRealFlags);
+ retval.remove_from_alphabet(removeFromAlphabet);
+ return retval;
+
+ }
+
+
+
+ HfstTransducer disjunctVectorMembers( const HfstTransducerVector &trVector )
+ {
+ HfstTransducer retval( trVector[0] );
+ for ( unsigned int i = 1; i < trVector.size(); i++ )
+ {
+ retval.disjunct(trVector[i]).optimize();
+ }
+ return retval;
+ }
+
+
+
+ //////////////////////////////////////
+
+
+
+ HfstTransducer removeMarkers( const HfstTransducer &tr )
+ {
+ HfstTransducer retval(tr);
+
+
+ String leftMarker("@LM@");
+ String rightMarker("@RM@");
+
+ retval.substitute(StringPair(leftMarker, leftMarker), StringPair("@_EPSILON_SYMBOL_@", "@_EPSILON_SYMBOL_@")).optimize();
+ retval.substitute(StringPair(rightMarker, rightMarker), StringPair("@_EPSILON_SYMBOL_@", "@_EPSILON_SYMBOL_@")).optimize();
+
+
+ retval.remove_from_alphabet(leftMarker);
+ retval.remove_from_alphabet(rightMarker);
+
+ retval.optimize();
+
+
+
+ //printf("tr without markers: \n");
+ //retval.write_in_att_format(stdout, 1);
+
+ retval = decodeFlagDiacritics(retval);
+
+ return retval;
+ }
+
+ float zero_weight(float f) {
+ (void)f;
+ return 0;
+ }
+
+ /*
+ * Generalized Lenient Composition, described in Anssi Yli-Jyrä 2008b
+ */
+ // tmp = t.1 .o. Constr .o. t.1
+ // (t.1 - tmp.2) .o. t
+ HfstTransducer constraintComposition( const HfstTransducer &t, const HfstTransducer &Constraint )
+ {
+
+// printf("---- \n");
+// printf("orig: \n");
+// t.write_in_att_format(stdout, 1);
+// printf("---- \n");
+
+ HfstTransducer retval(t);
+ retval.transform_weights(&zero_weight);
+
+ retval.input_project().optimize();
+
+ HfstTransducer tmp(retval);
+ tmp.compose(Constraint).optimize();
+ //printf("tmp: \n");
+ //tmp.write_in_att_format(stdout, 1);
+
+ tmp.compose(retval).optimize();
+ //printf("tmp 2: \n");
+ //tmp.write_in_att_format(stdout, 1);
+ tmp.output_project().optimize();
+ retval.subtract(tmp).optimize();
+
+ //transform weights to zero
+ retval.transform_weights(&zero_weight);
+ retval.compose(t).optimize();
+
+ return retval;
+ }
+
+ void insertFreelyAllTheBrackets( HfstTransducer &t, bool optional )
+ {
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+ String leftMarker("@LM@");
+ String rightMarker("@RM@");
+ String leftMarker2("@LM2@");
+ String rightMarker2("@RM2@");
+
+ TOK.add_multichar_symbol(leftMarker);
+ TOK.add_multichar_symbol(rightMarker);
+ TOK.add_multichar_symbol(leftMarker2);
+ TOK.add_multichar_symbol(rightMarker2);
+
+ ImplementationType type = t.get_type();
+
+ HfstTransducer leftBracket(leftMarker, TOK, type);
+ HfstTransducer rightBracket(rightMarker, TOK, type);
+
+ t.insert_freely(leftBracket, false).optimize();
+ t.insert_freely(rightBracket, false).optimize();
+
+ if ( !optional )
+ {
+ HfstTransducer leftBracket2(leftMarker2, TOK, type);
+ HfstTransducer rightBracket2(rightMarker2, TOK, type);
+
+ t.insert_freely(leftBracket2, false).optimize();
+ t.insert_freely(rightBracket2, false).optimize();
+ }
+ }
+
+ HfstTransducer expandContextsWithMapping (const HfstTransducerPairVector &ContextVector,
+ const HfstTransducer &mappingWithBracketsAndTmpBoundary,
+ const HfstTransducer &identityExpanded,
+ ReplaceType replType,
+ bool optional)
+ {
+ ImplementationType type = identityExpanded.get_type();
+
+ HfstTransducer unionContextReplace(type);
+
+ HfstTokenizer TOK;
+ // TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+ // HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
+
+
+ for ( unsigned int i = 0; i < ContextVector.size(); i++ )
+ {
+
+ // Expand context with mapping
+ // Cr' = (Rc .*) << Markers (<,>,|) .o. [I:I | <a:b>]*
+ // Cr = Cr|Cr'
+ // (same for left context)
+
+ // Lc = (*. Lc) << {<,>}
+
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ HfstTransducer identityStar(identityPair);
+ identityStar.repeat_star();
+
+ HfstTransducer firstContext( identityStar);
+ firstContext.concatenate(ContextVector[i].first);
+ firstContext.transform_weights(&zero_weight);
+ firstContext.optimize();
+
+ insertFreelyAllTheBrackets( firstContext, optional );
+
+ // Rc = (Rc .*) << {<,>}
+ HfstTransducer secondContext(ContextVector[i].second);
+ secondContext.concatenate(identityStar);
+ secondContext.transform_weights(&zero_weight);
+ secondContext.optimize();
+ insertFreelyAllTheBrackets( secondContext, optional);
+
+ /* RULE: LC: RC:
+ * up up up
+ * left up down
+ * right down up
+ * down down down
+ */
+
+ HfstTransducer leftContextExpanded(type);
+ HfstTransducer rightContextExpanded(type);
+
+ // both contexts are in upper language
+ if ( replType == REPL_UP)
+ {
+ // compose them with [I:I | <a:b>]*
+ leftContextExpanded = firstContext;
+ rightContextExpanded = secondContext;
+
+ leftContextExpanded.compose(identityExpanded);
+ rightContextExpanded.compose(identityExpanded);
+ }
+ // left context is in lower language, right in upper ( // )
+ if ( replType == REPL_RIGHT )
+ {
+ // compose them with [I:I | <a:b>]*
+
+ // left compose opposite way
+ leftContextExpanded = identityExpanded;
+ rightContextExpanded = secondContext;
+
+ leftContextExpanded.compose(firstContext);
+ rightContextExpanded.compose(identityExpanded);
+ }
+ // right context is in lower language, left in upper ( \\ )
+ if ( replType == REPL_LEFT )
+ {
+ // compose them with [I:I | <a:b>]*
+ leftContextExpanded = firstContext;
+ rightContextExpanded = identityExpanded;
+
+ leftContextExpanded.compose(identityExpanded);
+ rightContextExpanded.compose(secondContext);
+ }
+ if ( replType == REPL_DOWN )
+ {
+ // compose them with [I:I | <a:b>]*
+ leftContextExpanded = identityExpanded;
+ rightContextExpanded = identityExpanded;
+
+ leftContextExpanded.compose(firstContext);
+ rightContextExpanded.compose(secondContext);
+ }
+
+
+ leftContextExpanded.transform_weights(&zero_weight);
+ rightContextExpanded.transform_weights(&zero_weight);
+ leftContextExpanded.optimize();
+ rightContextExpanded.optimize();
+
+ firstContext.disjunct(leftContextExpanded);
+ firstContext.optimize();
+
+ secondContext.disjunct(rightContextExpanded);
+ secondContext.optimize();
+
+ // add boundary symbol before/after contexts
+ String boundaryMarker(".#.");
+ TOK.add_multichar_symbol(boundaryMarker);
+ HfstTransducer boundary(boundaryMarker, TOK, type);
+
+ identityStar.insert_to_alphabet(boundaryMarker);
+
+ // to firstContext
+ StringSet firstContextAlphabet = firstContext.get_alphabet();
+ bool hasBoundary = false;
+ for (StringSet::const_iterator s = firstContextAlphabet.begin();
+ s != firstContextAlphabet.end();
+ ++s)
+ {
+ if( boundaryMarker.compare(s->c_str()) == 0 )
+ hasBoundary = true;
+ }
+
+ if ( hasBoundary == false )
+ {
+ //printf("\n - Left context boundary false! - \n");
+ firstContext.insert_to_alphabet(boundaryMarker);
+ HfstTransducer tmp(boundary);
+ tmp.concatenate(identityStar).optimize();
+ //printf("tmp \n");
+ //tmp.write_in_att_format(stdout, 1);
+ tmp.concatenate(firstContext);
+
+ //printf("tmp2 \n");
+ //tmp.write_in_att_format(stdout, 1);
+ firstContext = tmp;
+
+ }
+
+ // to secondContext
+ StringSet secondContextAlphabet = secondContext.get_alphabet();
+ hasBoundary = false;
+ for (StringSet::const_iterator s = secondContextAlphabet.begin();
+ s != secondContextAlphabet.end();
+ ++s)
+ {
+ if( boundaryMarker.compare(s->c_str()) == 0 )
+ hasBoundary = true;
+ }
+
+ if ( hasBoundary == false )
+ {
+ // printf("\n - Right context boundary false! - \n");
+ secondContext.insert_to_alphabet(boundaryMarker);
+ secondContext.concatenate(identityStar).concatenate(boundary).optimize();
+ }
+
+ // put mapping between (expanded) contexts
+ HfstTransducer oneContextReplace(firstContext);
+ oneContextReplace.concatenate(mappingWithBracketsAndTmpBoundary).
+ concatenate(secondContext);
+
+
+ oneContextReplace.transform_weights(&zero_weight);
+ unionContextReplace.disjunct(oneContextReplace);
+ unionContextReplace.optimize();
+
+ }
+ return unionContextReplace;
+ }
+
+
+
+ /*
+ * unconditional replace, in multiple contexts
+ * first: (.* T<a:b>T .*) - [( .* L1 T<a:b>T R1 .*) u (.* L2 T<a:b>T R2 .*)...],
+ * where .* = [I:I (+ {tmpMarker (T), <,>} in alphabet) | <a:b>]*
+ * then: remove tmpMarker from transducer and alphabet, and do negation:
+ * .* - result from upper operations
+ *
+ * Also note that context is expanded in this way:
+ * Cr' = (Rc .*) << Markers (<,>,|) .o. [I:I | <a:b>]*
+ * Cr = Cr | Cr'
+ * (same for left context, (.* Cl))
+ */
+
+ HfstTransducer bracketedReplace(const Rule &rule, bool optional)
+ {
+ //printf("bracketedReplace function..... \n");
+
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+ TOK.add_multichar_symbol("@_UNKNOWN_SYMBOL_@");
+ String leftMarker("@LM@");
+ String rightMarker("@RM@");
+ String tmpMarker("@TMPM@");
+ String leftMarker2("@LM2@");
+ String rightMarker2("@RM2@");
+ String newEpsilon("$Epsilon$");
+
+ TOK.add_multichar_symbol(leftMarker);
+ TOK.add_multichar_symbol(rightMarker);
+ TOK.add_multichar_symbol(leftMarker2);
+ TOK.add_multichar_symbol(rightMarker2);
+ TOK.add_multichar_symbol(tmpMarker);
+ TOK.add_multichar_symbol(newEpsilon);
+ TOK.add_multichar_symbol( ".#.");
+
+ //first, encode all flag diacritics
+ Rule ruletmp(rule);
+ ruletmp.encodeFlags();
+
+ HfstTransducerPairVector mappingPairVector( ruletmp.get_mapping() );
+ HfstTransducerPairVector ContextVector( ruletmp.get_context() );
+ ReplaceType replType( ruletmp.get_replType() );
+
+ ImplementationType type = mappingPairVector[0].first.get_type();
+
+ // Identity (normal)
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ HfstTransducer identity (identityPair);
+ identity.repeat_star().optimize();
+
+
+ HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
+ HfstTransducer mapping(type);
+ for ( unsigned int i = 0; i < mappingPairVector.size(); i++ )
+ {
+ HfstTransducer oneMappingPair(mappingPairVector[i].first);
+
+ //markup rules are already cross product in mappingPairVector[i].first (second is empty)
+ //so the cross product should not be done for markup rules
+ if ( mappingPairVector[i].first.get_property("isMarkup") != "yes" )
+ {
+ oneMappingPair.cross_product(mappingPairVector[i].second);
+ }
+ // printf("aftrer cross product \n");
+ // oneMappingPair.write_in_att_format(stdout, 1);
+
+
+ // for removing .#. from the center
+ HfstTransducer identityWithoutBoundary(identity);
+ identityWithoutBoundary.insert_to_alphabet(".#.");
+ HfstTransducer removeHash(identityWithoutBoundary);
+ HfstTransducer boundary(".#.", TOK, type);
+ removeHash.concatenate(boundary).concatenate(identityWithoutBoundary).optimize();
+ //printf("removeHash \n");
+ //removeHash.write_in_att_format(stdout, 1);
+
+
+ // printf("oneMappingPair kkkk\n");
+ // oneMappingPair.write_in_att_format(stdout, 1);
+
+
+ if ( i == 0 )
+ {
+ // remove .#. from the center
+ // center - (?* .#. ?*)
+ oneMappingPair.subtract(removeHash, false).optimize();
+ oneMappingPair.remove_from_alphabet(".#.");
+ mapping = oneMappingPair;
+ }
+ else
+ {
+ oneMappingPair.subtract(removeHash, false).optimize();
+ oneMappingPair.remove_from_alphabet(".#.");
+ mapping.disjunct(oneMappingPair).optimize();
+ }
+ }
+ //printf("mapping all after cross product \n");
+ // mapping.write_in_att_format(stdout, 1);
+
+
+ // In case of ? -> x replacement
+ // If left side is empty, return identity transducer
+ // If right side is empty, return identity transducer
+ // with alphabet from the left side
+ HfstTransducer empty(type);
+ if ( mapping.compare(empty) )
+ {
+ mapping = identity;
+ if (mappingPairVector[0].second.compare(empty))
+ {
+ //printf("alphabet: \n");
+ StringSet transducerAlphabet = mappingPairVector[0].first.get_alphabet();
+ for (StringSet::const_iterator s = transducerAlphabet.begin();
+ s != transducerAlphabet.end();
+ ++s)
+ {
+ //printf("%s \n", s->c_str());
+ mapping.insert_to_alphabet(s->c_str());
+ }
+ //printf("------------------ \n");
+ }
+ }
+
+ mapping.insert_to_alphabet(leftMarker);
+ mapping.insert_to_alphabet(rightMarker);
+ mapping.insert_to_alphabet(tmpMarker);
+
+ HfstTransducer leftBracket(leftMarker, TOK, type);
+ HfstTransducer rightBracket(rightMarker, TOK, type);
+ HfstTransducer leftBracket2(leftMarker2, TOK, type);
+ HfstTransducer rightBracket2(rightMarker2, TOK, type);
+ HfstTransducer tmpBracket(tmpMarker, TOK, type);
+
+ // Surround mapping with brackets
+ HfstTransducer tmpMapping(leftBracket);
+ tmpMapping.concatenate(mapping).concatenate(rightBracket).optimize();
+
+ HfstTransducer mappingWithBrackets(tmpMapping);
+
+ //printf("mappingWithBrackets: \n");
+ //mappingWithBrackets.optimize().write_in_att_format(stdout, 1);
+
+ // Identity pair
+ // HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ // for non-optional replacements
+ if ( optional != true )
+ {
+ // non - optional
+ // mapping = <a:b> u <2a:a>2
+
+ HfstTransducer mappingWithBrackets2(leftBracket2);
+ HfstTransducer leftMappingUnion(type);
+ leftMappingUnion = mappingPairVector[0].first;
+ for ( unsigned int i = 1; i < mappingPairVector.size(); i++ )
+ {
+ leftMappingUnion.disjunct(mappingPairVector[i].first).optimize();
+ }
+ // needed in case of ? -> x replacement
+ leftMappingUnion.insert_to_alphabet(leftMarker2);
+ leftMappingUnion.insert_to_alphabet(rightMarker2);
+ leftMappingUnion.insert_to_alphabet(leftMarker);
+ leftMappingUnion.insert_to_alphabet(rightMarker);
+ leftMappingUnion.insert_to_alphabet(tmpMarker);
+
+ //printf("leftMappingUnion: \n");
+ //leftMappingUnion.optimize().write_in_att_format(stdout, 1);
+
+
+ mappingWithBrackets2.concatenate(leftMappingUnion).concatenate(rightBracket2).optimize();
+
+ //printf("mappingWithBrackets2: \n");
+ //mappingWithBrackets2.optimize().write_in_att_format(stdout, 1);
+
+ // mappingWithBrackets...... expanded
+ mappingWithBrackets.insert_to_alphabet(leftMarker2);
+ mappingWithBrackets.insert_to_alphabet(rightMarker2);
+ //mappingWithBrackets.insert_to_alphabet(leftMarker);
+ //mappingWithBrackets.insert_to_alphabet(rightMarker);
+ mappingWithBrackets.disjunct(mappingWithBrackets2).optimize();
+ }
+
+ //printf("mappingWithBrackets: \n");
+ //mappingWithBrackets.optimize().write_in_att_format(stdout, 1);
+
+
+ // Identity with bracketed mapping and marker symbols and TmpMarker in alphabet
+ // [I:I | <a:b>]* (+ tmpMarker in alphabet)
+ HfstTransducer identityExpanded (identityPair);
+
+ identityExpanded.insert_to_alphabet(leftMarker);
+ identityExpanded.insert_to_alphabet(rightMarker);
+ identityExpanded.insert_to_alphabet(tmpMarker);
+
+ if ( optional != true )
+ {
+ identityExpanded.insert_to_alphabet(leftMarker2);
+ identityExpanded.insert_to_alphabet(rightMarker2);
+ }
+
+ identityExpanded.disjunct(mappingWithBrackets).optimize();
+ identityExpanded.repeat_star().optimize();
+
+ // when there aren't any contexts, result is identityExpanded
+ if ( ContextVector.size() == 1 )
+ {
+ HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
+ if ( ContextVector[0].first.compare(epsilon) && ContextVector[0].second.compare(epsilon) )
+ {
+ identityExpanded.remove_from_alphabet(tmpMarker);
+ //printf("identityExpanded: \n");
+ //identityExpanded.optimize().write_in_att_format(stdout, 1);
+ return identityExpanded;
+ }
+ }
+
+ // Surround mapping with tmp boudaries
+ HfstTransducer mappingWithBracketsAndTmpBoundary(tmpBracket);
+ mappingWithBracketsAndTmpBoundary.concatenate(mappingWithBrackets).concatenate(tmpBracket).optimize();
+ //printf("mappingWithBracketsAndTmpBoundary: \n");
+ //mappingWithBracketsAndTmpBoundary.write_in_att_format(stdout, 1);
+
+
+ // .* |<a:b>| :*
+ HfstTransducer bracketedReplace(identityExpanded);
+ bracketedReplace.concatenate(mappingWithBracketsAndTmpBoundary).concatenate(identityExpanded).optimize();
+
+ //printf("mappingWithBracketsAndTmpBoundary: \n");
+ //mappingWithBracketsAndTmpBoundary.write_in_att_format(stdout, 1);
+
+ // printf("identityExpanded: \n");
+ //identityExpanded.write_in_att_format(stdout, 1);
+
+ // Expand all contexts with mapping taking in consideration replace type
+ // result is their union
+ HfstTransducer unionContextReplace(type);
+ unionContextReplace = expandContextsWithMapping (ContextVector,
+ mappingWithBracketsAndTmpBoundary,
+ identityExpanded,
+ replType,
+ optional);
+
+ //printf("unionContextReplace \n");
+ //unionContextReplace.write_in_att_format(stdout, 1);
+
+ // subtract all mappings in contexts from replace without contexts
+ HfstTransducer replaceWithoutContexts(bracketedReplace);
+ replaceWithoutContexts.subtract(unionContextReplace).optimize();
+
+ //printf("replaceWithoutContexts \n");
+ //replaceWithoutContexts.write_in_att_format(stdout, 1);
+
+ // remove tmpMaprker
+ replaceWithoutContexts.substitute(StringPair(tmpMarker, tmpMarker),
+ StringPair("@_EPSILON_SYMBOL_@", "@_EPSILON_SYMBOL_@")).optimize();
+ replaceWithoutContexts.remove_from_alphabet(tmpMarker);
+ replaceWithoutContexts.optimize();
+
+ identityExpanded.remove_from_alphabet(tmpMarker);
+
+ // final negation
+ HfstTransducer uncondidtionalTr(identityExpanded);
+ uncondidtionalTr.subtract(replaceWithoutContexts).optimize();
+
+ return uncondidtionalTr;
+
+ }
+
+
+ // Return the string "@N@" where N is the string representation of i.
+ static std::string getMarkerString(unsigned int i)
+ {
+ std::ostringstream oss;
+ oss << i;
+ return std::string("@") + oss.str() + std::string("@");
+ }
+
+ // Return the number representation of N in string "@N@".
+ static unsigned int getMarkerNumber(const std::string & str)
+ {
+ std::string number_str = str.substr(1, str.size()-2);
+ std::istringstream iss; // should be iss(number_str); i guess, but cannot be fixed, because some HfstXeroxRules tests will fail...
+ unsigned int retval;
+ iss >> retval;
+ //return retval;
+ return 100000;
+ }
+
+ // Bracketed replace for parallel rules.
+ HfstTransducer parallelBracketedReplace
+ (const std::vector<Rule> &ruleVector, bool optional)
+ {
+ // For each parallel rule, we need to concatenate a special marker symbol
+ // to its output side. This is needed so that overlapping mappings with
+ // different weights and contexts are kept separate. If we have N rules,
+ // we need marker symbols "@1@", "@2@", ... , "@N@" ("@0@" is reserved
+ // for epsilon symbol). At the end, we must substitute any marker symbols
+ // with epsilons.
+ // For an example, consider the following case:
+ // a -> b::1 || c _ c ,, a -> b::2 || d _ d
+ // If no markers are added, input 'dad' will yield 'dbd' with weight 1
+ // because the transition <a:b::2> will be eliminated by the transition
+ // <a:b::1> as it has a lower weight.
+
+ StringSet marker_symbols; // "@1@", "@2@", ... , "@N@"
+ HfstSymbolSubstitutions marker_substitutions; // any-marker-to-epsilon
+ for (unsigned int i=0; i < ruleVector.size(); i++)
+ {
+ std::string marker_string = getMarkerString(i+1);
+ marker_symbols.insert(marker_string);
+ marker_substitutions[marker_string] = hfst::internal_epsilon;
+ }
+
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+ String leftMarker("@LM@");
+ String rightMarker("@RM@");
+
+ String leftMarker2("@LM2@");
+ String rightMarker2("@RM2@");
+
+ String tmpMarker("@TMPM@");
+ TOK.add_multichar_symbol(leftMarker);
+ TOK.add_multichar_symbol(rightMarker);
+ TOK.add_multichar_symbol(leftMarker2);
+ TOK.add_multichar_symbol(rightMarker2);
+ TOK.add_multichar_symbol(tmpMarker);
+ TOK.add_multichar_symbol( ".#.");
+
+ ImplementationType type
+ = ruleVector[0].get_mapping()[0].first.get_type();
+
+ HfstTransducer leftBracket(leftMarker, TOK, type);
+ HfstTransducer rightBracket(rightMarker, TOK, type);
+ HfstTransducer leftBracket2(leftMarker2, TOK, type);
+ HfstTransducer rightBracket2(rightMarker2, TOK, type);
+ HfstTransducer tmpBracket(tmpMarker, TOK, type);
+
+
+ // Identity pair (unknowns/identities must not be expanded to marker
+ // symbols)
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ identityPair.insert_to_alphabet(marker_symbols);
+
+ HfstTransducer identity (identityPair);
+ // unknowns/identities must not be expanded to marker symbols
+ identity.insert_to_alphabet(marker_symbols);
+ identity.repeat_star().optimize();
+
+ HfstTransducer identityExpanded(identityPair);
+ identityExpanded.insert_to_alphabet(leftMarker);
+ identityExpanded.insert_to_alphabet(rightMarker);
+ identityExpanded.insert_to_alphabet(leftMarker2);
+ identityExpanded.insert_to_alphabet(rightMarker2);
+ identityExpanded.insert_to_alphabet(tmpMarker);
+ identityExpanded.insert_to_alphabet(marker_symbols);
+ // will be expanded with mappings
+
+ // for removing .#. from the center
+ HfstTransducer identityWithoutBoundary(identity);
+ identityWithoutBoundary.insert_to_alphabet(".#.");
+ // (must not be expanded to marker symbols)
+ identityWithoutBoundary.insert_to_alphabet(marker_symbols);
+ HfstTransducer removeHash(identityWithoutBoundary);
+ HfstTransducer boundary(".#.", TOK, type);
+ removeHash.concatenate(boundary)
+ .concatenate(identityWithoutBoundary).optimize();
+ //printf("removeHash \n");
+ //removeHash.write_in_att_format(stdout, 1);
+
+ HfstTransducerVector mappingWithBracketsVector;
+ bool noContexts = true;
+
+ // go through vector and do everything for each rule
+ for ( unsigned int i = 0; i < ruleVector.size(); i++ )
+ {
+ Rule ruletmp(ruleVector[i]);
+ ruletmp.encodeFlags();
+
+ HfstTransducerPairVector mappingPairVector
+ = ruletmp.get_mapping();
+ HfstTransducer mapping(type);
+ for ( unsigned int j = 0; j < mappingPairVector.size(); j++ )
+ {
+ // i+1 because @0@ is epsilon..
+ std::string marker_string = getMarkerString(i+1);
+ HfstTransducer marker(marker_string, type);
+ HfstTransducer oneMappingPair(mappingPairVector[j].first);
+ // unknowns/identities must not be expanded to marker symbols
+ oneMappingPair.insert_to_alphabet(marker_symbols);
+ HfstTransducer foo(mappingPairVector[j].second);
+ foo.insert_to_alphabet(marker_symbols);
+ oneMappingPair.cross_product
+ (foo.concatenate(marker));
+ //oneMappingPair.cross_product(mappingPairVector[j].second);
+
+ //printf("oneMappingPair \n");
+ //oneMappingPair.write_in_att_format(stdout, 1);
+
+
+ if ( j == 0 )
+ {
+ // remove .#. from the center
+ // center - (?* .#. ?*)
+ oneMappingPair.subtract(removeHash, false).optimize();
+ oneMappingPair.remove_from_alphabet(".#.");
+ mapping = oneMappingPair;
+ }
+ else
+ {
+ oneMappingPair.subtract(removeHash, false).optimize();
+ oneMappingPair.remove_from_alphabet(".#.");
+ mapping.disjunct(oneMappingPair).optimize();
+ }
+ }
+
+
+ HfstTransducerPairVector contextVector = ruletmp.get_context();
+ //ReplaceType replaceType = ruletmp.get_replType();
+
+ // when there aren't any contexts, result is identityExpanded
+ if ( contextVector.size() == 1 )
+ {
+ HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
+ if ( !(contextVector[0].first.compare(epsilon) &&
+ contextVector[0].second.compare(epsilon)) )
+ {
+ noContexts = false;
+ }
+ }
+
+ //////////////////////////////////////////////////////////////////
+ // In case of ? -> x replacement
+ // If left side is empty, return identity transducer
+ // If right side is empty, return identity transducer
+ // with alphabet from the left side
+ HfstTransducer empty(type);
+
+ if ( mapping.compare(empty) )
+ {
+ mapping = identity;
+ if (mappingPairVector[0].second.compare(empty))
+ {
+ //printf("alphabet: \n");
+ StringSet transducerAlphabet
+ = mappingPairVector[0].first.get_alphabet();
+ for (StringSet::const_iterator s
+ = transducerAlphabet.begin();
+ s != transducerAlphabet.end();
+ ++s)
+ {
+ //printf("%s \n", s->c_str());
+ mapping.insert_to_alphabet(s->c_str());
+ }
+ //printf("------------------ \n");
+ }
+ }
+ //////////////////////////////////////////////////////////////////
+
+
+ mapping.insert_to_alphabet(leftMarker);
+ mapping.insert_to_alphabet(rightMarker);
+ mapping.insert_to_alphabet(tmpMarker);
+
+ // Surround mapping with brackets
+ HfstTransducer mappingWithBrackets(leftBracket);
+ mappingWithBrackets.concatenate(mapping)
+ .concatenate(rightBracket).optimize();
+
+
+ // non - optional
+ // mapping = <a:b> u <2a:a>2
+ if ( optional != true )
+ {
+ // needed in case of ? -> x replacement
+ mapping.insert_to_alphabet(leftMarker2);
+ mapping.insert_to_alphabet(rightMarker2);
+ mappingWithBrackets.insert_to_alphabet(leftMarker2);
+ mappingWithBrackets.insert_to_alphabet(rightMarker2);
+
+
+ HfstTransducer mappingProject(mapping);
+ mappingProject.input_project().optimize();
+
+ HfstTransducer mappingWithBracketsNonOptional(leftBracket2);
+ // needed in case of ? -> x replacement
+ //mappingWithBracketsNonOptional.insert_to_alphabet(leftMarker2);
+ //mappingWithBracketsNonOptional.
+ //insert_to_alphabet(rightMarker2);
+
+ mappingWithBracketsNonOptional.concatenate(mappingProject).
+ concatenate(rightBracket2).
+ optimize();
+ // mappingWithBrackets...... expanded
+ mappingWithBrackets.disjunct(mappingWithBracketsNonOptional)
+ .optimize();
+
+ }
+
+ identityExpanded.disjunct(mappingWithBrackets).optimize();
+ mappingWithBracketsVector.push_back(mappingWithBrackets);
+ }
+
+ identityExpanded.repeat_star().optimize();
+
+ // if none of the rules have contexts, return identityExpanded
+ if ( noContexts )
+ {
+ identityExpanded.remove_from_alphabet(tmpMarker);
+ // substitute markers with epsilons
+ identityExpanded.substitute(marker_substitutions);
+ identityExpanded.remove_from_alphabet(marker_symbols);
+ return identityExpanded;
+ }
+
+ // if they have contexts, process them
+ if ( ruleVector.size() != mappingWithBracketsVector.size() )
+ {
+ HFST_THROW_MESSAGE(TransducerTypeMismatchException,
+ "Vector sizes don't match");
+ }
+
+ std::map<std::string,hfst::HfstBasicTransducer> contextReplaceMap;
+
+
+ HfstTransducer unionContextReplace(type);
+ HfstTransducer bracketedReplace(type);
+ //THIS is for disjuncting labels first, and then substitute them with
+ // transducers
+ //HfstTransducer unionContextReplace_labels(type);
+ for ( unsigned int i = 0; i < ruleVector.size(); i++ )
+ {
+ Rule ruletmp(ruleVector[i]);
+ ruletmp.encodeFlags();
+
+ // Surround mapping with brackets with tmp boudaries
+ HfstTransducer mappingWithBracketsAndTmpBoundary(tmpBracket);
+ mappingWithBracketsAndTmpBoundary
+ .concatenate(mappingWithBracketsVector[i])
+ .concatenate(tmpBracket)
+ .optimize();
+ // .* |<a:b>| :*
+ HfstTransducer bracketedReplaceTmp(identityExpanded);
+ bracketedReplaceTmp.concatenate(mappingWithBracketsAndTmpBoundary)
+ .concatenate(identityExpanded)
+ .optimize();
+
+ bracketedReplaceTmp.transform_weights(&zero_weight);
+ bracketedReplace.disjunct(bracketedReplaceTmp).optimize();
+
+ //Create context part
+ HfstTransducer unionContextReplaceTmp(type);
+
+ // For each context that uses the output side (REPL_DOWN,
+ // REPL_LEFT, REPL_RIGHT) we must freely allow all markers that can
+ // be generated by other rules. Consider e.g. the following case:
+ //
+ // C -> d::1 || \/ b _ b ,, a::1 -> b || c _ C , C _ c
+ //
+ // where the second rule yields caCac -> cbCbc and the first one
+ // again cbCbc -> cbdbc.
+ HfstTransducerPairVector cont = ruletmp.get_context();
+
+ if (ruletmp.get_replType() != REPL_UP)
+ {
+ for (HfstTransducerPairVector::iterator cont_it = cont.begin();
+ cont_it != cont.end(); cont_it++)
+ {
+ for (StringSet::const_iterator sit
+ = marker_symbols.begin();
+ sit != marker_symbols.end(); sit++)
+ {
+ if (getMarkerNumber(*sit) != i)
+ {
+ StringPair marker_pair(*sit, *sit);
+ // 'false' makes sure harmonization is not done
+ cont_it->first.insert_freely(marker_pair, false);
+ cont_it->second.insert_freely(marker_pair, false);
+ }
+ }
+ }
+ }
+
+ unionContextReplaceTmp
+ = expandContextsWithMapping ( cont,
+ mappingWithBracketsAndTmpBoundary,
+ identityExpanded,
+ ruletmp.get_replType(),
+ optional);
+
+ unionContextReplaceTmp.transform_weights(&zero_weight);
+
+ unionContextReplace.disjunct(unionContextReplaceTmp).optimize();
+ /*
+ //THIS part is for disjuncting labels first, and then substitute
+ // them with transducers
+ //create map with context keys and basic transducers
+ stringstream tmpString;
+ tmpString << "$_contextRepl." << i << "_$";
+ string contx_key = tmpString.str();
+ TOK.add_multichar_symbol(contx_key);
+ // cout << "lexicon " << regex_key << "\n";
+
+ if (contextReplaceMap.find(contx_key) == contextReplaceMap.end())
+ {
+ contextReplaceMap.insert(pair<string,HfstBasicTransducer>(contx_key,
+ HfstBasicTransducer(unionContextReplaceTmp)));
+ }
+
+ //disjunct all keys
+ unionContextReplace_labels.disjunct(HfstTransducer(contx_key,
+ TOK, type)).optimize();
+
+ // printf("unionContextReplace_labels: \n");
+ // unionContextReplace_labels.write_in_att_format(stdout, 1);
+
+ */
+ }
+
+ //printf("unionContextReplace_labels: \n");
+ //unionContextReplace.write_in_att_format(stdout, 1);
+
+
+
+ /*
+ //THIS part is for disjuncting labels first, and then substitute them
+ with transducers
+ printf("substitute labels with real tr: \n");
+ //substitute labels with real tr
+ HfstBasicTransducer btr(unionContextReplace_labels);
+ btr.substitute(contextReplaceMap, true);
+ btr.prune_alphabet();
+
+ printf("Substitute done. \n");
+ HfstTransducer unionContextReplace(btr, type);
+ printf("Created tr \n");
+
+
+ //weight can be zero, because it will be subtracted
+ bracketedReplace.transform_weights(&zero_weight);
+ unionContextReplace.transform_weights(&zero_weight);
+
+ printf("min \n");
+ unionContextReplace.optimize();
+ printf("min done. \n");
+
+ */
+
+ /////////////////////}
+ //printf("subtract all mappings in contexts from replace without
+ //contexts: \n");
+ // subtract all mappings in contexts from replace without contexts
+ HfstTransducer replaceWithoutContexts(bracketedReplace);
+ replaceWithoutContexts.subtract(unionContextReplace).optimize();
+
+ //printf("remove bla bla: \n");
+ // remove tmpMaprker
+ replaceWithoutContexts.substitute
+ (StringPair(tmpMarker, tmpMarker),
+ StringPair("@_EPSILON_SYMBOL_@", "@_EPSILON_SYMBOL_@")).optimize();
+ replaceWithoutContexts.remove_from_alphabet(tmpMarker);
+ replaceWithoutContexts.optimize();
+
+ identityExpanded.remove_from_alphabet(tmpMarker);
+
+ //printf("before final negation: \n");
+ //replaceWithoutContexts.write_in_att_format(stdout, 1);
+
+ //printf("final subtract: \n");
+ // final negation
+ HfstTransducer uncondidtionalTr(identityExpanded);
+ uncondidtionalTr.subtract(replaceWithoutContexts).optimize();
+
+ //printf("uncondidtionalTr: \n");
+ //uncondidtionalTr.write_in_att_format(stdout, 1);
+
+ // substitute markers with epsilons
+ uncondidtionalTr.substitute(marker_substitutions);
+ uncondidtionalTr.remove_from_alphabet(marker_symbols);
+
+ return uncondidtionalTr;
+
+ }
+
+
+ //---------------------------------
+ // CONSTRAINTS
+ //---------------------------------
+
+ // (help function)
+ // returns: [ B:0 | 0:B | ?-B ]*
+ // which is used in some constraints
+ HfstTransducer constraintsRightPart( ImplementationType type )
+ {
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+
+ String leftMarker("@LM@");
+ String rightMarker("@RM@");
+ TOK.add_multichar_symbol(leftMarker);
+ TOK.add_multichar_symbol(rightMarker);
+
+ // Identity pair (normal)
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+
+
+ HfstTransducer leftBracket(leftMarker, TOK, type);
+ HfstTransducer rightBracket(rightMarker, TOK, type);
+
+ // Create Right Part
+ HfstTransducer B(leftBracket);
+ B.disjunct(rightBracket).optimize();
+
+ HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
+ HfstTransducer epsilonToLeftMark("@_EPSILON_SYMBOL_@", leftMarker, TOK, type);
+ HfstTransducer LeftMarkToEpsilon(leftMarker, "@_EPSILON_SYMBOL_@", TOK, type);
+
+ HfstTransducer epsilonToBrackets(epsilon);
+ epsilonToBrackets.cross_product(B);
+
+ HfstTransducer bracketsToEpsilon(B);
+ bracketsToEpsilon.cross_product(epsilon);
+
+ HfstTransducer identityPairMinusBrackets(identityPair);
+ identityPairMinusBrackets.subtract(B).optimize();//.repeat_plus().optimize();
+
+ HfstTransducer rightPart(epsilonToBrackets);
+ rightPart.disjunct(bracketsToEpsilon).disjunct(identityPairMinusBrackets).optimize().repeat_star().optimize();
+
+ return rightPart;
+ }
+
+ // .#. ?* <:0 0:> ?* .#.
+ // filters out empty string
+ HfstTransducer oneBetterthanNoneConstraint( const HfstTransducer &uncondidtionalTr )
+ {
+ ImplementationType type = uncondidtionalTr.get_type();
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+ TOK.add_multichar_symbol(".#.");
+
+ String leftMarker("@LM@");
+ String rightMarker("@RM@");
+ TOK.add_multichar_symbol(leftMarker);
+ TOK.add_multichar_symbol(rightMarker);
+
+ // Identity (normal)
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ HfstTransducer identity (identityPair);
+ identity.repeat_star().optimize();
+
+ HfstTransducer leftBracketToZero(leftMarker, "@_EPSILON_SYMBOL_@", TOK, type);
+ HfstTransducer rightBracketToZero(rightMarker, "@_EPSILON_SYMBOL_@", TOK, type);
+
+
+ HfstTransducer boundary(".#.", TOK, type);
+ HfstTransducer Constraint(boundary);
+ Constraint.concatenate(identity);
+ Constraint.concatenate(leftBracketToZero)
+ .concatenate(rightBracketToZero)
+ .concatenate(boundary)
+ .concatenate(identity)
+ .optimize();
+
+// printf("Constraint: \n");
+// Constraint.write_in_att_format(stdout, 1);
+
+ //// Compose with unconditional replace transducer
+ // tmp = t.1 .o. Constr .o. t.1
+ // (t.1 - tmp.2) .o. t
+
+
+ HfstTransducer retval(type);
+ retval = constraintComposition(uncondidtionalTr, Constraint);
+
+
+ return retval;
+ }
+
+
+
+ // .#. ?* <:0 [B:0]* [I-B] [ B:0 | 0:B | ?-B ]* .#.
+ HfstTransducer leftMostConstraint( const HfstTransducer &uncondidtionalTr )
+ {
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+ TOK.add_multichar_symbol("@_UNKNOWN_SYMBOL_@");
+ TOK.add_multichar_symbol(".#.");
+
+ String leftMarker("@LM@");
+ String rightMarker("@RM@");
+ TOK.add_multichar_symbol(leftMarker);
+ TOK.add_multichar_symbol(rightMarker);
+
+ ImplementationType type = uncondidtionalTr.get_type();
+
+ HfstTransducer leftBracket(leftMarker, TOK, type);
+ HfstTransducer rightBracket(rightMarker, TOK, type);
+
+
+ // Identity (normal)
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ HfstTransducer identity (identityPair);
+ identity.repeat_star().optimize();
+
+
+ // Create Right Part: [ B:0 | 0:B | ?-B ]*
+ HfstTransducer rightPart(type);
+ rightPart = constraintsRightPart(type);
+
+
+ // epsilon
+ HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
+ // B
+ HfstTransducer B(leftBracket);
+ B.disjunct(rightBracket).optimize();
+ // (B:0)*
+
+
+ HfstTransducer bracketsToEpsilonStar(B);
+ bracketsToEpsilonStar.cross_product(epsilon).optimize().repeat_star().optimize();
+
+ // (I-B) and (I-B)+
+ HfstTransducer identityPairMinusBrackets(identityPair);
+ identityPairMinusBrackets.subtract(B).optimize();
+
+ HfstTransducer identityPairMinusBracketsPlus(identityPairMinusBrackets);
+ identityPairMinusBracketsPlus.repeat_plus().optimize();
+
+ /*
+ HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
+ HfstTransducer identityPairMinusBracketsOrEpsilon(identityPairMinusBrackets);
+ identityPairMinusBracketsOrEpsilon.disjunct(epsilon).optimize();
+ */
+ HfstTransducer LeftBracketToEpsilon(leftMarker, "@_EPSILON_SYMBOL_@", TOK, type);
+
+
+ HfstTransducer boundary(".#.", TOK, type);
+
+ HfstTransducer Constraint(boundary);
+ Constraint.concatenate(identity);
+ //HfstTransducer Constraint(identity);
+
+ // ?* <:0 [B:0]* [I-B] [ B:0 | 0:B | ?-B ]*
+ Constraint.concatenate(LeftBracketToEpsilon).
+ concatenate(bracketsToEpsilonStar).
+ concatenate(identityPairMinusBrackets).
+ concatenate(rightPart).
+ optimize();
+
+ Constraint.concatenate(boundary).optimize();
+
+ // printf("Constraint: \n");
+ // Constraint.write_in_att_format(stdout, 1);
+
+ //// Compose with unconditional replace transducer
+ // tmp = t.1 .o. Constr .o. t.1
+ // (t.1 - tmp.2) .o. t
+
+ HfstTransducer retval(type);
+ retval = constraintComposition(uncondidtionalTr, Constraint);
+
+ //printf("Constraint: \n");
+ //Constraint.write_in_att_format(stdout, 1);
+ return retval;
+
+ }
+
+ // [ B:0 | 0:B | ?-B ]* [I-B]+ >:0 [ ?-B ]*
+ HfstTransducer rightMostConstraint( const HfstTransducer &uncondidtionalTr )
+ {
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+ TOK.add_multichar_symbol("@_UNKNOWN_SYMBOL_@");
+
+ String leftMarker("@LM@");
+ String rightMarker("@RM@");
+ TOK.add_multichar_symbol(leftMarker);
+ TOK.add_multichar_symbol(rightMarker);
+
+ ImplementationType type = uncondidtionalTr.get_type();
+
+ HfstTransducer leftBracket(leftMarker, TOK, type);
+ HfstTransducer rightBracket(rightMarker, TOK, type);
+
+
+ // Identity (normal)
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ HfstTransducer identity (identityPair);
+ identity.repeat_star().optimize();
+
+
+ // Create Right Part: [ B:0 | 0:B | ?-B ]*
+ HfstTransducer rightPart(type);
+ rightPart = constraintsRightPart(type);
+
+
+ // epsilon
+ HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
+ // B
+ HfstTransducer B(leftBracket);
+ B.disjunct(rightBracket).optimize();
+ // (B:0)*
+ HfstTransducer bracketsToEpsilonStar(B);
+ bracketsToEpsilonStar.cross_product(epsilon).optimize().repeat_star().optimize();
+
+ // (I-B) and (I-B)+
+ HfstTransducer identityPairMinusBrackets(identityPair);
+ identityPairMinusBrackets.subtract(B).optimize();
+
+ HfstTransducer identityPairMinusBracketsPlus(identityPairMinusBrackets);
+ identityPairMinusBracketsPlus.repeat_plus().optimize();
+
+ HfstTransducer identityPairMinusBracketsStar(identityPairMinusBrackets);
+ identityPairMinusBracketsStar.repeat_star().optimize();
+
+
+ HfstTransducer RightBracketToEpsilon(rightMarker, "@_EPSILON_SYMBOL_@", TOK, type);
+
+ HfstTransducer Constraint(rightPart);
+ // [ B:0 | 0:B | ?-B ]* [I-B]+ >:0 [ ?-B ]*
+
+ Constraint.concatenate(identityPairMinusBracketsPlus).
+ concatenate(RightBracketToEpsilon).
+ concatenate(identity).
+ optimize();
+
+ //// Compose with unconditional replace transducer
+ // tmp = t.1 .o. Constr .o. t.1
+ // (t.1 - tmp.2) .o. t
+
+ HfstTransducer retval(type);
+ retval = constraintComposition(uncondidtionalTr, Constraint);
+
+
+ //printf("Constraint: \n");
+ //Constraint.write_in_att_format(stdout, 1);
+ return retval;
+
+ }
+
+
+ // Longest match
+ // it should be composed to left most transducer........
+ // ?* < [?-B]+ 0:> [ ? | 0:< | <:0 | 0:> | B ] [ B:0 | 0:B | ?-B ]*
+ HfstTransducer longestMatchLeftMostConstraint( const HfstTransducer &uncondidtionalTr )
+ {
+
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+
+ String leftMarker("@LM@");
+ String rightMarker("@RM@");
+ TOK.add_multichar_symbol(leftMarker);
+ TOK.add_multichar_symbol(rightMarker);
+
+ ImplementationType type = uncondidtionalTr.get_type();
+
+ HfstTransducer leftBracket(leftMarker, TOK, type);
+ HfstTransducer rightBracket(rightMarker, TOK, type);
+
+ // Identity
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ HfstTransducer identity(identityPair);
+ identity.repeat_star().optimize();
+
+ // epsilon
+ HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
+ // B
+ HfstTransducer B(leftBracket);
+ B.disjunct(rightBracket).optimize();
+ // (B:0)*
+ HfstTransducer bracketsToEpsilonStar(B);
+ bracketsToEpsilonStar.cross_product(epsilon).optimize().repeat_star().optimize();
+
+ // (I-B) and (I-B)+
+ HfstTransducer identityPairMinusBrackets(identityPair);
+ identityPairMinusBrackets.subtract(B).optimize();
+
+ HfstTransducer identityPairMinusBracketsPlus(identityPairMinusBrackets);
+ identityPairMinusBracketsPlus.repeat_plus().optimize();
+
+
+
+ // Create Right Part: [ B:0 | 0:B | ?-B ]*
+ HfstTransducer rightPart(type);
+ rightPart = constraintsRightPart(type);
+
+ HfstTransducer RightBracketToEpsilon(rightMarker, "@_EPSILON_SYMBOL_@", TOK, type);
+ HfstTransducer epsilonToRightBracket("@_EPSILON_SYMBOL_@", rightMarker, TOK, type);
+ HfstTransducer LeftBracketToEpsilon(leftMarker, "@_EPSILON_SYMBOL_@", TOK, type);
+ HfstTransducer epsilonToLeftBracket("@_EPSILON_SYMBOL_@", leftMarker, TOK, type);
+
+ //[ ? | 0:< | <:0 | 0:> | B ]
+ // HfstTransducer nonClosingBracketInsertion(identityPair);
+ HfstTransducer nonClosingBracketInsertion(epsilonToLeftBracket);
+ nonClosingBracketInsertion.
+ //disjunct(epsilonToLeftBracket).
+ disjunct(LeftBracketToEpsilon).
+ disjunct(epsilonToRightBracket).
+ disjunct(B).
+ optimize();
+ // printf("nonClosingBracketInsertion: \n");
+ // nonClosingBracketInsertion.write_in_att_format(stdout, 1);
+
+
+ nonClosingBracketInsertion.concatenate(identityPairMinusBracketsPlus).optimize();
+
+ HfstTransducer middlePart(identityPairMinusBrackets);
+ middlePart.disjunct(nonClosingBracketInsertion).optimize();
+
+
+ // ?* < [?-B]+ 0:> [ ? | 0:< | <:0 | 0:> | B ] [?-B]+ [ B:0 | 0:B | ?-B ]*
+ HfstTransducer Constraint(identity);
+ Constraint.concatenate(leftBracket).
+ concatenate(identityPairMinusBracketsPlus).
+ concatenate(epsilonToRightBracket).
+ // concatenate(nonClosingBracketInsertion).
+ // concatenate(identityPairMinusBracketsPlus).
+ concatenate(middlePart).
+ concatenate(rightPart).
+ optimize();
+ //printf("Constraint Longest Match: \n");
+ //Constraint.write_in_att_format(stdout, 1);
+
+
+ //uncondidtionalTr should be left most for the left most longest match
+ HfstTransducer retval(type);
+ retval = constraintComposition(uncondidtionalTr, Constraint);
+
+
+ return retval;
+
+ }
+
+ // Longest match RIGHT most
+ HfstTransducer longestMatchRightMostConstraint(const HfstTransducer &uncondidtionalTr )
+ {
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+
+ String leftMarker("@LM@");
+ String rightMarker("@RM@");
+ TOK.add_multichar_symbol(leftMarker);
+ TOK.add_multichar_symbol(rightMarker);
+
+ ImplementationType type = uncondidtionalTr.get_type();
+
+ HfstTransducer leftBracket(leftMarker, TOK, type);
+ HfstTransducer rightBracket(rightMarker, TOK, type);
+
+ // Identity
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ HfstTransducer identity(identityPair);
+ identity.repeat_star().optimize();
+
+ // epsilon
+ HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
+ // B
+ HfstTransducer B(leftBracket);
+ B.disjunct(rightBracket).optimize();
+ // (B:0)*
+ HfstTransducer bracketsToEpsilonStar(B);
+ bracketsToEpsilonStar.cross_product(epsilon).optimize().repeat_star().optimize();
+
+ // (I-B) and (I-B)+
+ HfstTransducer identityPairMinusBrackets(identityPair);
+ identityPairMinusBrackets.subtract(B).optimize();
+
+ HfstTransducer identityPairMinusBracketsPlus(identityPairMinusBrackets);
+ identityPairMinusBracketsPlus.repeat_plus().optimize();
+
+
+
+ // Create Right Part: [ B:0 | 0:B | ?-B ]*
+ HfstTransducer rightPart(type);
+ rightPart = constraintsRightPart(type);
+
+
+
+ HfstTransducer RightBracketToEpsilon(rightMarker, "@_EPSILON_SYMBOL_@", TOK, type);
+
+ HfstTransducer epsilonToRightBracket("@_EPSILON_SYMBOL_@", rightMarker, TOK, type);
+ HfstTransducer LeftBracketToEpsilon(leftMarker, "@_EPSILON_SYMBOL_@", TOK, type);
+ HfstTransducer epsilonToLeftBracket("@_EPSILON_SYMBOL_@", leftMarker, TOK, type);
+
+
+ //[ ? | 0:< | >:0 | 0:> | B ]
+ HfstTransducer nonClosingBracketInsertion(identityPair);
+ nonClosingBracketInsertion.disjunct(epsilonToLeftBracket).
+ disjunct(RightBracketToEpsilon).
+ disjunct(epsilonToRightBracket).
+ disjunct(B).
+ optimize();
+
+
+ // [ B:0 | 0:B | ?-B ]* [?-B]+ [ ? | 0:< | <:0 | 0:> | B ] 0:< [?-B]+ > ?*
+
+ HfstTransducer Constraint(rightPart);
+ Constraint.concatenate(identityPairMinusBracketsPlus).
+ concatenate(nonClosingBracketInsertion).optimize().
+ concatenate(epsilonToLeftBracket).
+ concatenate(identityPairMinusBracketsPlus).
+ concatenate(rightBracket).
+ concatenate(identity).
+ optimize();
+ //printf("Constraint Longest Match: \n");
+ //Constraint.write_in_att_format(stdout, 1);
+
+
+ //uncondidtionalTr should be left most for the left most longest match
+ HfstTransducer retval(type);
+ retval = constraintComposition(uncondidtionalTr, Constraint);
+
+
+ return retval;
+ }
+
+ // Shortest match
+ // it should be composed to left most transducer........
+ // ?* < [?-B]+ >:0
+ // [?-B] or [ ? | 0:< | <:0 | >:0 | B ][?-B]+
+ // [ B:0 | 0:B | ?-B ]*
+ HfstTransducer shortestMatchLeftMostConstraint( const HfstTransducer &uncondidtionalTr )
+ {
+
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+
+ String leftMarker("@LM@");
+ String rightMarker("@RM@");
+ TOK.add_multichar_symbol(leftMarker);
+ TOK.add_multichar_symbol(rightMarker);
+
+ ImplementationType type = uncondidtionalTr.get_type();
+
+ HfstTransducer leftBracket(leftMarker, TOK, type);
+ HfstTransducer rightBracket(rightMarker, TOK, type);
+
+ // Identity
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ HfstTransducer identity(identityPair);
+ identity.repeat_star().optimize();
+
+
+ // Create Right Part: [ B:0 | 0:B | ?-B ]*
+ HfstTransducer rightPart(type);
+ rightPart = constraintsRightPart(type);
+
+ // [?-B] and [?-B]+
+ HfstTransducer B(leftBracket);
+ B.disjunct(rightBracket).optimize();
+ HfstTransducer identityPairMinusBrackets(identityPair);
+ identityPairMinusBrackets.subtract(B).optimize();
+ HfstTransducer identityPairMinusBracketsPlus(identityPairMinusBrackets);
+ identityPairMinusBracketsPlus.repeat_plus().optimize();
+
+
+ HfstTransducer RightBracketToEpsilon(rightMarker, "@_EPSILON_SYMBOL_@", TOK, type);
+ HfstTransducer epsilonToRightBracket("@_EPSILON_SYMBOL_@", rightMarker, TOK, type);
+ HfstTransducer LeftBracketToEpsilon(leftMarker, "@_EPSILON_SYMBOL_@", TOK, type);
+ HfstTransducer epsilonToLeftBracket("@_EPSILON_SYMBOL_@", leftMarker, TOK, type);
+
+
+ // [ 0:< | <:0 | >:0 | B ][?-B]+
+ HfstTransducer nonClosingBracketInsertion(epsilonToLeftBracket);
+ nonClosingBracketInsertion.
+ //disjunct(epsilonToLeftBracket).
+ disjunct(LeftBracketToEpsilon).
+ disjunct(RightBracketToEpsilon).
+ disjunct(B).
+ optimize();
+
+ nonClosingBracketInsertion.concatenate(identityPairMinusBracketsPlus).optimize();
+
+ HfstTransducer middlePart(identityPairMinusBrackets);
+ middlePart.disjunct(nonClosingBracketInsertion).optimize();
+
+ // printf("nonClosingBracketInsertion: \n");
+ // nonClosingBracketInsertion.write_in_att_format(stdout, 1);
+
+ // ?* < [?-B]+ >:0
+ // [?-B] or [ ? | 0:< | <:0 | >:0 | B ][?-B]+
+ //[ B:0 | 0:B | ?-B ]*
+ HfstTransducer Constraint(identity);
+ Constraint.concatenate(leftBracket).
+ concatenate(identityPairMinusBracketsPlus).
+ concatenate(RightBracketToEpsilon).
+ concatenate(middlePart).optimize().
+ concatenate(rightPart).
+ optimize();
+
+ //printf("Constraint Shortest Match: \n");
+ //Constraint.write_in_att_format(stdout, 1);
+
+
+ //uncondidtionalTr should be left most for the left most shortest match
+ HfstTransducer retval(type);
+ retval = constraintComposition(uncondidtionalTr, Constraint);
+
+
+ return retval;
+
+ }
+
+ // Shortest match
+ // it should be composed to left most transducer........
+ //[ B:0 | 0:B | ?-B ]*
+ // [?-B] or [?-B]+ [ ? | 0:> | >:0 | <:0 | B ]
+ // <:0 [?-B]+ > ?*
+ HfstTransducer shortestMatchRightMostConstraint( const HfstTransducer &uncondidtionalTr )
+ {
+
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+
+ String leftMarker("@LM@");
+ String rightMarker("@RM@");
+ TOK.add_multichar_symbol(leftMarker);
+ TOK.add_multichar_symbol(rightMarker);
+
+ ImplementationType type = uncondidtionalTr.get_type();
+
+ HfstTransducer leftBracket(leftMarker, TOK, type);
+ HfstTransducer rightBracket(rightMarker, TOK, type);
+
+ // Identity
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ HfstTransducer identity(identityPair);
+ identity.repeat_star().optimize();
+
+
+ // Create Right Part: [ B:0 | 0:B | ?-B ]*
+ HfstTransducer rightPart(type);
+ rightPart = constraintsRightPart(type);
+
+ // [?-B] and [?-B]+
+ HfstTransducer B(leftBracket);
+ B.disjunct(rightBracket).optimize();
+ HfstTransducer identityPairMinusBrackets(identityPair);
+ identityPairMinusBrackets.subtract(B).optimize();
+ HfstTransducer identityPairMinusBracketsPlus(identityPairMinusBrackets);
+ identityPairMinusBracketsPlus.repeat_plus().optimize();
+
+
+
+ HfstTransducer RightBracketToEpsilon(rightMarker, "@_EPSILON_SYMBOL_@", TOK, type);
+ HfstTransducer epsilonToRightBracket("@_EPSILON_SYMBOL_@", rightMarker, TOK, type);
+ HfstTransducer LeftBracketToEpsilon(leftMarker, "@_EPSILON_SYMBOL_@", TOK, type);
+ HfstTransducer epsilonToLeftBracket("@_EPSILON_SYMBOL_@", leftMarker, TOK, type);
+
+
+ // [?-B]+ [ 0:> | >:0 | <:0 | B ]
+ HfstTransducer nonClosingBracketInsertionTmp(epsilonToRightBracket);
+ nonClosingBracketInsertionTmp.
+ disjunct(RightBracketToEpsilon).
+ disjunct(LeftBracketToEpsilon).
+ disjunct(B).optimize();
+ HfstTransducer nonClosingBracketInsertion(identityPairMinusBracketsPlus);
+ nonClosingBracketInsertion.concatenate(nonClosingBracketInsertionTmp).optimize();
+
+
+ HfstTransducer middlePart(identityPairMinusBrackets);
+ middlePart.disjunct(nonClosingBracketInsertion).optimize();
+
+
+ //[ B:0 | 0:B | ?-B ]*
+ // [?-B] or [?-B]+ [ ? | 0:> | >:0 | <:0 | B ]
+ // <:0 [?-B]+ > ?*
+
+ HfstTransducer Constraint(rightPart);
+ Constraint.concatenate(middlePart).
+ concatenate(LeftBracketToEpsilon).
+ concatenate(identityPairMinusBracketsPlus).
+ concatenate(rightBracket).
+ concatenate(identity).
+ optimize();
+
+ //printf("Constraint Shortest Match: \n");
+ //Constraint.write_in_att_format(stdout, 1);
+
+ //uncondidtionalTr should be left most for the left most longest match
+ HfstTransducer retval(type);
+ retval = constraintComposition(uncondidtionalTr, Constraint);
+
+ return retval;
+
+ }
+
+
+ // ?* [ BL:0 (?-B)+ BR:0 ?* ]+
+ HfstTransducer mostBracketsPlusConstraint( const HfstTransducer &uncondidtionalTr )
+ {
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+
+ String leftMarker("@LM@");
+ String rightMarker("@RM@");
+ String leftMarker2("@LM2@");
+ String rightMarker2("@RM2@");
+
+ TOK.add_multichar_symbol(leftMarker);
+ TOK.add_multichar_symbol(rightMarker);
+ TOK.add_multichar_symbol(leftMarker2);
+ TOK.add_multichar_symbol(rightMarker2);
+
+ ImplementationType type = uncondidtionalTr.get_type();
+
+ HfstTransducer leftBracket(leftMarker, TOK, type);
+ HfstTransducer rightBracket(rightMarker, TOK, type);
+ HfstTransducer leftBracket2(leftMarker2, TOK, type);
+ HfstTransducer rightBracket2(rightMarker2, TOK, type);
+
+ // Identity (normal)
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ HfstTransducer identity (identityPair);
+ identity.repeat_star().optimize();
+
+ HfstTransducer identityPlus (identityPair);
+ identityPlus.repeat_plus().optimize();
+
+ HfstTransducer identityStar (identityPair);
+ identityStar.repeat_star().optimize();
+
+ // epsilon
+ String epsilon("@_EPSILON_SYMBOL_@");
+
+ // BL:0 ( <1 : 0, <2 : 0)
+ HfstTransducer leftBracketToEpsilon(leftMarker, epsilon, TOK, type );
+ HfstTransducer leftBracket2ToEpsilon(leftMarker2, epsilon, TOK, type );
+ HfstTransducer allLeftBracketsToEpsilon(leftBracketToEpsilon);
+ allLeftBracketsToEpsilon.disjunct(leftBracket2ToEpsilon).optimize();
+
+
+ // printf("allLeftBracketsToEpsilon: \n");
+ // allLeftBracketsToEpsilon.write_in_att_format(stdout, 1);
+
+ // BR:0 ( >1 : 0, >2 : 0)
+ HfstTransducer rightBracketToEpsilon(rightMarker, epsilon, TOK, type );
+ HfstTransducer rightBracket2ToEpsilon(rightMarker2, epsilon, TOK, type );
+ HfstTransducer allRightBracketsToEpsilon(rightBracketToEpsilon);
+ allRightBracketsToEpsilon.disjunct(rightBracket2ToEpsilon).optimize();
+
+ // B (B1 | B2)
+ HfstTransducer B(leftBracket);
+ B.disjunct(rightBracket).optimize();
+ B.disjunct(leftBracket2).optimize();
+ B.disjunct(rightBracket2).optimize();
+
+
+
+ // (? - B)+
+ HfstTransducer identityPairMinusBracketsPlus(identityPair);
+ identityPairMinusBracketsPlus.subtract(B).optimize().repeat_plus().optimize();
+
+ // repeatingPart ( BL:0 (?-B)+ BR:0 ?* )+
+ HfstTransducer repeatingPart(allLeftBracketsToEpsilon);
+ repeatingPart.concatenate(identityPairMinusBracketsPlus).optimize();
+ repeatingPart.concatenate(allRightBracketsToEpsilon).optimize();
+ repeatingPart.concatenate(identityStar).optimize();
+ repeatingPart.repeat_plus().optimize();
+ //printf("middlePart: \n");
+ //middlePart.write_in_att_format(stdout, 1);
+
+ HfstTransducer Constraint(identityStar);
+ Constraint.concatenate(repeatingPart).optimize();
+ //printf("Constraint: \n");
+ //Constraint.write_in_att_format(stdout, 1);
+
+
+ //// Compose with unconditional replace transducer
+ // tmp = t.1 .o. Constr .o. t.1
+ // (t.1 - tmp.2) .o. t
+
+ HfstTransducer retval(type);
+ retval = constraintComposition(uncondidtionalTr, Constraint);
+
+ //printf("After composition: \n");
+ //retval.write_in_att_format(stdout, 1);
+
+ return retval;
+ }
+
+ // ?* [ BL:0 (?-B)* BR:0 ?* ]+
+ HfstTransducer mostBracketsStarConstraint( const HfstTransducer &uncondidtionalTr )
+ {
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+
+ String leftMarker("@LM@");
+ String rightMarker("@RM@");
+ String leftMarker2("@LM2@");
+ String rightMarker2("@RM2@");
+ TOK.add_multichar_symbol(leftMarker);
+ TOK.add_multichar_symbol(rightMarker);
+ TOK.add_multichar_symbol(leftMarker2);
+ TOK.add_multichar_symbol(rightMarker2);
+
+ ImplementationType type = uncondidtionalTr.get_type();
+
+ HfstTransducer leftBracket(leftMarker, TOK, type);
+ HfstTransducer rightBracket(rightMarker, TOK, type);
+
+ HfstTransducer leftBracket2(leftMarker2, TOK, type);
+ HfstTransducer rightBracket2(rightMarker2, TOK, type);
+
+
+ // Identity (normal)
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ HfstTransducer identity (identityPair);
+ identity.repeat_star().optimize();
+
+ HfstTransducer identityPlus (identityPair);
+ identityPlus.repeat_plus().optimize();
+
+
+ HfstTransducer identityStar (identityPair);
+ identityStar.repeat_star().optimize();
+
+
+ // epsilon
+ String epsilon("@_EPSILON_SYMBOL_@");
+
+ // BL:0 ( <1 : 0, <2 : 0)
+ HfstTransducer leftBracketToEpsilon(leftMarker, epsilon, TOK, type );
+ HfstTransducer leftBracket2ToEpsilon(leftMarker2, epsilon, TOK, type );
+ HfstTransducer allLeftBracketsToEpsilon(leftBracketToEpsilon);
+ allLeftBracketsToEpsilon.disjunct(leftBracket2ToEpsilon).optimize();
+
+ // printf("allLeftBracketsToEpsilon: \n");
+ // allLeftBracketsToEpsilon.write_in_att_format(stdout, 1);
+
+ // BR:0 ( >1 : 0, >2 : 0)
+ HfstTransducer rightBracketToEpsilon(rightMarker, epsilon, TOK, type );
+ HfstTransducer rightBracket2ToEpsilon(rightMarker2, epsilon, TOK, type );
+ HfstTransducer allRightBracketsToEpsilon(rightBracketToEpsilon);
+ allRightBracketsToEpsilon.disjunct(rightBracket2ToEpsilon).optimize();
+
+ // B (B1 | B2)
+ HfstTransducer B(leftBracket);
+ B.disjunct(rightBracket).optimize();
+ B.disjunct(leftBracket2).optimize();
+ B.disjunct(rightBracket2).optimize();
+
+ // (? - B)*
+ HfstTransducer identityPairMinusBracketsStar(identityPair);
+ identityPairMinusBracketsStar.subtract(B).optimize().repeat_star().optimize();
+
+ // repeatingPart [ BL:0 (?-B)* BR:0 ?* ]+
+ HfstTransducer repeatingPart(allLeftBracketsToEpsilon);
+ repeatingPart.concatenate(identityPairMinusBracketsStar).optimize();
+ repeatingPart.concatenate(allRightBracketsToEpsilon).optimize();
+ repeatingPart.concatenate(identityStar).optimize();
+ repeatingPart.repeat_plus().optimize();
+ //printf("middlePart: \n");
+ //repeatingPart.write_in_att_format(stdout, 1);
+
+ HfstTransducer Constraint(identityStar);
+ Constraint.concatenate(repeatingPart).optimize();
+ //printf("Constraint: \n");
+ //Constraint.write_in_att_format(stdout, 1);
+
+ //// Compose with unconditional replace transducer
+ // tmp = t.1 .o. Constr .o. t.1
+ // (t.1 - tmp.2) .o. t
+ HfstTransducer retval(type);
+ retval = constraintComposition(uncondidtionalTr, Constraint);
+
+ //printf("After composition: \n");
+ //retval.write_in_att_format(stdout, 1);
+ return retval;
+
+ }
+ // ?* B2 ?*
+ HfstTransducer removeB2Constraint( const HfstTransducer &t )
+ {
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+
+ String leftMarker2("@LM2@");
+ String rightMarker2("@RM2@");
+
+ TOK.add_multichar_symbol(leftMarker2);
+ TOK.add_multichar_symbol(rightMarker2);
+
+ ImplementationType type = t.get_type();
+
+ HfstTransducer leftBracket2(leftMarker2, TOK, type);
+ HfstTransducer rightBracket2(rightMarker2, TOK, type);
+
+ // Identity (normal)
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ HfstTransducer identity (identityPair);
+ identity.repeat_star().optimize();
+
+ HfstTransducer identityStar (identityPair);
+ identityStar.repeat_star().optimize();
+
+ // B (B2)
+ HfstTransducer B(leftBracket2);
+ B.disjunct(rightBracket2).optimize();
+
+
+ HfstTransducer Constraint(identityStar);
+ Constraint.concatenate(B).optimize();
+ Constraint.concatenate(identityStar).optimize();
+
+ //// Compose with unconditional replace transducer
+ // tmp = t.1 .o. Constr .o. t.1
+ // (t.1 - tmp.2) .o. t
+
+ HfstTransducer retval(type);
+ retval = constraintComposition(t, Constraint);
+
+ retval.remove_from_alphabet(leftMarker2);
+ retval.remove_from_alphabet(rightMarker2);
+
+ //printf("Remove B2 After composition: \n");
+ //retval.write_in_att_format(stdout, 1);
+
+ return retval;
+
+ }
+ // to avoid repetition in empty replace rule
+ HfstTransducer noRepetitionConstraint( const HfstTransducer &t )
+ {
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+
+ String leftMarker("@LM@");
+ String rightMarker("@RM@");
+ TOK.add_multichar_symbol(leftMarker);
+ TOK.add_multichar_symbol(rightMarker);
+
+ String leftMarker2("@LM2@");
+ String rightMarker2("@RM2@");
+
+ //if the transdcuer is optional, LM2 and RM2 are not there
+ bool optional = true;
+ StringSet transducerAlphabet = t.get_alphabet();
+ for (StringSet::const_iterator s = transducerAlphabet.begin();
+ s != transducerAlphabet.end();
+ ++s)
+ {
+ String alph = *s;
+ if ( alph == leftMarker2)
+ {
+ optional = false;
+ break;
+ }
+ }
+
+ TOK.add_multichar_symbol(leftMarker2);
+ TOK.add_multichar_symbol(rightMarker2);
+
+ ImplementationType type = t.get_type();
+
+
+ HfstTransducer leftBracket(leftMarker, TOK, type);
+ HfstTransducer rightBracket(rightMarker, TOK, type);
+
+ HfstTransducer leftBracket2(leftMarker2, TOK, type);
+ HfstTransducer rightBracket2(rightMarker2, TOK, type);
+
+
+ HfstTransducer leftBrackets(leftBracket);
+ if (!optional)
+ {
+ leftBrackets.disjunct(leftBracket2).optimize();
+ }
+
+ HfstTransducer rightBrackets(rightBracket);
+ if (!optional)
+ {
+ rightBrackets.disjunct(rightBracket2).optimize();
+ }
+ // Identity (normal)
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ /*
+ identityPair.insert_to_alphabet(leftMarker);
+ identityPair.insert_to_alphabet(rightMarker);
+ identityPair.insert_to_alphabet(leftMarker);
+ identityPair.insert_to_alphabet(rightMarker2);
+ */
+
+ HfstTransducer identityStar (identityPair);
+ identityStar.repeat_star().optimize();
+
+
+ HfstTransducer Constraint(identityStar);
+ Constraint.concatenate(leftBrackets).
+ concatenate(rightBrackets).
+ concatenate(leftBrackets).
+ concatenate(rightBrackets).
+ concatenate(identityStar).optimize();
+
+
+ //// Compose with unconditional replace transducer
+ // tmp = t.1 .o. Constr .o. t.1
+ // (t.1 - tmp.2) .o. t
+
+ //printf("...Constraint: \n");
+ //Constraint.write_in_att_format(stdout, 1);
+
+
+ HfstTransducer retval(type);
+ retval = constraintComposition(t, Constraint);
+
+ // retval = removeB2Constraint(retval);
+
+ return retval;
+
+ }
+
+
+
+ // to apply boundary marker (.#.)
+ /*
+ * [0:.#. | ? - .#.]*
+ * .o.
+ * tr., ie. a -> b || .#. _ ;
+ * .o.
+ * .#. (? - .#.)* .#.
+ * .o.
+ * [.#.:0 | ? - .#.]*
+ */
+ HfstTransducer applyBoundaryMark( const HfstTransducer &t )
+ {
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+ TOK.add_multichar_symbol("@_UNKNOWN_SYMBOL_@");
+ TOK.add_multichar_symbol("@TMP_UNKNOWN@");
+ ImplementationType type = t.get_type();
+
+ String boundaryMarker(".#.");
+ TOK.add_multichar_symbol(boundaryMarker);
+ HfstTransducer boundary(boundaryMarker, TOK, type);
+
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ identityPair.insert_to_alphabet(boundaryMarker);
+ // ? - .#.
+ HfstTransducer identityMinusBoundary(identityPair);
+ identityMinusBoundary.subtract(boundary).optimize();
+
+ // (? - .#.)*
+ HfstTransducer identityMinusBoundaryStar(identityMinusBoundary);
+ identityMinusBoundaryStar.repeat_star().optimize();
+
+ // .#. (? - .#.)* .#.
+ HfstTransducer boundaryAnythingBoundary(boundary);
+ boundaryAnythingBoundary.concatenate(identityMinusBoundaryStar)
+ .concatenate(boundary)
+ .optimize();
+
+ // [0:.#. | ? - .#.]*
+ HfstTransducer zeroToBoundary("@_EPSILON_SYMBOL_@", boundaryMarker, TOK, type);
+ HfstTransducer retval(zeroToBoundary);
+ retval.disjunct(identityMinusBoundary)
+ .optimize()
+ .repeat_star()
+ .optimize();
+
+ //printf("retval .o. t: \n");
+ //retval.write_in_att_format(stdout, 1);
+ // [.#.:0 | ? - .#.]*
+ HfstTransducer boundaryToZero(boundaryMarker, "@_EPSILON_SYMBOL_@", TOK, type);
+ HfstTransducer removeBoundary(boundaryToZero);
+ removeBoundary.disjunct(identityMinusBoundary)
+ .optimize()
+ .repeat_star()
+ .optimize();
+
+ // apply boundary to the transducer
+ // compose [0:.#. | ? - .#.]* .o. t
+ HfstTransducer tr(t);
+
+ //tr.insert_to_alphabet(boundaryMarker);
+ // substitutute unknowns with tmp symbol
+ // this is necessary because of first composition
+ tr.substitute("@_UNKNOWN_SYMBOL_@", "@TMP_UNKNOWN@");
+
+ //printf("----first: ----\n");
+ //tr.write_in_att_format(stdout, 1);
+
+ retval.compose(tr).optimize();
+
+
+// printf("first composition: \n");
+// retval.write_in_att_format(stdout, 1);
+
+ // compose with .#. (? - .#.)* .#.
+ retval.compose(boundaryAnythingBoundary)
+ .optimize();
+
+// printf("2. composition: \n");
+// retval.write_in_att_format(stdout, 1);
+
+ // compose with [.#.:0 | ? - .#.]*
+ retval.compose(removeBoundary).optimize();
+
+// printf("3. composition: \n");
+// retval.write_in_att_format(stdout, 1);
+
+ // bring back unknown symbols
+ retval.substitute("@TMP_UNKNOWN@", "@_UNKNOWN_SYMBOL_@");
+ retval.remove_from_alphabet("@TMP_UNKNOWN@");
+
+ // remove boundary from alphabet
+ retval.remove_from_alphabet(boundaryMarker);
+ return retval;
+ }
+
+
+ //---------------------------------
+ // INTERFACE HELPING FUNCTIONS
+ //---------------------------------
+
+ // used by hfst-regexp parser
+ // creates markup crossproduct and sets property of the first transducer in the mapping to "isMarkup" = "yes"
+ // the other transducer in the mapping is set to epsilon transducer
+ HfstTransducerPair create_mapping_for_mark_up_replace( const HfstTransducerPair &mappingPair,
+ const HfstTransducerPair &marks )
+ {
+ HfstTokenizer TOK;
+ String epsilon = "@_EPSILON_SYMBOL_@";
+ TOK.add_multichar_symbol(epsilon);
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+
+ ImplementationType type = mappingPair.first.get_type();
+
+ HfstTransducer leftMark(marks.first);
+ HfstTransducer rightMark(marks.second);
+
+ HfstTransducer epsilonToLeftMark("@_EPSILON_SYMBOL_@", TOK, type);
+ epsilonToLeftMark.cross_product(leftMark).optimize();
+
+ HfstTransducer epsilonToRightMark(epsilon, TOK, type);
+ epsilonToRightMark.cross_product(rightMark).optimize();
+
+ //Go through left part of every mapping pair
+ // and concatenate: epsilonToLeftMark.leftMapping.epsilonToRightMark
+ //then put it into right part of the new transducerPairVector
+ HfstTransducer mappingCrossProduct(epsilonToLeftMark);
+ mappingCrossProduct.concatenate(mappingPair.first).
+ concatenate(epsilonToRightMark).
+ optimize();
+
+ mappingCrossProduct.set_property("isMarkup", "yes");
+
+ HfstTransducer epsilonTr(epsilon, TOK, type);
+ HfstTransducerPair retval(mappingCrossProduct, epsilonTr);
+
+ return retval;
+ }
+
+
+
+
+ //---------------------------------
+ // REPLACE FUNCTIONS - INTERFACE
+ //---------------------------------
+
+
+ // replace up, left, right, down
+ HfstTransducer replace( const Rule &rule, bool optional)
+ {
+ HfstTransducer retval( bracketedReplace(rule, optional) );
+
+ //printf("---bracketed replace done---: \n");
+ //retval.optimize().write_in_att_format(stdout, 1);
+
+ // for epenthesis rules
+ // it can't have more than one epsilon repetition in a row
+
+ retval = noRepetitionConstraint( retval );
+
+ //printf("-----noRepetitionConstraint-----: \n");
+ //retval.write_in_att_format(stdout, 1);
+
+ // deals with boundary symbol, must be before mostBracketsStarConstraint
+ retval = applyBoundaryMark( retval );
+
+ //printf("----after applyBoundaryMark: ----\n");
+ //retval.write_in_att_format(stdout, 1);
+ if ( !optional )
+ {
+
+ //printf(" ---------- mostBracketsStarConstraint --------------\n");
+ // Epenthesis rules behave differently if used mostBracketsPlusConstraint
+ //retval = mostBracketsPlusConstraint(retval);
+ retval = mostBracketsStarConstraint(retval);
+ //printf("after non optional: \n");
+ //retval.write_in_att_format(stdout, 1);
+ }
+ retval = removeB2Constraint(retval);
+ retval = removeMarkers( retval );
+ //printf("after removeMarkers: \n");
+ //retval.write_in_att_format(stdout, 1);
+ return retval;
+ }
+
+ // for parallel rules
+ HfstTransducer replace( const std::vector<Rule> &ruleVector, bool optional)
+ {
+
+ // std::cerr << "replace"<< std::endl;
+
+ HfstTransducer retval;
+ // If there is only one rule in the vector, it is not parallel
+ if ( ruleVector.size() == 1 )
+ {
+ retval = bracketedReplace( ruleVector[0], optional) ;
+ }
+ else
+ {
+ retval = parallelBracketedReplace(ruleVector, optional);
+ }
+
+ //std::cerr << "after bracketed replace"<< std::endl;
+ // printf("- bracketed replace -\n");
+ // retval.write_in_att_format(stdout, 1);
+
+ // for epenthesis rules
+ // it can't have more than one epsilon repetition in a row
+ retval = noRepetitionConstraint( retval );
+
+
+ // printf("----after noRepetitionConstraint: ----\n");
+ // retval.write_in_att_format(stdout, 1);
+
+
+ // deals with boundary symbol
+ retval = applyBoundaryMark( retval );
+
+ //printf("----after applyBoundaryMark: ----\n");
+ //retval.write_in_att_format(stdout, 1);
+
+ if ( !optional )
+ {
+ // Epenthesis rules behave differently if used mostBracketsPlusConstraint
+ // retval = mostBracketsPlusConstraint(retval);
+ retval = mostBracketsStarConstraint(retval);
+ }
+
+ // printf("----after mostBracketsStarConstraint: ----\n");
+ // retval.write_in_att_format(stdout, 1);
+
+ retval = removeB2Constraint(retval);
+
+ //printf("----after removeB2Constraint: ----\n");
+ // retval.write_in_att_format(stdout, 1);
+
+ retval = removeMarkers( retval );
+
+ //printf("----after removeMarkers: ----\n");
+ //retval.write_in_att_format(stdout, 1);
+ return retval;
+
+ }
+ // replace left
+ HfstTransducer replace_left( const Rule &rule, bool optional)
+ {
+ HfstTransducerPairVector mappingPairVector = rule.get_mapping();
+ //HfstTransducer newMapping = rule.get_mapping();
+ //newMapping.invert().optimize();
+
+ HfstTransducerPairVector newMappingPairVector;
+ for ( unsigned int i = 0; i < mappingPairVector.size(); i++ )
+ {
+ // in every mapping pair invert first and second
+ //HfstTransducer newMapping = rule.get_mapping();
+ HfstTransducer first = mappingPairVector[i].first;
+ HfstTransducer second = mappingPairVector[i].second;
+ newMappingPairVector.push_back(HfstTransducerPair(second, first));
+
+ }
+
+ Rule newRule ( newMappingPairVector, rule.get_context(), rule.get_replType());
+ HfstTransducer retval (replace( newRule, optional));
+
+ retval.invert().optimize();
+ return retval;
+ }
+ // replace left parallel
+ HfstTransducer replace_left( const std::vector<Rule> &ruleVector, bool optional)
+ {
+ std::vector<Rule> leftRuleVector;
+
+ for ( unsigned int i = 0; i < ruleVector.size(); i++ )
+ {
+
+ HfstTransducerPairVector mappingPairVector = ruleVector[i].get_mapping();
+ //HfstTransducer newMapping = rule.get_mapping();
+ //newMapping.invert().optimize();
+
+ HfstTransducerPairVector newMappingPairVector;
+ for ( unsigned int j = 0; j < mappingPairVector.size(); j++ )
+ {
+ // in every mapping pair invert first and second
+ //HfstTransducer newMapping = rule.get_mapping();
+ HfstTransducer first = mappingPairVector[j].first;
+ HfstTransducer second = mappingPairVector[j].second;
+ newMappingPairVector.push_back(HfstTransducerPair(second, first));
+
+ }
+
+ Rule newRule(newMappingPairVector, ruleVector[i].get_context(), ruleVector[i].get_replType());
+
+ leftRuleVector.push_back(newRule);
+ }
+
+ HfstTransducer retval(replace(leftRuleVector, optional));
+ retval.invert().optimize();
+
+ return retval;
+ }
+
+ // left to right
+ HfstTransducer replace_leftmost_longest_match( const Rule &rule )
+ {
+
+ HfstTransducer uncondidtionalTr( bracketedReplace(rule, true) );
+ //uncondidtionalTr = bracketedReplace(rule, true);
+
+ //printf("LM uncondidtionalTr: \n");
+ //uncondidtionalTr.write_in_att_format(stdout, 1);
+
+ // for epenthesis rules
+ // it can't have more than one epsilon repetition in a row
+ // it should be before leftMostConstraint
+ uncondidtionalTr = noRepetitionConstraint( uncondidtionalTr );
+
+ HfstTransducer retval (leftMostConstraint(uncondidtionalTr));
+
+ //to remove empty strings
+ retval = oneBetterthanNoneConstraint(retval);
+
+
+ // printf("leftMostConstraint: \n");
+ // retval.write_in_att_format(stdout, 1);
+ retval = longestMatchLeftMostConstraint( retval );
+
+ //printf("longestMatchLeftMostConstraint: \n");
+ //retval.write_in_att_format(stdout, 1);
+
+
+ retval = removeB2Constraint(retval);
+ retval = removeMarkers( retval );
+
+ // deals with boundary symbol
+ retval = applyBoundaryMark( retval );
+
+ return retval;
+ }
+ // left to right
+ HfstTransducer replace_leftmost_longest_match( const std::vector<Rule> &ruleVector )
+ {
+ //printf("\n replace_leftmost_longest_match \n");
+
+ HfstTransducer uncondidtionalTr;
+ if ( ruleVector.size() == 1 )
+ {
+ uncondidtionalTr = bracketedReplace( ruleVector[0], true) ;
+ }
+ else
+ {
+ uncondidtionalTr = parallelBracketedReplace(ruleVector, true);
+ }
+
+ //printf("retval unconditional 1 \n");
+ // uncondidtionalTr.write_in_att_format(stdout, 1);
+
+ // for epenthesis rules
+ // it can't have more than one epsilon repetition in a row
+ // it should be before leftMostConstraint
+ uncondidtionalTr = noRepetitionConstraint( uncondidtionalTr );
+ //printf("uncondidtionalTr epenthesis \n");
+ //uncondidtionalTr.write_in_att_format(stdout, 1);
+
+ HfstTransducer retval (leftMostConstraint(uncondidtionalTr));
+
+ //to remove empty strings
+ retval = oneBetterthanNoneConstraint(retval);
+
+ retval = longestMatchLeftMostConstraint( retval );
+ //printf("retval longestMatchLeftMostConstraint \n");
+ //retval.write_in_att_format(stdout, 1);
+
+ // remove LM2, RM2
+ retval = removeB2Constraint(retval);
+
+ //printf("retval removeB2Constraint \n");
+ //retval.write_in_att_format(stdout, 1);
+
+ retval = removeMarkers( retval );
+
+// printf("LM removeMarkers: \n");
+// retval.write_in_att_format(stdout, 1);
+
+ // deals with boundary symbol
+ retval = applyBoundaryMark( retval );
+
+ // printf("LM applyBoundaryMark: \n");
+ // retval.write_in_att_format(stdout, 1);
+
+ return retval;
+ }
+
+
+ // right to left
+ HfstTransducer replace_rightmost_longest_match( const Rule &rule )
+ {
+
+ HfstTransducer uncondidtionalTr(bracketedReplace(rule, true));
+ //uncondidtionalTr = bracketedReplace(rule, true);
+
+ HfstTransducer retval (rightMostConstraint(uncondidtionalTr));
+ //retval = rightMostConstraint(uncondidtionalTr);
+
+ //printf("rightMostConstraint: \n");
+ //retval.write_in_att_format(stdout, 1);
+
+ retval = longestMatchRightMostConstraint( retval );
+
+ //printf("longestMatchLeftMostConstraint: \n");
+ //retval.write_in_att_format(stdout, 1);
+
+ // for epenthesis rules
+ // it can't have more than one epsilon repetition in a row
+ retval = noRepetitionConstraint( retval );
+ // remove LM2, RM2
+ retval = removeB2Constraint(retval);
+
+ retval = removeMarkers( retval );
+
+
+ // deals with boundary symbol
+ retval = applyBoundaryMark( retval );
+
+ return retval;
+ }
+
+
+ // right to left
+ HfstTransducer replace_rightmost_longest_match( const std::vector<Rule> &ruleVector )
+ {
+ HfstTransducer uncondidtionalTr;
+ if ( ruleVector.size() == 1 )
+ {
+ uncondidtionalTr = bracketedReplace( ruleVector[0], true) ;
+ }
+ else
+ {
+ uncondidtionalTr = parallelBracketedReplace(ruleVector, true);
+ }
+
+ HfstTransducer retval (rightMostConstraint(uncondidtionalTr));
+ //retval = rightMostConstraint(uncondidtionalTr);
+
+ //printf("rightMostConstraint: \n");
+ //retval.write_in_att_format(stdout, 1);
+
+ retval = longestMatchRightMostConstraint( retval );
+
+ //printf("longestMatchLeftMostConstraint: \n");
+ //retval.write_in_att_format(stdout, 1);
+
+ // for epenthesis rules
+ // it can't have more than one epsilon repetition in a row
+ retval = noRepetitionConstraint( retval );
+ // remove LM2, RM2
+ retval = removeB2Constraint(retval);
+
+ retval = removeMarkers( retval );
+
+ // deals with boundary symbol
+ retval = applyBoundaryMark( retval );
+
+ return retval;
+ }
+
+ HfstTransducer replace_leftmost_shortest_match( const Rule &rule)
+ {
+
+ HfstTransducer uncondidtionalTr(bracketedReplace(rule, true));
+ // uncondidtionalTr = bracketedReplace(rule, true);
+
+ // for epenthesis rules
+ // it can't have more than one epsilon repetition in a row
+ //has to be before leftMostConstraint
+ uncondidtionalTr = noRepetitionConstraint( uncondidtionalTr );
+
+ HfstTransducer retval (leftMostConstraint(uncondidtionalTr));
+ //to remove empty strings
+ retval = oneBetterthanNoneConstraint(retval);
+
+ retval = shortestMatchLeftMostConstraint( retval );
+
+ //printf("sh tr: \n");
+ //retval.write_in_att_format(stdout, 1);
+
+
+ // remove LM2, RM2
+ retval = removeB2Constraint(retval);
+
+ retval = removeMarkers( retval );
+
+ // deals with boundary symbol
+ retval = applyBoundaryMark( retval );
+
+ return retval;
+ }
+
+
+ HfstTransducer replace_leftmost_shortest_match(const std::vector<Rule> &ruleVector )
+ {
+ HfstTransducer uncondidtionalTr;
+ if ( ruleVector.size() == 1 )
+ {
+ uncondidtionalTr = bracketedReplace( ruleVector[0], true) ;
+ }
+ else
+ {
+ uncondidtionalTr = parallelBracketedReplace(ruleVector, true);
+ }
+
+ // for epenthesis rules
+ // it can't have more than one epsilon repetition in a row
+ uncondidtionalTr = noRepetitionConstraint( uncondidtionalTr );
+
+
+ HfstTransducer retval (leftMostConstraint(uncondidtionalTr));
+
+ //to remove empty strings
+ retval = oneBetterthanNoneConstraint(retval);
+
+ retval = shortestMatchLeftMostConstraint( retval );
+
+ //printf("sh tr: \n");
+ //retval.write_in_att_format(stdout, 1);
+
+ // remove LM2, RM2
+ retval = removeB2Constraint(retval);
+
+ retval = removeMarkers( retval );
+
+ // deals with boundary symbol
+ retval = applyBoundaryMark( retval );
+
+ return retval;
+ }
+
+ HfstTransducer replace_rightmost_shortest_match( const Rule &rule )
+ {
+
+ HfstTransducer uncondidtionalTr(bracketedReplace( rule, true));
+ //uncondidtionalTr = bracketedReplace( rule, true);
+
+ HfstTransducer retval (rightMostConstraint(uncondidtionalTr));
+ //retval = rightMostConstraint(uncondidtionalTr);
+ retval = shortestMatchRightMostConstraint( retval );
+
+ //printf("sh tr: \n");
+ //retval.write_in_att_format(stdout, 1);
+
+
+ // for epenthesis rules
+ // it can't have more than one epsilon repetition in a row
+ retval = noRepetitionConstraint( retval );
+ // remove LM2, RM2
+ retval = removeB2Constraint(retval);
+
+ retval = removeMarkers( retval );
+
+ // deals with boundary symbol
+ retval = applyBoundaryMark( retval );
+
+ return retval;
+ }
+
+ HfstTransducer replace_rightmost_shortest_match( const std::vector<Rule> &ruleVector )
+ {
+ HfstTransducer uncondidtionalTr;
+ if ( ruleVector.size() == 1 )
+ {
+ uncondidtionalTr = bracketedReplace( ruleVector[0], true) ;
+ }
+ else
+ {
+ uncondidtionalTr = parallelBracketedReplace(ruleVector, true);
+ }
+ HfstTransducer retval (rightMostConstraint(uncondidtionalTr));
+ //retval = rightMostConstraint(uncondidtionalTr);
+ retval = shortestMatchRightMostConstraint( retval );
+
+ //printf("sh tr: \n");
+ //retval.write_in_att_format(stdout, 1);
+
+
+ // for epenthesis rules
+ // it can't have more than one epsilon repetition in a row
+ retval = noRepetitionConstraint( retval );
+ // remove LM2, RM2
+ retval = removeB2Constraint(retval);
+
+ retval = removeMarkers( retval );
+
+ // deals with boundary symbol
+ retval = applyBoundaryMark( retval );
+
+ return retval;
+ }
+
+
+
+ // replace up, left, right, down
+ HfstTransducer replace_epenthesis( const Rule &rule, bool optional)
+ {
+ return replace(rule, optional);
+ }
+
+ // replace up, left, right, down
+ HfstTransducer replace_epenthesis( const std::vector<Rule> &ruleVector, bool optional)
+ {
+ return replace(ruleVector, optional);
+ }
+
+
+ //---------------------------------
+ // RESTRICTION FUNCTIONS
+ //---------------------------------
+
+
+ /*
+ define U [ ? - %<D%> ] ;
+
+ define CENTER [ x y | x x y y ];
+
+ define L1 [ a ] ;
+ define R1 [ b ] ;
+
+ define L2 [ x ] ;
+ define R2 [ y ] ;
+
+ define RES1 [ U* L1 %<D%> U* %<D%> R1 U* ] ;
+ define RES2 [ U* L2 %<D%> U* %<D%> R2 U* ] ;
+
+ define CEN1 [ U* %<D%> CENTER %<D%> U* ] ;
+
+ define NODU [ U | 0:%<D%> ]* ;
+ define NODL [ U | %<D%>:0 ]* ;
+
+ regex U* - [ NODU .o. [ CEN1 - [ RES1 | RES2 ] ] .o. NODL ] ;
+ */
+ HfstTransducer restriction( const HfstTransducer &_center, const HfstTransducerPairVector &context)
+ {
+ //check if the center is automata
+ HfstTransducer c_proj1(_center);
+ c_proj1.input_project();
+ HfstTransducer c_proj2(_center);
+ c_proj2.output_project();
+
+ if ( ! c_proj1.compare(_center) || ! c_proj2.compare(_center) )
+ {
+ HFST_THROW_MESSAGE(TransducersAreNotAutomataException, "HfstXeroxRules::restriction");
+ }
+
+ ImplementationType type = _center.get_type();
+ String restrictionMark("@_D_@");
+
+ HfstTokenizer TOK;
+ TOK.add_multichar_symbol(restrictionMark);
+ TOK.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+
+ HfstTransducer mark(restrictionMark, TOK, type);
+ HfstTransducer epsilon("@_EPSILON_SYMBOL_@", TOK, type);
+
+ // Identity
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ HfstTransducer identity (identityPair);
+ identity.repeat_star().optimize();
+
+ HfstTransducer universalWithoutD(identity);
+ universalWithoutD.insert_to_alphabet(restrictionMark);
+ HfstTransducer universalWithoutDStar(universalWithoutD);
+ universalWithoutDStar.repeat_star().optimize();
+
+ // NODU
+ HfstTransducer noDUpper("@_EPSILON_SYMBOL_@", restrictionMark, TOK, type );
+ noDUpper.disjunct(universalWithoutD).repeat_star().optimize();
+
+ // NODL
+ HfstTransducer noDLower(restrictionMark, "@_EPSILON_SYMBOL_@", TOK, type );
+ noDLower.disjunct(universalWithoutD).repeat_star().optimize();
+
+ // 1. Surround center with marks
+ // [ U* %<D%> CENTER %<D%> U* ]
+ HfstTransducer center(_center);
+ center.insert_to_alphabet(restrictionMark);
+
+ HfstTransducer centerMarked(universalWithoutDStar);
+ centerMarked.concatenate(mark)
+ .concatenate(center)
+ .concatenate(mark)
+ .concatenate(universalWithoutDStar)
+ .optimize();
+
+ // 2. Put mark in context
+ // [ U* L1 %<D%> U* %<D%> R1 U* ]
+ HfstTransducer contextMarked;
+ for ( unsigned int i = 0; i < context.size(); i++ )
+ {
+ HfstTransducer lefContext(context[i].first);
+ lefContext.insert_to_alphabet(restrictionMark);
+
+ HfstTransducer rightContext(context[i].second);
+ rightContext.insert_to_alphabet(restrictionMark);
+
+ HfstTransducer RES(universalWithoutDStar);
+ RES.concatenate(lefContext)
+ .concatenate(mark)
+ .concatenate(universalWithoutDStar)
+ .concatenate(mark)
+ .concatenate(rightContext)
+ .concatenate(universalWithoutDStar)
+ .optimize();
+
+ if ( i == 0 )
+ {
+ contextMarked = RES;
+ }
+ else
+ {
+ contextMarked.disjunct(RES).optimize();
+ }
+ }
+ HfstTransducer centerMinusCtx(centerMarked);
+ centerMinusCtx.subtract(contextMarked).optimize();
+
+ HfstTransducer tmp(noDUpper);
+ tmp.compose(centerMinusCtx)
+ .compose(noDLower)
+ .optimize();
+
+ HfstTransducer retval(universalWithoutDStar);
+ retval.subtract(tmp).optimize();
+
+ retval.remove_from_alphabet(restrictionMark);
+
+ // deals with boundary symbol
+ retval = applyBoundaryMark( retval );
+
+ return retval;
+ }
+
+
+ // a < b
+ HfstTransducer before( const HfstTransducer &left, const HfstTransducer &right)
+ {
+ //check if the center is automata
+ HfstTransducer l_proj1(left);
+ l_proj1.input_project();
+ HfstTransducer l_proj2(left);
+ l_proj2.output_project();
+ HfstTransducer r_proj1(right);
+ r_proj1.input_project();
+ HfstTransducer r_proj2(right);
+ r_proj2.output_project();
+
+ if ( ! l_proj1.compare(left) || ! l_proj2.compare(left)
+ || ! r_proj1.compare(right) || ! r_proj2.compare(right) )
+ {
+ HFST_THROW_MESSAGE(TransducersAreNotAutomataException, "HfstXeroxRules::restriction");
+ }
+
+ ImplementationType type = left.get_type();
+
+ // Identity
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ HfstTransducer identity (identityPair);
+ identity.repeat_star().optimize();
+
+ HfstTransducer tmp(identity);
+ tmp.concatenate(right)
+ .concatenate(identity)
+ .concatenate(left)
+ .concatenate(identity)
+ .optimize();
+
+ HfstTransducer retval(identity);
+ retval.subtract(tmp).optimize();
+
+ return retval;
+ }
+ // a > b
+ HfstTransducer after( const HfstTransducer &left, const HfstTransducer &right)
+ {
+ //check if the center is automata
+ HfstTransducer l_proj1(left);
+ l_proj1.input_project();
+ HfstTransducer l_proj2(left);
+ l_proj2.output_project();
+ HfstTransducer r_proj1(right);
+ r_proj1.input_project();
+ HfstTransducer r_proj2(right);
+ r_proj2.output_project();
+
+ if ( ! l_proj1.compare(left) || ! l_proj2.compare(left)
+ || ! r_proj1.compare(right) || ! r_proj2.compare(right) )
+ {
+ HFST_THROW_MESSAGE(TransducersAreNotAutomataException, "HfstXeroxRules::restriction");
+ }
+
+ ImplementationType type = left.get_type();
+
+ // Identity
+ HfstTransducer identityPair = HfstTransducer::identity_pair( type );
+ HfstTransducer identity (identityPair);
+ identity.repeat_star().optimize();
+
+ HfstTransducer tmp(identity);
+ tmp.concatenate(left)
+ .concatenate(identity)
+ .concatenate(right)
+ .concatenate(identity)
+ .optimize();
+
+ HfstTransducer retval(identity);
+ retval.subtract(tmp).optimize();
+
+ return retval;
+ }
+
+
+
+
+
+ }
+}
+#else
+
+#include "HfstXeroxRulesTest.cc"
+
+int main(int argc, char * argv[])
+ {
+ std::cout << "Unit tests for " __FILE__ ":" << std::endl;
+
+
+ ImplementationType types[] = {SFST_TYPE, TROPICAL_OPENFST_TYPE, FOMA_TYPE};
+ unsigned int NUMBER_OF_TYPES=3;
+
+
+ for (unsigned int i=0; i < NUMBER_OF_TYPES; i++)
+ {
+ if (! HfstTransducer::is_implementation_type_available(types[i]))
+ continue;
+
+ // std::cout << "----- Type --------- " << i << std::endl;
+
+
+
+ // a -> b || ? - a _
+ //test8( types[i] );
+
+ test1( types[i] );
+
+ test1b( types[i]);
+
+ // ? -> a
+ test1c( types[i]);
+
+
+ // a -> b || .#. _ ;
+ test1d( types[i]);
+
+ // a+ -> x a_a
+ // also @-> and @>
+ test2a( types[i] );
+
+ // >@ ->@
+ test2b( types[i] );
+ test2c( types[i] );
+
+ // ? @-> a ... b;
+ // test2d( types[i] );
+
+ // testing unconditional replace with and without contexts
+
+ test3a( types[i] );
+ test3b( types[i] );
+ test3c( types[i] );
+ test3d( types[i] );
+
+ // left - right - down tests
+ // b -> a || _a (r: bbaa)
+ // b -> a \\ _a (r:aaaa)
+ // input: bbba
+ test4a( types[i] );
+ test4b( types[i] );
+ test4c( types[i] );
+
+ // mark up rule - doesn't have api interface anymore
+ // a | b -> %[...%] ;
+ // test5( types[i] );
+
+ // epenthesis rules
+ test6a( types[i] );
+ test6b( types[i] );
+ // [. 0 .] -> b || _ a a ;
+ test6c( types[i] );
+
+ //parralel rules
+ // a -> b , b -> c
+ test7a( types[i] );
+ // [. .] -> b , a -> c ;
+ test7b( types[i] );
+ // a @-> x , b @-> y
+ // a @> x , b @> y
+ // a -> x \/ m _ ,, b -> y || x _
+ test7c( types[i] );
+ // 0 .o. [ [. 0 .] -> a \/ _ b a , a b _ ,, [. 0 .] -> b \/ a _ a ]
+ test7d( types[i] );
+ // ? -> x , a -> b
+ test7e( types[i] );
+ // markup parallel rules
+ //test7e( types[i] );
+
+ // a -> b, b -> a
+ test7f( types[i] );
+
+ // a -> b b , a -> b
+ test7g( types[i] );
+ //[..] @-> a;
+ test7h( types[i] );
+
+ test9a( types[i] );
+ test9b( types[i] );
+
+ // rules with empty language
+ test10a( types[i] );
+ // empty -> non-empty
+ test10b( types[i] );
+
+
+ // flag diacritics
+ //a -> "@P.FOO.BAR@" || a _ "@u.a.b@"
+ //test11 (types[i] );
+
+
+ //markup rules
+ //test12 (types[i] );
+
+ // restriction functions =>
+
+ restriction_test1( types[i] );
+ restriction_test1a( types[i] );
+ restriction_test1b( types[i] );
+ restriction_test2( types[i] );
+ restriction_test3( types[i] );
+ restriction_test3a( types[i] );
+ restriction_test3b( types[i] );
+ restriction_test3c( types[i] );
+ restriction_test4( types[i] );
+ restriction_test5( types[i] );
+ restriction_test5a( types[i] );
+ restriction_test6( types[i] );
+ restriction_test7( types[i] );
+ restriction_test8( types[i] );
+
+ before_test1( types[i] );
+ }
+
+ std::cout << "ok" << std::endl;
+ return 0;
+ }
+#endif // MAIN_TEST
diff --git a/libhfst/src/HfstXeroxRules.h b/libhfst/src/HfstXeroxRules.h
index ebe2bbc..ecc1617 100644
--- a/libhfst/src/HfstXeroxRules.h
+++ b/libhfst/src/HfstXeroxRules.h
@@ -57,6 +57,8 @@ namespace hfst
//copy
Rule ( const Rule& );
+ // default constructor needed for SWIG...
+ Rule();
HfstTransducerPairVector get_mapping() const;
HfstTransducerPairVector get_context() const;
@@ -75,7 +77,7 @@ namespace hfst
HfstTransducer encodeFlagDiacritics( const HfstTransducer &tr );
/**
- * \brief In the transducer \tr, change back all "non-special" flag diacritics to normal,
+ * \brief In the transducer \tr, change back all "non-special" flag diacritics to normal,
* functional flag diacritics
* It means that $ sign will be changed to @ sign
* ie. $P.FOO.BAR$ will be changed into @P.FOO.BAR@
@@ -251,7 +253,7 @@ namespace hfst
// the problem is that the mark-up rules can be mixed with ordinary rules
- // ie a -> b ... c , a -> d ;
+ // ie a -> b ... c , a -> d ;
// this is why each markup mapping should be accessed seperratly
// HfstTransducer mark_up_replace( const Rule &rule,
// const StringPair &marks,
@@ -289,10 +291,10 @@ namespace hfst
// Contexts
// define NOS1 [ %[ -> 0 || b / B _ ];
// define NOF1 [ %] -> 0 || _ c / B ];
- static HfstTransducer removeBracketsInContext( const HfstTransducerPairVector &context,
+ /*static HfstTransducer removeBracketsInContext( const HfstTransducerPairVector &context,
const HfstTransducer &leftMark,
const HfstTransducer &rightMark,
- int i);
+ int i);*/ // NOT DEFINED
// Restriction function "=>"
HfstTransducer restriction( const HfstTransducer &automata, const HfstTransducerPairVector &context);
HfstTransducer before( const HfstTransducer &left, const HfstTransducer &right);
diff --git a/libhfst/src/HfstXeroxRulesTest.cc b/libhfst/src/HfstXeroxRulesTest.cc
index 43a0cfa..9ded7d6 100644
--- a/libhfst/src/HfstXeroxRulesTest.cc
+++ b/libhfst/src/HfstXeroxRulesTest.cc
@@ -1118,7 +1118,7 @@ void test9b( ImplementationType type )
Rule rule1(mappingPairVector1);
Rule rule2(mappingPairVector2);
- vector<Rule> ruleVector;
+ std::vector<Rule> ruleVector;
ruleVector.push_back(rule1);
ruleVector.push_back(rule2);
@@ -2607,7 +2607,7 @@ void test7a( ImplementationType type )
Rule rule1(mappingPairVector1);
Rule rule2(mappingPairVector2);
- vector<Rule> ruleVector;
+ std::vector<Rule> ruleVector;
ruleVector.push_back(rule1);
ruleVector.push_back(rule2);
@@ -2664,7 +2664,7 @@ void test7b( ImplementationType type )
Rule rule1(mappingPairVector1);
Rule rule2(mappingPairVector2);
- vector<Rule> ruleVector;
+ std::vector<Rule> ruleVector;
ruleVector.push_back(rule1);
ruleVector.push_back(rule2);
@@ -2733,7 +2733,7 @@ void test7c( ImplementationType type )
Rule rule1(mappingPairVector1);
Rule rule2(mappingPairVector2);
- vector<Rule> ruleVector;
+ std::vector<Rule> ruleVector;
ruleVector.push_back(rule1);
ruleVector.push_back(rule2);
@@ -2803,7 +2803,7 @@ void test7c( ImplementationType type )
Rule rule2aUp(mappingPairVector1, ContextVector1, REPL_UP);
Rule rule2bUp(mappingPairVector2, ContextVector2, REPL_UP);
- vector<Rule> ruleVector2;
+ std::vector<Rule> ruleVector2;
ruleVector2.push_back(rule2aUp);
ruleVector2.push_back(rule2bUp);
@@ -2828,7 +2828,7 @@ void test7c( ImplementationType type )
Rule rule2aDown(mappingPairVector1, ContextVector1, REPL_DOWN);
Rule rule2bDown(mappingPairVector2, ContextVector2, REPL_DOWN);
- vector<Rule> ruleVector3;
+ std::vector<Rule> ruleVector3;
ruleVector3.push_back(rule2aDown);
ruleVector3.push_back(rule2bDown);
@@ -2895,7 +2895,7 @@ void test7d( ImplementationType type )
Rule rule1(mappingPairVector1, ContextVector1, REPL_DOWN);
Rule rule2(mappingPairVector2, ContextVector2, REPL_DOWN);
- vector<Rule> ruleVector;
+ std::vector<Rule> ruleVector;
ruleVector.push_back(rule1);
ruleVector.push_back(rule2);
@@ -2949,7 +2949,7 @@ void test7e( ImplementationType type )
Rule rule1(mappingPairVector1);
Rule rule2(mappingPairVector2);
- vector<Rule> ruleVector;
+ std::vector<Rule> ruleVector;
ruleVector.push_back(rule1);
ruleVector.push_back(rule2);
@@ -3058,7 +3058,7 @@ void test7f( ImplementationType type )
Rule rule1(mappingPairVector1);
Rule rule2(mappingPairVector2);
- vector<Rule> ruleVector;
+ std::vector<Rule> ruleVector;
ruleVector.push_back(rule1);
ruleVector.push_back(rule2);
@@ -3113,7 +3113,7 @@ void test7g( ImplementationType type )
Rule rule1(mappingPairVector1);
Rule rule2(mappingPairVector2);
- vector<Rule> ruleVector;
+ std::vector<Rule> ruleVector;
ruleVector.push_back(rule1);
ruleVector.push_back(rule2);
diff --git a/libhfst/src/Makefile.am b/libhfst/src/Makefile.am
index ad77484..c20bc6d 100644
--- a/libhfst/src/Makefile.am
+++ b/libhfst/src/Makefile.am
@@ -1,13 +1,13 @@
## Process this file with automake to produce Makefile.in
-# Copyright (c) 2016 University of Helsinki
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 3 of the License, or (at your option) any later version.
-# See the file COPYING included with this distribution for more
-# information.
+# Copyright (c) 2016 University of Helsinki
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+# See the file COPYING included with this distribution for more
+# information.
SUBDIRS=implementations parsers
AUTOMAKE_OPTIONS=subdir-objects
@@ -32,8 +32,6 @@ libhfst_la_SOURCES = $(HFST_SRCS)
libhfst_la_LIBADD = implementations/libhfstimplementations.la \
parsers/libhfstparsers.la
-AM_CPPFLAGS += -I${top_srcdir}/back-ends/foma -I${top_srcdir}/back-ends
-
if WANT_SFST
libhfst_la_LIBADD += $(top_builddir)/back-ends/sfst/libsfst.la
endif
@@ -42,13 +40,10 @@ if WANT_FOMA
libhfst_la_LIBADD +=$(top_builddir)/back-ends/foma/libfoma.la
endif
-if WANT_MINGW
- AM_CPPFLAGS += -I${top_srcdir}/back-ends/openfstwin/src/include \
- -I${top_srcdir}/back-ends/dlfcn -DWINDOWS
+if WANT_MINGW
libhfst_la_LIBADD += $(top_builddir)/back-ends/dlfcn/libdl.la \
$(top_builddir)/back-ends/openfstwin/src/lib/libfst.la
else
- AM_CPPFLAGS += -I${top_srcdir}/back-ends/openfst/src/include
libhfst_la_LIBADD += $(top_builddir)/back-ends/openfst/src/lib/libfst.la
endif
@@ -76,7 +71,9 @@ HFST_HDRS = \
HfstTokenizer.h \
implementations/ConvertTransducerFormat.h \
implementations/HfstTransitionGraph.h \
+ implementations/HfstBasicTransducer.h \
implementations/HfstTransition.h \
+ implementations/HfstBasicTransition.h \
implementations/HfstTropicalTransducerTransitionData.h \
implementations/compose_intersect/ComposeIntersectRulePair.h \
implementations/compose_intersect/ComposeIntersectLexicon.h \
@@ -103,7 +100,7 @@ HFST_HDRS = \
hfstinclude_HEADERS = $(HFST_HDRS)
-libhfst_la_LDFLAGS = -no-undefined -version-info 45:0:0
+libhfst_la_LDFLAGS = -no-undefined -version-info 48:0:0
LIBHFST_TSTS=HfstApply HfstInputStream HfstTransducer \
HfstOutputStream HfstXeroxRules HfstRules HfstSymbolDefs \
diff --git a/libhfst/src/hfst-string-conversions.cc b/libhfst/src/hfst-string-conversions.cc
index 604fb68..2aba76e 100644
--- a/libhfst/src/hfst-string-conversions.cc
+++ b/libhfst/src/hfst-string-conversions.cc
@@ -1,14 +1,16 @@
#ifdef WINDOWS
-
+#pragma warning(disable: 4161)
+#pragma warning(disable: 4103)
#include <windows.h>
#include <string>
#include <cstdarg>
#include <cstdio>
+#include "HfstDataTypes.h"
#include <iostream> // for debugging
-namespace hfst
+namespace hfst
{
int hfst_fprintf_console(FILE * stream, const char * format, ...)
{
@@ -32,7 +34,7 @@ namespace hfst
MultiByteToWideChar(CP_UTF8 , 0 , pstr.c_str() , -1, NULL , 0 );
wchar_t* wstr = new wchar_t[wchars_num];
MultiByteToWideChar(CP_UTF8 , 0 ,
- pstr.c_str() , -1, wstr , wchars_num );
+ pstr.c_str() , -1, wstr , wchars_num );
int retval = WriteConsoleW(stdHandle, wstr, wchars_num-1, &numWritten, NULL);
delete[] wstr;
@@ -48,12 +50,12 @@ namespace hfst
bool get_line_from_console(std::string & str, size_t buffer_size, bool keep_newline /* = false*/)
{
- bool DEBUG = false;
+ bool DEBUG = false;
SetConsoleCP(65001);
const HANDLE stdIn = GetStdHandle(STD_INPUT_HANDLE);
WCHAR * buffer = new WCHAR [buffer_size];
DWORD numRead = 0;
- if (ReadConsoleW(stdIn, buffer, size_t(buffer_size/4), &numRead, NULL))
+ if (ReadConsoleW(stdIn, buffer, hfst::size_t_to_uint(buffer_size/4), &numRead, NULL))
{
if (DEBUG) { std::cerr << "get_line_from_console: numRead is " << numRead << std::endl; }
diff --git a/libhfst/src/hfst-string-conversions.h b/libhfst/src/hfst-string-conversions.h
index 0c22571..0c1c9bf 100644
--- a/libhfst/src/hfst-string-conversions.h
+++ b/libhfst/src/hfst-string-conversions.h
@@ -3,12 +3,12 @@
#include <string>
#include <cstdio>
-namespace hfst
+namespace hfst
{
/* Convert utf-8 string \a wstr into an ordinary string. */
//std::string wide_string_to_string(const std::wstring & wstr);
/* Get a line from console input and store it into \a str.
- \a buffer_size defines maximum of input length.
+ \a buffer_size defines maximum of input length.
Return whether the read operation was succesful. */
bool get_line_from_console(std::string & str, size_t buffer_size, bool keep_newline = false);
diff --git a/libhfst/src/hfst.h b/libhfst/src/hfst.h
index c7f12a3..2ba7eab 100644
--- a/libhfst/src/hfst.h
+++ b/libhfst/src/hfst.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
/*
diff --git a/libhfst/src/hfst_apply_schemas.h b/libhfst/src/hfst_apply_schemas.h
index 1325539..004990c 100644
--- a/libhfst/src/hfst_apply_schemas.h
+++ b/libhfst/src/hfst_apply_schemas.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
/** \brief declarations for HFST functions that take two or more parameters */
@@ -30,10 +30,10 @@ HfstTransducer &apply(
//#if HAVE_MY_TRANSDUCER_LIBRARY
//my_namespace::MyFst * (*my_transducer_library_funct)(my_namespace::MyFst *),
//#endif
- bool dummy /* makes sure there is always a parameter after the
- function pointer parameters, so commas between parameters
+ bool dummy /* makes sure there is always a parameter after the
+ function pointer parameters, so commas between parameters
are easier to handle */
-);
+);
HfstTransducer &apply(
#if HAVE_SFST
@@ -64,7 +64,7 @@ HfstTransducer &apply(
SFST::Transducer * (*sfst_funct)(SFST::Transducer *, String, String),
#endif
#if HAVE_OPENFST
- fst::StdVectorFst * (*tropical_ofst_funct)(fst::StdVectorFst *,String,
+ fst::StdVectorFst * (*tropical_ofst_funct)(fst::StdVectorFst *,String,
String),
#if HAVE_OPENFST_LOG
hfst::implementations::LogFst * (*log_ofst_funct)
diff --git a/libhfst/src/hfstdll.h b/libhfst/src/hfstdll.h
index a01713c..b0ef7a7 100644
--- a/libhfst/src/hfstdll.h
+++ b/libhfst/src/hfstdll.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _HFSTDLL_H_
diff --git a/libhfst/src/implementations/ConvertFomaTransducer.cc b/libhfst/src/implementations/ConvertFomaTransducer.cc
index 1d2ca9e..c5e74cc 100644
--- a/libhfst/src/implementations/ConvertFomaTransducer.cc
+++ b/libhfst/src/implementations/ConvertFomaTransducer.cc
@@ -1,19 +1,26 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#if HAVE_CONFIG_H
# include <config.h>
#endif
+#if HAVE_FOMA
+ #ifndef _FOMALIB_H_
+ #define _FOMALIB_H_
+ #include "back-ends/foma/fomalib.h"
+ #endif
+#endif
+
#include "ConvertTransducerFormat.h"
-#include "HfstTransitionGraph.h"
-#include "HfstTransducer.h"
+#include "HfstBasicTransducer.h"
+#include "FomaTransducer.h"
#ifndef MAIN_TEST
namespace hfst { namespace implementations
@@ -21,7 +28,7 @@ namespace hfst { namespace implementations
/* -----------------------------------------------------------
- Conversion functions between HfstBasicTransducer and foma transducer.
+ Conversion functions between HfstBasicTransducer and foma transducer.
---------------------------------------------------------- */
@@ -31,7 +38,7 @@ namespace hfst { namespace implementations
/* -----------------------------------------------------------------
- Internal functions used by the actual conversion functions.
+ Internal functions used by the actual conversion functions.
----------------------------------------------------------------- */
@@ -49,8 +56,8 @@ namespace hfst { namespace implementations
return number_of_transitions;
}
- /*
- Handle a start state in a foma transducer.
+ /*
+ Handle a start state in a foma transducer.
@param fsm The start state.
@param start_state_id The number of the start state, the value -1 means
@@ -59,17 +66,17 @@ namespace hfst { namespace implementations
at some point.
*/
static void handle_start_state
- (struct fsm_state * fsm,
+ (struct fsm_state * fsm_,
int &start_state_id,
bool &start_state_found)
{
// If the start state has not yet been encountered.
if (! start_state_found) {
- start_state_id = (fsm)->state_no; // define the start state
+ start_state_id = (fsm_)->state_no; // define the start state
start_state_found=true; // define that it is found
}
- // If the start state is encountered again,
- else if ((fsm)->state_no == start_state_id) {
+ // If the start state is encountered again,
+ else if ((fsm_)->state_no == start_state_id) {
// do nothing.
}
// If there are several initial states in foma transducer,
@@ -81,9 +88,9 @@ namespace hfst { namespace implementations
}
}
- /* Copy the alphabet of foma transducer \a t to HFST basic transducer
+ /* Copy the alphabet of foma transducer \a t to HFST basic transducer
\a net. */
- static void copy_alphabet(const struct fsm * t, HfstBasicTransducer * net)
+ static void copy_alphabet(const fsm * t, HfstBasicTransducer * net)
{
struct sigma * p = t->sigma;
while (p != NULL) {
@@ -99,9 +106,9 @@ namespace hfst { namespace implementations
static void copy_alphabet(const HfstBasicTransducer * hfst_fsm,
struct fsm_construct_handle * h)
{
- const HfstBasicTransducer::HfstTransitionGraphAlphabet & alpha
+ const HfstBasicTransducer::HfstAlphabet & alpha
= hfst_fsm->get_alphabet();
- for (HfstBasicTransducer::HfstTransitionGraphAlphabet::iterator it
+ for (HfstBasicTransducer::HfstAlphabet::iterator it
= alpha.begin();
it != alpha.end(); it++)
{
@@ -117,12 +124,12 @@ namespace hfst { namespace implementations
/* ----------------------------------------------------------------------
- Create an HfstBasicTransducer equivalent to foma transducer \a t.
+ Create an HfstBasicTransducer equivalent to foma transducer \a t.
---------------------------------------------------------------------- */
HfstBasicTransducer * ConversionFunctions::
- foma_to_hfst_basic_transducer(struct fsm * t) {
+ foma_to_hfst_basic_transducer(fsm * t) {
#ifdef DEBUG_CONVERSION
StringSet alphabet_before;
@@ -149,7 +156,7 @@ namespace hfst { namespace implementations
bool start_state_found=false;
// For every line in foma transducer:
- for (int i=0; (fsm+i)->state_no != -1; i++) {
+ for (int i=0; (fsm+i)->state_no != -1; i++) {
// Count the number of transitions in the current state
// and initialize the transition vector of the net accordingly
@@ -157,7 +164,7 @@ namespace hfst { namespace implementations
{
unsigned int number_of_transitions = get_number_of_transitions(fsm+i);
net->initialize_transition_vector
- ((fsm+i)->state_no,
+ ((fsm+i)->state_no,
number_of_transitions);
}
@@ -167,16 +174,16 @@ namespace hfst { namespace implementations
}
// 2. If there are transitions leaving from the state,
- if ((fsm+i)->target != -1)
+ if ((fsm+i)->target != -1)
{
// copy the transition.
net->add_transition
((fsm+i)->state_no,
HfstBasicTransition
((fsm+i)->target,
- harmonization_vector.at((fsm+i)->in),
- harmonization_vector.at((fsm+i)->out),
- 0, false), false);
+ harmonization_vector.at((fsm+i)->in),
+ harmonization_vector.at((fsm+i)->out),
+ 0, false), false);
}
// 3. If the source state is final in foma,
@@ -197,7 +204,7 @@ namespace hfst { namespace implementations
"Foma transducer has no start state");*/
}
- /* If start state number (N) is not zero, swap state numbers N and zero
+ /* If start state number (N) is not zero, swap state numbers N and zero
in internal transducer. TODO */
if (start_state_id != 0) {
net->swap_state_numbers(start_state_id,0);
@@ -214,8 +221,8 @@ namespace hfst { namespace implementations
{
if (alphabet_before.find(*after_it) == alphabet_before.end())
{
- std::cerr << "ERROR: "
- << *after_it
+ std::cerr << "ERROR: "
+ << *after_it
<< " was inserted to the alphabet!"
<< std::endl;
}
@@ -225,8 +232,8 @@ namespace hfst { namespace implementations
{
if (alphabet_after.find(*before_it) == alphabet_after.end())
{
- std::cerr << "ERROR: "
- << *before_it
+ std::cerr << "ERROR: "
+ << *before_it
<< " was removed from the alphabet!"
<< std::endl;
}
@@ -242,11 +249,11 @@ namespace hfst { namespace implementations
/* ------------------------------------------------------------------------
- Create a foma transducer equivalent to HfstBasicTransducer \a hfst_fsm.
+ Create a foma transducer equivalent to HfstBasicTransducer \a hfst_fsm.
------------------------------------------------------------------------ */
- struct fsm * ConversionFunctions::
+ fsm * ConversionFunctions::
hfst_basic_transducer_to_foma(const HfstBasicTransducer * hfst_fsm) {
#ifdef DEBUG_CONVERSION
@@ -254,10 +261,10 @@ namespace hfst { namespace implementations
alphabet_before.erase(internal_epsilon);
alphabet_before.erase(internal_unknown);
alphabet_before.erase(internal_identity);
-#endif // DEBUG_CONVERSION
+#endif // DEBUG_CONVERSION
struct fsm_construct_handle *h;
- struct fsm *net;
+ fsm *net;
const char * emptystr = "";
h = fsm_construct_init(const_cast<char*>(emptystr));
//free(emptystr);
@@ -268,15 +275,15 @@ namespace hfst { namespace implementations
it != hfst_fsm->end(); it++)
{
// ----- Go through the set of transitions in each state -----
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
+ for (HfstBasicTransitions::const_iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
// Copy the transition
const char * input = tr_it->get_transition_data().get_input_symbol().c_str();
const char * output = tr_it->get_transition_data().get_output_symbol().c_str();
- fsm_construct_add_arc(h,
- (int)source_state,
+ fsm_construct_add_arc(h,
+ (int)source_state,
(int)tr_it->get_target_state(),
const_cast<char *>(input),
const_cast<char *>(output) );
@@ -285,14 +292,14 @@ namespace hfst { namespace implementations
}
// ----- transitions gone through -----
source_state++;
- }
+ }
// ----- all states gone through -----
// ----- Go through the final states -----
- for (HfstBasicTransducer::FinalWeightMap::const_iterator it
+ for (HfstBasicTransducer::FinalWeightMap::const_iterator it
= hfst_fsm->final_weight_map.begin();
- it != hfst_fsm->final_weight_map.end(); it++)
+ it != hfst_fsm->final_weight_map.end(); it++)
{
// Set the state as final
fsm_construct_set_final(h, (int)it->first);
diff --git a/libhfst/src/implementations/ConvertLogWeightTransducer.cc b/libhfst/src/implementations/ConvertLogWeightTransducer.cc
index 1f70aad..a0a4432 100644
--- a/libhfst/src/implementations/ConvertLogWeightTransducer.cc
+++ b/libhfst/src/implementations/ConvertLogWeightTransducer.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#if HAVE_CONFIG_H
@@ -12,8 +12,25 @@
#endif
#include "ConvertTransducerFormat.h"
-#include "HfstTransitionGraph.h"
-#include "HfstTransducer.h"
+#include "HfstBasicTransducer.h"
+#include "LogWeightTransducer.h"
+
+#ifdef _MSC_VER
+#include "back-ends/openfstwin/src/include/fst/fstlib.h"
+#else
+#include "back-ends/openfst/src/include/fst/fstlib.h"
+#endif // _MSC_VER
+
+#ifndef _MSC_VER
+namespace fst
+{
+ extern template class LogWeightTpl<float>;
+ extern template class ArcTpl<LogWeight>;
+ extern template class VectorFst<LogArc>;
+ extern template class ArcIterator<LogFst>;
+ extern template class StateIterator<LogFst>;
+}
+#endif // _MSC_VER
#ifndef MAIN_TEST
namespace hfst { namespace implementations
@@ -25,7 +42,7 @@ namespace hfst { namespace implementations
/* --- Conversion between log OpenFst and HfstBasicTransducer --- */
/* Create an HfstBasicTransducer equivalent to an OpenFst log weight
- transducer \a t. */
+ transducer \a t. */
HfstBasicTransducer * ConversionFunctions::
log_ofst_to_hfst_basic_transducer
(LogFst * t, bool has_hfst_header) {
@@ -42,34 +59,34 @@ namespace hfst { namespace implementations
// An empty transducer
if (t->Start() == fst::kNoStateId)
- {
+ {
/* An empty OpenFst transducer does not necessarily have to have
an input or output symbol table. */
if (inputsym != NULL) {
- for ( fst::SymbolTableIterator it =
+ for ( fst::SymbolTableIterator it =
fst::SymbolTableIterator(*(inputsym));
- not it.Done(); it.Next() ) {
+ ! it.Done(); it.Next() ) {
if (it.Value() != 0) // epsilon is not inserted
net->alphabet.insert( it.Symbol() );
- }
+ }
}
/* If the transducer is an OpenFst transducer, it might have an output
symbol table. If the transducer is an HFST log transducer, it
- can have an output symbol table, but it is equivalent to the
+ can have an output symbol table, but it is equivalent to the
input symbol table. */
- if (not has_hfst_header && outputsym != NULL) {
- for ( fst::SymbolTableIterator it =
+ if (! has_hfst_header && outputsym != NULL) {
+ for ( fst::SymbolTableIterator it =
fst::SymbolTableIterator(*(outputsym));
- not it.Done(); it.Next() ) {
+ ! it.Done(); it.Next() ) {
if (it.Value() != 0) // epsilon is not inserted
net->alphabet.insert( it.Symbol() );
- }
+ }
}
return net;
- }
+ }
/* A non-empty OpenFst transducer must have at least an input symbol table.
- If the output symbol table is missing, we assume that it would be
+ If the output symbol table is missing, we assume that it would be
equivalent to the input symbol table. */
if (inputsym == NULL) {
HFST_THROW(MissingOpenFstInputSymbolTableException);
@@ -88,8 +105,8 @@ namespace hfst { namespace implementations
}
/* Go through all states */
- for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ for (fst::StateIterator<LogFst> siter(*t);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s == initial_state) {
@@ -102,7 +119,7 @@ namespace hfst { namespace implementations
origin = (int)s;
/* Go through all transitions in a state */
- for (fst::ArcIterator<LogFst> aiter(*t,s);
+ for (fst::ArcIterator<LogFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const fst::LogArc &arc = aiter.Value();
@@ -121,7 +138,7 @@ namespace hfst { namespace implementations
istring = std::string(internal_epsilon);
if (arc.olabel == 0)
ostring = std::string(internal_epsilon);
- net->add_transition(origin,
+ net->add_transition(origin,
HfstBasicTransition
(target,
istring,
@@ -137,8 +154,8 @@ namespace hfst { namespace implementations
}
}
- for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ for (fst::StateIterator<LogFst> siter(*t);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s != initial_state) {
@@ -149,7 +166,7 @@ namespace hfst { namespace implementations
origin = 0;
else
origin = (int)s;
- for (fst::ArcIterator<LogFst> aiter(*t,s);
+ for (fst::ArcIterator<LogFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const fst::LogArc &arc = aiter.Value();
@@ -167,7 +184,7 @@ namespace hfst { namespace implementations
istring = std::string(internal_epsilon);
if (arc.olabel == 0)
ostring = std::string(internal_epsilon);
- net->add_transition(origin,
+ net->add_transition(origin,
HfstBasicTransition
(target,
istring,
@@ -182,18 +199,18 @@ namespace hfst { namespace implementations
/* Make sure that also the symbols that occur only in the alphabet
but not in transitions are copied. */
- for ( fst::SymbolTableIterator it =
+ for ( fst::SymbolTableIterator it =
fst::SymbolTableIterator(*(inputsym));
- not it.Done(); it.Next() ) {
+ ! it.Done(); it.Next() ) {
if (it.Value() != 0) // epsilon is not inserted
net->alphabet.insert( it.Symbol() );
- }
- for ( fst::SymbolTableIterator it =
+ }
+ for ( fst::SymbolTableIterator it =
fst::SymbolTableIterator(*(outputsym));
- not it.Done(); it.Next() ) {
+ ! it.Done(); it.Next() ) {
if (it.Value() != 0) // epsilon is not inserted
net->alphabet.insert( it.Symbol() );
- }
+ }
return net;
}
@@ -201,10 +218,10 @@ namespace hfst { namespace implementations
/* Get a state id for a state in transducer \a t that corresponds
- to HfstState s as defined in \a state_map.
+ to HfstState s as defined in \a state_map.
Used by function hfst_basic_transducer_to_log_ofst. */
StateId ConversionFunctions::hfst_state_to_state_id
- (HfstState s, std::map<HfstState, StateId> &state_map,
+ (HfstState s, std::map<HfstState, StateId> &state_map,
LogFst * t)
{
std::map<HfstState, StateId>::iterator it = state_map.find(s);
@@ -242,12 +259,12 @@ namespace hfst { namespace implementations
it != net->end(); it++)
{
// Go through the set of transitions in each state
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
+ for (HfstBasicTransitions::const_iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
// Copy the transition
- t->AddArc( hfst_state_to_state_id(source_state, state_map, t),
+ t->AddArc( hfst_state_to_state_id(source_state, state_map, t),
fst::LogArc
( st.AddSymbol(tr_it->get_input_symbol()),
st.AddSymbol(tr_it->get_output_symbol()),
@@ -258,23 +275,23 @@ namespace hfst { namespace implementations
}
// Go through the final states
- for (HfstBasicTransducer::FinalWeightMap::const_iterator it
+ for (HfstBasicTransducer::FinalWeightMap::const_iterator it
= net->final_weight_map.begin();
- it != net->final_weight_map.end(); it++)
+ it != net->final_weight_map.end(); it++)
{
- t->SetFinal(hfst_state_to_state_id(it->first, state_map, t),
+ t->SetFinal(hfst_state_to_state_id(it->first, state_map, t),
it->second);
}
// Add also symbols that do not occur in transitions
- for (HfstBasicTransducer::HfstTransitionGraphAlphabet::iterator it
+ for (HfstBasicTransducer::HfstAlphabet::iterator it
= net->alphabet.begin();
it != net->alphabet.end(); it++) {
st.AddSymbol(*it);
}
t->SetInputSymbols(&st);
- return t;
+ return t;
}
#endif
diff --git a/libhfst/src/implementations/ConvertOlTransducer.cc b/libhfst/src/implementations/ConvertOlTransducer.cc
index 049b0e9..73ae0dd 100644
--- a/libhfst/src/implementations/ConvertOlTransducer.cc
+++ b/libhfst/src/implementations/ConvertOlTransducer.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#if HAVE_CONFIG_H
@@ -15,8 +15,8 @@
#include "ConvertTransducerFormat.h"
#include "optimized-lookup/convert.h"
-#include "HfstTransitionGraph.h"
-#include "HfstTransducer.h"
+#include "HfstBasicTransducer.h"
+//#include "HfstTransducer.h"
#ifndef MAIN_TEST
namespace hfst { namespace implementations
@@ -24,14 +24,14 @@ namespace hfst { namespace implementations
/* -----------------------------------------------------------
- Conversion functions between HfstBasicTransducer and
- optimized lookup transducers
+ Conversion functions between HfstBasicTransducer and
+ optimized lookup transducers
---------------------------------------------------------- */
/* An auxiliary function. */
unsigned int hfst_ol_to_hfst_basic_add_state
-(hfst_ol::Transducer * t,
+(hfst_ol::Transducer * t,
HfstBasicTransducer * basic,
hfst_ol::HfstOlToBasicStateMap & state_map,
bool weighted,
@@ -50,9 +50,10 @@ unsigned int hfst_ol_to_hfst_basic_add_state
basic->add_state(new_state);
basic->set_final_weight(new_state,
weighted ?
- dynamic_cast<const hfst_ol::TransitionWIndex&>
- (transition_index).final_weight() :
- 0.0);
+ hfst::double_to_float
+ (dynamic_cast<const hfst_ol::TransitionWIndex&>
+ (transition_index).final_weight()) :
+ (float)0.0);
}
}
else // indexes transition table
@@ -64,9 +65,10 @@ unsigned int hfst_ol_to_hfst_basic_add_state
basic->add_state(new_state);
basic->set_final_weight(new_state,
weighted ?
- dynamic_cast<const hfst_ol::TransitionW&>
- (transition).get_weight() :
- 0.0);
+ hfst::double_to_float
+ (dynamic_cast<const hfst_ol::TransitionW&>
+ (transition).get_weight()) :
+ (float)0.0);
}
}
return new_state;
@@ -79,15 +81,15 @@ unsigned int hfst_ol_to_hfst_basic_add_state
{
HfstBasicTransducer * basic = new HfstBasicTransducer();
bool weighted = t->get_header().probe_flag(hfst_ol::Weighted);
- const hfst_ol::SymbolTable& symbols
+ const hfst_ol::SymbolTable& symbols
= t->get_alphabet().get_symbol_table();
- /* This contains indices to either (1) the start of a set of entries
- in the transition index table, or (2) the boundary before a set
- of entries in the transition table; in this case, the following
- entries will all have the same input symbol. In either case
- the index represents a state and may be final The will already be
+ /* This contains indices to either (1) the start of a set of entries
+ in the transition index table, or (2) the boundary before a set
+ of entries in the transition table; in this case, the following
+ entries will all have the same input symbol. In either case
+ the index represents a state and may be final The will already be
an entry in state_map for each value in agenda */
std::vector<hfst_ol::TransitionTableIndex> agenda;
hfst_ol::HfstOlToBasicStateMap state_map;
@@ -103,7 +105,7 @@ unsigned int hfst_ol_to_hfst_basic_add_state
unsigned int current_state = state_map[current_index];
- hfst_ol::TransitionTableIndexSet transitions
+ hfst_ol::TransitionTableIndexSet transitions
= t->get_transitions_from_state(current_index);
for(hfst_ol::TransitionTableIndexSet::const_iterator it
=transitions.begin();it!=transitions.end();it++)
@@ -114,7 +116,7 @@ unsigned int hfst_ol_to_hfst_basic_add_state
{
state_number++;
hfst_ol_to_hfst_basic_add_state
- (t, basic, state_map, weighted,
+ (t, basic, state_map, weighted,
transition.get_target(), state_number);
agenda.push_back(transition.get_target());
}
@@ -170,11 +172,11 @@ void get_states_and_symbols(
unsigned int first_transition = 0;
unsigned int state_number = 0;
- for (HfstBasicTransducer::const_iterator it = t->begin();
+ for (HfstBasicTransducer::const_iterator it = t->begin();
it != t->end(); ++it) {
hfst_ol::Weight final_w = 0.0;
if (t->is_final_state(state_number)) {
- final_w = t->get_final_weight(state_number);
+ final_w = t->get_final_weight(state_number);
}
state_placeholders.push_back(hfst_ol::StatePlaceholder(
state_number,
@@ -182,7 +184,7 @@ void get_states_and_symbols(
first_transition,
final_w));
++first_transition; // there's a padding entry between states
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
+ for (HfstBasicTransitions::const_iterator tr_it
= it->begin();
tr_it != it->end(); ++tr_it) {
++first_transition;
@@ -207,14 +209,14 @@ void get_states_and_symbols(
if (harmonizer == NULL) {
// 1) epsilon
- string_symbol_map[internal_epsilon] = symbol_table.size();
+ string_symbol_map[internal_epsilon] = hfst::size_t_to_ushort(symbol_table.size());
symbol_table.push_back(internal_epsilon);
// 2) input symbols
for (std::set<std::string>::iterator it = input_symbols->begin();
it != input_symbols->end(); ++it) {
if (!is_epsilon(*it)) {
- string_symbol_map[*it] = symbol_table.size();
+ string_symbol_map[*it] = hfst::size_t_to_ushort(symbol_table.size());
symbol_table.push_back(*it);
++seen_input_symbols;
}
@@ -224,7 +226,7 @@ void get_states_and_symbols(
for (std::set<std::string>::iterator it = flag_diacritics->begin();
it != flag_diacritics->end(); ++it) {
if (!is_epsilon(*it)) {
- string_symbol_map[*it] = symbol_table.size();
+ string_symbol_map[*it] = hfst::size_t_to_ushort(symbol_table.size());
// TODO: cl.exe: conversion from 'size_t' to 'char16_t'
flag_symbols.insert((unsigned short)symbol_table.size());
symbol_table.push_back(*it);
@@ -238,7 +240,7 @@ void get_states_and_symbols(
it != other_symbols->end(); ++it) {
if (!is_epsilon(*it) && input_symbols->count(*it) == 0 &&
flag_diacritics->count(*it) == 0) {
- string_symbol_map[*it] = symbol_table.size();
+ string_symbol_map[*it] = hfst::size_t_to_ushort(symbol_table.size());
symbol_table.push_back(*it);
}
}
@@ -262,9 +264,9 @@ void get_states_and_symbols(
// about the states except starting indices
state_number = 0;
- for (HfstBasicTransducer::const_iterator it = t->begin();
+ for (HfstBasicTransducer::const_iterator it = t->begin();
it != t->end(); ++it) {
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
+ for (HfstBasicTransitions::const_iterator tr_it
= it->begin();
tr_it != it->end(); ++tr_it) {
// add input in case we're seeing it the first time
@@ -277,7 +279,7 @@ void get_states_and_symbols(
string_symbol_map[tr_it->get_input_symbol()],
string_symbol_map[tr_it->get_output_symbol()],
tr_it->get_weight());
- SymbolNumber input_sym = string_symbol_map[tr_it->get_input_symbol()];
+ //SymbolNumber input_sym = string_symbol_map[tr_it->get_input_symbol()];
state_placeholders[state_number].add_transition(trans);
}
++state_number;
@@ -291,9 +293,9 @@ void get_states_and_symbols(
(const HfstBasicTransducer * t, bool weighted, std::string options,
HfstTransducer * harmonizer)
{
- const float packing_aggression = 0.85;
+ const float packing_aggression = (float)0.85; // double -> const float
const int floor_jump_threshold = 4; // a packing aggression parameter
- bool quick = options == "quick";
+ //bool quick = options == "quick";
// The transition array is indexed starting from this constant
const unsigned int TA_OFFSET = 2147483648u;
@@ -396,7 +398,7 @@ void get_states_and_symbols(
unsigned int greatest_index = 0;
if (used_indices->indices.size() != 0) {
- greatest_index = used_indices->indices.size() - 1;
+ greatest_index = hfst::size_t_to_uint(used_indices->indices.size() - 1);
}
for(unsigned int i = 0; i <= greatest_index; ++i) {
@@ -441,7 +443,7 @@ void get_states_and_symbols(
hfst_ol::TransducerAlphabet alphabet(symbol_table);
hfst_ol::TransducerHeader header(seen_input_symbols,
- symbol_table.size(),
+ hfst::size_t_to_ushort(symbol_table.size()),
windex_table.size(),
wtransition_table.size(),
weighted);
diff --git a/libhfst/src/implementations/ConvertSfstTransducer.cc b/libhfst/src/implementations/ConvertSfstTransducer.cc
index 6d0b826..5f0b49e 100644
--- a/libhfst/src/implementations/ConvertSfstTransducer.cc
+++ b/libhfst/src/implementations/ConvertSfstTransducer.cc
@@ -1,19 +1,24 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#if HAVE_CONFIG_H
# include <config.h>
#endif
+#if HAVE_SFST
+ #include "back-ends/sfst/interface.h"
+ #include "back-ends/sfst/fst.h"
+#endif
+
#include "ConvertTransducerFormat.h"
-#include "HfstTransitionGraph.h"
-#include "HfstTransducer.h"
+#include "HfstBasicTransducer.h"
+#include "SfstTransducer.h"
#include <stdexcept>
@@ -24,7 +29,7 @@ namespace hfst { namespace implementations
/* -----------------------------------------------------------
- Conversion functions between HfstBasicTransducer and SFST transducer.
+ Conversion functions between HfstBasicTransducer and SFST transducer.
----------------------------------------------------------- */
@@ -41,7 +46,7 @@ namespace hfst { namespace implementations
*/
void ConversionFunctions::sfst_to_hfst_basic_transducer
- ( SFST::Node *node,
+ ( SFST::Node *node,
HfstBasicTransducer *net,
std::vector<unsigned int> &harmonization_vector) {
@@ -49,12 +54,12 @@ namespace hfst { namespace implementations
if (not node->was_visited(VMARK)) {
SFST::Arcs *arcs=node->arcs();
- // Count the number of transitions in the current state
+ // Count the number of transitions in the current state
// and initialize the transition vector of net for maximum efficiency
unsigned int number_of_arcs=0;
for( SFST::ArcsIter p(arcs); p; p++ ) {
number_of_arcs++;
- }
+ }
net->initialize_transition_vector(node->index, number_of_arcs);
// Go through all transitions and copy them to \a net
@@ -62,7 +67,7 @@ namespace hfst { namespace implementations
SFST::Arc *arc=p;
net->add_transition
- (node->index,
+ (node->index,
HfstBasicTransition
(arc->target_node()->index,
harmonization_vector.at(arc->label().lower_char()),
@@ -94,14 +99,14 @@ namespace hfst { namespace implementations
#ifdef DEBUG_CONVERSION
StringSet alphabet_before;
SFST::Alphabet::CharMap CM = t->alphabet.get_char_map();
- for (SFST::Alphabet::CharMap::const_iterator it
- = CM.begin(); it != CM.end(); it++)
+ for (SFST::Alphabet::CharMap::const_iterator it
+ = CM.begin(); it != CM.end(); it++)
{
if (it->first != 0)
alphabet_before.insert(std::string(it->second));
else
alphabet_before.insert(internal_epsilon);
- }
+ }
#endif
// The resulting HfstBasicTransducer
@@ -130,8 +135,8 @@ namespace hfst { namespace implementations
// Copy the alphabet
SFST::Alphabet::CharMap cm = t->alphabet.get_char_map();
- for (SFST::Alphabet::CharMap::const_iterator it
- = cm.begin(); it != cm.end(); it++)
+ for (SFST::Alphabet::CharMap::const_iterator it
+ = cm.begin(); it != cm.end(); it++)
{
assert(strcmp(it->second, "") != 0);
if (it->first != 0) { // The epsilon symbol "<>" is not inserted
@@ -159,7 +164,7 @@ namespace hfst { namespace implementations
t->alphabet.add_symbol(internal_identity.c_str(), 2);
// Copy the alphabet
- for (HfstBasicTransducer::HfstTransitionGraphAlphabet::iterator it
+ for (HfstBasicTransducer::HfstAlphabet::iterator it
= net->alphabet.begin();
it != net->alphabet.end(); it++) {
if (not is_epsilon(*it) && not is_unknown(*it) && not is_identity(*it))
@@ -189,7 +194,7 @@ namespace hfst { namespace implementations
it != net->end(); it++)
{
// Go through the set of transitions in each state
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
+ for (HfstBasicTransitions::const_iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
@@ -200,15 +205,15 @@ namespace hfst { namespace implementations
// Copy transition to node
state_vector[source_state]->add_arc
- (l, state_vector[tr_it->get_target_state()], t);
+ (l, state_vector[tr_it->get_target_state()], t);
}
source_state++;
}
// Go through the final states
- for (HfstBasicTransducer::FinalWeightMap::const_iterator it
+ for (HfstBasicTransducer::FinalWeightMap::const_iterator it
= net->final_weight_map.begin();
- it != net->final_weight_map.end(); it++)
+ it != net->final_weight_map.end(); it++)
{
if (it->first >= state_vector.size()) { // should not happen..
state_vector.push_back(t->new_node());
diff --git a/libhfst/src/implementations/ConvertTransducerFormat.cc b/libhfst/src/implementations/ConvertTransducerFormat.cc
index 2e19fc7..f4ca4c5 100644
--- a/libhfst/src/implementations/ConvertTransducerFormat.cc
+++ b/libhfst/src/implementations/ConvertTransducerFormat.cc
@@ -1,35 +1,46 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#if HAVE_CONFIG_H
# include <config.h>
#endif
+#if HAVE_SFST
+#include "back-ends/sfst/fst.h"
+#endif // HAVE_SFST
+
+#if HAVE_FOMA
+#ifndef _FOMALIB_H_
+#define _FOMALIB_H_
+#include "back-ends/foma/fomalib.h"
+#endif // _FOMALIB_H_
+#endif // HAVE_FOMA
+
#include "ConvertTransducerFormat.h"
#include "optimized-lookup/convert.h"
-#include "HfstTransitionGraph.h"
+#include "HfstBasicTransducer.h"
#include "HfstTransducer.h"
#ifndef MAIN_TEST
namespace hfst { namespace implementations
{
- ConversionFunctions::String2NumberMap
+ ConversionFunctions::String2NumberMap
ConversionFunctions::string_to_number_map;
StringVector ConversionFunctions::number_to_string_vector;
- StringVectorInitializer
+ StringVectorInitializer
dummy3(ConversionFunctions::number_to_string_vector);
- String2NumberMapInitializer
+ String2NumberMapInitializer
dummy4(ConversionFunctions::string_to_number_map);
- std::string ConversionFunctions::get_string(unsigned int number)
+ std::string ConversionFunctions::get_string(unsigned int number)
{
if (number >= number_to_string_vector.size()) {
return std::string(""); } // number not found
@@ -42,14 +53,14 @@ namespace hfst { namespace implementations
string_to_number_map.find(str);
if (it == string_to_number_map.end()) { // string not found
number_to_string_vector.push_back(str);
- unsigned int new_index = number_to_string_vector.size()-1;
+ unsigned int new_index = hfst::size_t_to_uint(number_to_string_vector.size()-1);
string_to_number_map[str] = new_index;
return new_index;
}
return it->second;
}
- ConversionFunctions::NumberVector
+ ConversionFunctions::NumberVector
ConversionFunctions::get_harmonization_vector
(const StringVector &coding_vector)
{
@@ -69,30 +80,44 @@ namespace hfst { namespace implementations
HfstBasicTransducer * ConversionFunctions::
hfst_transducer_to_hfst_basic_transducer
(const hfst::HfstTransducer &t) {
-
#if HAVE_SFST
- if (t.type == SFST_TYPE)
- return sfst_to_hfst_basic_transducer(t.implementation.sfst);
+ if (t.type == SFST_TYPE) {
+ HfstBasicTransducer * retval = sfst_to_hfst_basic_transducer(t.implementation.sfst);
+ retval->name = t.get_name();
+ return retval;
+ }
#endif // HAVE_SFST
#if HAVE_OPENFST
- if (t.type == TROPICAL_OPENFST_TYPE)
- return tropical_ofst_to_hfst_basic_transducer
- (t.implementation.tropical_ofst);
+ if (t.type == TROPICAL_OPENFST_TYPE) {
+ HfstBasicTransducer * retval = tropical_ofst_to_hfst_basic_transducer
+ (t.implementation.tropical_ofst);
+ retval->name = t.get_name();
+ return retval;
+ }
#if HAVE_OPENFST_LOG
- if (t.type == LOG_OPENFST_TYPE)
- return log_ofst_to_hfst_basic_transducer(t.implementation.log_ofst);
+ if (t.type == LOG_OPENFST_TYPE) {
+ HfstBasicTransducer * retval = log_ofst_to_hfst_basic_transducer(t.implementation.log_ofst);
+ retval->name = t.get_name();
+ return retval;
+ }
#endif
#endif // HAVE_OPENFST
#if HAVE_FOMA
- if (t.type == FOMA_TYPE)
- return foma_to_hfst_basic_transducer(t.implementation.foma);
+ if (t.type == FOMA_TYPE) {
+ HfstBasicTransducer * retval = foma_to_hfst_basic_transducer(t.implementation.foma);
+ retval->name = t.get_name();
+ return retval;
+ }
#endif // HAVE_FOMA
#if HAVE_XFSM
- if (t.type == XFSM_TYPE)
- return xfsm_to_hfst_basic_transducer(t.implementation.xfsm);
+ if (t.type == XFSM_TYPE) {
+ HfstBasicTransducer * retval = xfsm_to_hfst_basic_transducer(t.implementation.xfsm);
+ retval->name = t.get_name();
+ return retval;
+ }
#endif // HAVE_FOMA
@@ -100,11 +125,15 @@ namespace hfst { namespace implementations
//#if HAVE_MY_TRANSDUCER_LIBRARY
// if (t.type == MY_TRANSDUCER_LIBRARY_TYPE)
// return my_transducer_library_transducer_to_hfst_basic_transducer
- // (t.implementation.my_transducer_library);
+ // (t.implementation.my_transducer_library);
//#endif // HAVE_MY_TRANSDUCER_LIBRARY
if (t.type == HFST_OL_TYPE || t.type == HFST_OLW_TYPE)
- return hfst_ol_to_hfst_basic_transducer(t.implementation.hfst_ol);
+ {
+ HfstBasicTransducer * retval = hfst_ol_to_hfst_basic_transducer(t.implementation.hfst_ol);
+ retval->name = t.get_name();
+ return retval;
+ }
HFST_THROW(FunctionNotImplementedException);
}
@@ -135,20 +164,35 @@ namespace hfst { namespace implementations
#include <cstdlib>
#include <cassert>
+#include "../HfstTransducer.h"
+
using namespace hfst;
using namespace hfst::implementations;
int main(void)
{
std::cout << "Unit tests for " __FILE__ ":";
- HfstBasicTransducer net;
- net.add_transition(0, HfstBasicTransition(1, "c", "d", 1));
- net.add_transition(1, HfstBasicTransition(2, "a", "o", 2));
- net.add_transition(2, HfstBasicTransition(3, "t", "g", 3));
- net.set_final_weight(3, 4);
- std::cout << std::endl << "Conversions: ";
- std::cout << "skipped everything " <<
- "since they've disappeared into thin air" << std::endl;
- return 77;
+
+ hfst::ImplementationType types [3] = {hfst::SFST_TYPE, hfst::FOMA_TYPE, hfst::TROPICAL_OPENFST_TYPE};
+
+ for (unsigned int i=0; i < 3; i++)
+ {
+ if (hfst::HfstTransducer::is_implementation_type_available(types[i]))
+ {
+ HfstTokenizer tok;
+ HfstTransducer fsm1("cat", "dog", tok, types[i]);
+ fsm1.set_final_weights(4);
+
+ HfstBasicTransducer * fsm1_converted =
+ ConversionFunctions::hfst_transducer_to_hfst_basic_transducer(fsm1);
+
+ HfstTransducer fsm1_converted_twice(*fsm1_converted, types[i]);
+
+ if (! fsm1.compare(fsm1_converted_twice))
+ return 1;
+ }
+ }
+
+ return 0;
}
#endif // MAIN_TEST
diff --git a/libhfst/src/implementations/ConvertTransducerFormat.h b/libhfst/src/implementations/ConvertTransducerFormat.h
index 8b64572..c910fee 100644
--- a/libhfst/src/implementations/ConvertTransducerFormat.h
+++ b/libhfst/src/implementations/ConvertTransducerFormat.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _CONVERT_TRANSDUCER_H_
@@ -15,19 +15,34 @@
#endif // HAVE_CONFIG_H
#include <map>
-#include <iostream>
+#include <iosfwd>
#include <vector>
-#include <map>
#if HAVE_OPENFST
-#ifdef _MSC_VER
-#include "back-ends/openfstwin/src/include/fst/fstlib.h"
-#else
-#include "back-ends/openfst/src/include/fst/fstlib.h"
-#endif // _MSC_VER
+#if OPENFST_VERSION_1_5_4
+ #include "back-ends/openfst/src/include/fst/fst-decl.h"
+ namespace fst { typedef fst::VectorFst<LogArc> LogFst; }
+#else
+namespace fst
+{
+ template <class W> class TropicalWeightTpl;
+ typedef TropicalWeightTpl<float> TropicalWeight;
+
+ template <class W> class LogWeightTpl;
+ typedef LogWeightTpl<float> LogWeight;
+
+ template <class W> class ArcTpl;
+ typedef ArcTpl<TropicalWeight> StdArc;
+ typedef ArcTpl<LogWeight> LogArc;
+
+ template <class A> class VectorFst;
+ typedef VectorFst<StdArc> StdVectorFst;
+ typedef VectorFst<LogArc> LogFst;
+}
+#endif // OPENFST_VERSION_1_5_4
#endif // HAVE_OPENFST
-#if HAVE_SFST
+/*#if HAVE_SFST
#include "back-ends/sfst/fst.h"
#endif // HAVE_SFST
@@ -36,7 +51,7 @@
#define _FOMALIB_H_
#include "back-ends/foma/fomalib.h"
#endif // _FOMALIB_H_
-#endif // HAVE_FOMA
+#endif // HAVE_FOMA*/
#if HAVE_XFSM
#include "xfsm/xfsm_api.h"
@@ -54,26 +69,22 @@
#include "optimized-lookup/transducer.h"
//#include "HfstConstantTransducer.h"
+namespace SFST { class Node; class Transducer; }
struct fsm;
#include "../HfstDataTypes.h"
/** @file ConvertTransducerFormat.h
- \brief Declarations of functions for converting between
+ \brief Declarations of functions for converting between
transducer backend formats. */
-namespace hfst {
+namespace hfst {
namespace implementations {
#if HAVE_OPENFST
- typedef fst::StdArc::StateId StateId;
- typedef fst::ArcIterator<fst::StdVectorFst> StdArcIterator;
-
-#if HAVE_OPENFST_LOG
- typedef fst::ArcTpl<fst::LogWeight> LogArc;
- typedef fst::VectorFst<LogArc> LogFst;
-#endif
+ typedef /*fst::StdArc::StateId*/ unsigned int StateId;
+ //typedef fst::ArcIterator<fst::StdVectorFst> StdArcIterator;
#endif
@@ -98,8 +109,8 @@ namespace implementations {
If \a str is not found, add it to the next free index. */
static unsigned int get_number(const std::string &str);
- /* Get a vector that tells how a transducer that follows
- the number-to-symbol encoding of \a coding should be harmonized so that
+ /* Get a vector that tells how a transducer that follows
+ the number-to-symbol encoding of \a coding should be harmonized so that
it will follow the one of number_to_string_vector. */
static NumberVector get_harmonization_vector
(const StringVector &coding_vector);
@@ -109,7 +120,7 @@ namespace implementations {
#if HAVE_SFST
static void sfst_to_hfst_basic_transducer
- ( SFST::Node *node,
+ ( SFST::Node *node,
HfstBasicTransducer *net,
std::vector<unsigned int> &harmonization_vector);
@@ -120,7 +131,7 @@ namespace implementations {
(const HfstBasicTransducer * t);
/* static void sfst_to_hfst_fast_transducer
- ( SFST::Node *node,
+ ( SFST::Node *node,
HfstFastTransducer *net, NumberVector &harmonization_vector);
static HfstFastTransducer * sfst_to_hfst_fast_transducer
@@ -130,7 +141,7 @@ namespace implementations {
(const HfstFastTransducer * t);
static void sfst_to_hfst_constant_transducer
- ( SFST::Node *node,
+ ( SFST::Node *node,
HfstConstantTransducer *net);
static HfstConstantTransducer * sfst_to_hfst_constant_transducer
@@ -141,9 +152,9 @@ namespace implementations {
#endif // HAVE_SFST
#if HAVE_FOMA
- static HfstBasicTransducer * foma_to_hfst_basic_transducer(struct fsm * t);
+ static HfstBasicTransducer * foma_to_hfst_basic_transducer(fsm * t);
- static struct fsm * hfst_basic_transducer_to_foma
+ static fsm * hfst_basic_transducer_to_foma
(const HfstBasicTransducer * t);
@@ -170,7 +181,7 @@ namespace implementations {
(fst::StdVectorFst * t, bool has_hfst_header=true);
static StateId hfst_state_to_state_id
- (HfstState s, std::map<HfstState, StateId> &state_map,
+ (HfstState s, std::map<HfstState, StateId> &state_map,
fst::StdVectorFst * t);
static fst::StdVectorFst * hfst_basic_transducer_to_tropical_ofst
@@ -193,13 +204,13 @@ namespace implementations {
#if HAVE_OPENFST_LOG
static HfstBasicTransducer * log_ofst_to_hfst_basic_transducer
- (LogFst * t, bool had_hfst_header=true);
+ (fst::LogFst * t, bool had_hfst_header=true);
static StateId hfst_state_to_state_id
- (HfstState s, std::map<HfstState, StateId> &state_map,
- LogFst * t);
+ (HfstState s, std::map<HfstState, StateId> &state_map,
+ fst::LogFst * t);
- static LogFst * hfst_basic_transducer_to_log_ofst
+ static fst::LogFst * hfst_basic_transducer_to_log_ofst
(const HfstBasicTransducer * t);
@@ -218,7 +229,7 @@ namespace implementations {
#endif
-#endif // HAVE_OPENFST
+#endif // HAVE_OPENFST
static HfstBasicTransducer * hfst_ol_to_hfst_basic_transducer
@@ -234,14 +245,14 @@ namespace implementations {
// And the reverse
static hfst_ol::Transducer * hfst_transducer_to_hfst_ol(HfstTransducer * t);
- /* Define here the functions that convert between HfstBasicTransducer and
+ /* Define here the functions that convert between HfstBasicTransducer and
your transducer class. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
- //static HfstBasicTransducer *
+ //static HfstBasicTransducer *
// my_transducer_library_transducer_to_hfst_basic_transducer
// (my_namespace::MyFst * t);
//
- //static my_namespace::MyFst *
+ //static my_namespace::MyFst *
// hfst_basic_transducer_to_my_transducer_library_transducer
// (const HfstBasicTransducer * t);
//#endif // HAVE_MY_TRANSDUCER_LIBRARY
@@ -251,7 +262,7 @@ namespace implementations {
};
- // Initialization of static members in class
+ // Initialization of static members in class
// ConvertTransducerFormat..
class StringVectorInitializer {
public:
diff --git a/libhfst/src/implementations/ConvertTropicalWeightTransducer.cc b/libhfst/src/implementations/ConvertTropicalWeightTransducer.cc
index ca8b66b..c87018a 100644
--- a/libhfst/src/implementations/ConvertTropicalWeightTransducer.cc
+++ b/libhfst/src/implementations/ConvertTropicalWeightTransducer.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#if HAVE_CONFIG_H
@@ -12,8 +12,25 @@
#endif
#include "ConvertTransducerFormat.h"
-#include "HfstTransitionGraph.h"
-#include "HfstTransducer.h"
+#include "HfstBasicTransducer.h"
+#include "TropicalWeightTransducer.h"
+
+#ifdef _MSC_VER
+#include "back-ends/openfstwin/src/include/fst/fstlib.h"
+#else
+#include "back-ends/openfst/src/include/fst/fstlib.h"
+#endif // _MSC_VER
+
+#ifndef _MSC_VER
+namespace fst
+{
+ extern template class TropicalWeightTpl<float>;
+ extern template class ArcTpl<TropicalWeight>;
+ extern template class VectorFst<StdArc>;
+ extern template class ArcIterator<StdVectorFst>;
+ extern template class StateIterator<StdVectorFst>;
+}
+#endif // _MSC_VER
#ifndef MAIN_TEST
namespace hfst { namespace implementations
@@ -37,37 +54,37 @@ namespace hfst { namespace implementations
// An empty transducer
if (t->Start() == fst::kNoStateId)
- {
+ {
/* An empty OpenFst transducer does not necessarily have to have
an input or output symbol table. */
if (inputsym != NULL) {
- for ( fst::SymbolTableIterator it =
+ for ( fst::SymbolTableIterator it =
fst::SymbolTableIterator(*(inputsym));
! it.Done(); it.Next() ) {
assert(it.Symbol() != "");
if (it.Value() != 0) // epsilon is not inserted
net->add_symbol_to_alphabet( it.Symbol() );
- }
+ }
}
/* If the transducer is an OpenFst transducer, it might have an output
symbol table. If the transducer is an HFST tropical transducer, it
- can have an output symbol table, but it is equivalent to the
+ can have an output symbol table, but it is equivalent to the
input symbol table. */
if (! has_hfst_header && outputsym != NULL) {
- for ( fst::SymbolTableIterator it =
+ for ( fst::SymbolTableIterator it =
fst::SymbolTableIterator(*(outputsym));
! it.Done(); it.Next() ) {
assert(it.Symbol() != "");
if (it.Value() != 0) // epsilon is not inserted
net->add_symbol_to_alphabet( it.Symbol() );
- }
+ }
}
return;
}
/* A non-empty OpenFst transducer must have at least an input symbol table.
- If the output symbol table is missing, we assume that it would be
+ If the output symbol table is missing, we assume that it would be
equivalent to the input symbol table. */
if (inputsym == NULL) {
HFST_THROW(MissingOpenFstInputSymbolTableException);
@@ -82,27 +99,27 @@ namespace hfst { namespace implementations
const fst::SymbolTable *inputsym = t->InputSymbols();
const fst::SymbolTable *outputsym = t->OutputSymbols();
- if (inputsym != NULL)
+ if (inputsym != NULL)
{
- for ( fst::SymbolTableIterator it =
+ for ( fst::SymbolTableIterator it =
fst::SymbolTableIterator(*(inputsym));
- ! it.Done(); it.Next() )
+ ! it.Done(); it.Next() )
{
assert(it.Symbol() != "");
if (it.Value() != 0) // epsilon is not inserted
net->add_symbol_to_alphabet( it.Symbol() );
- }
+ }
}
if (outputsym != NULL)
{
- for ( fst::SymbolTableIterator it =
+ for ( fst::SymbolTableIterator it =
fst::SymbolTableIterator(*(outputsym));
- ! it.Done(); it.Next() )
+ ! it.Done(); it.Next() )
{
assert(it.Symbol() != "");
if (it.Value() != 0) // epsilon is not inserted
net->add_symbol_to_alphabet( it.Symbol() );
- }
+ }
}
}
@@ -111,9 +128,9 @@ namespace hfst { namespace implementations
/* ------------------------------------------------------------------------
Create an HfstBasicTransducer equivalent to an OpenFst tropical weight
- transducer \a t.
+ transducer \a t.
- ------------------------------------------------------------------------ */
+ ------------------------------------------------------------------------ */
HfstBasicTransducer * ConversionFunctions::
tropical_ofst_to_hfst_basic_transducer
@@ -125,7 +142,7 @@ namespace hfst { namespace implementations
StringVector symbol_vector = TropicalWeightTransducer::get_symbol_vector(t);
- std::vector<unsigned int> harmonization_vector
+ std::vector<unsigned int> harmonization_vector
= HfstTropicalTransducerTransitionData::get_harmonization_vector(symbol_vector);
/* This takes care that initial state is always number zero
@@ -135,8 +152,8 @@ namespace hfst { namespace implementations
StateId initial_state = t->Start();
/* Go through all states */
- for (fst::StateIterator<fst::StdVectorFst> siter(*t);
- ! siter.Done(); siter.Next())
+ for (fst::StateIterator<fst::StdVectorFst> siter(*t);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
@@ -150,7 +167,7 @@ namespace hfst { namespace implementations
net->initialize_transition_vector(s, number_of_arcs);
/* Go through all transitions in a state */
- for (fst::ArcIterator<fst::StdVectorFst> aiter(*t,s);
+ for (fst::ArcIterator<fst::StdVectorFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const fst::StdArc &arc = aiter.Value();
@@ -176,15 +193,15 @@ namespace hfst { namespace implementations
//exit(1);
}
- net->add_transition(origin,
+ net->add_transition(origin,
HfstBasicTransition
(target,
harmonization_vector[arc.ilabel],
harmonization_vector[arc.olabel],
arc.weight.Value(),
- false), // dummy parameter needed because numbers are used
+ false), // dummy parameter needed because numbers are used
false); // do not insert symbols to alphabet
- }
+ }
if (t->Final(s) != fst::TropicalWeight::Zero()) {
// Set the state as final
@@ -203,7 +220,7 @@ namespace hfst { namespace implementations
/* ---------------------------------------------------------------------------
- Create an OpenFst transducer equivalent to HfstBasicTransducer \a net.
+ Create an OpenFst transducer equivalent to HfstBasicTransducer \a net.
--------------------------------------------------------------------------- */
@@ -228,7 +245,7 @@ namespace hfst { namespace implementations
st.AddSymbol(internal_identity, 2);
// Copy the alphabet
- for (HfstBasicTransducer::HfstTransitionGraphAlphabet::iterator it
+ for (HfstBasicTransducer::HfstAlphabet::iterator it
= net->alphabet.begin();
it != net->alphabet.end(); it++) {
assert(! it->empty());
@@ -241,7 +258,7 @@ namespace hfst { namespace implementations
it != net->end(); it++)
{
// Go through the set of transitions in each state...
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
+ for (HfstBasicTransitions::const_iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
@@ -262,9 +279,9 @@ namespace hfst { namespace implementations
} // ... all states gone through
// Go through the final states...
- for (HfstBasicTransducer::FinalWeightMap::const_iterator it
+ for (HfstBasicTransducer::FinalWeightMap::const_iterator it
= net->final_weight_map.begin();
- it != net->final_weight_map.end(); it++)
+ it != net->final_weight_map.end(); it++)
{
t->SetFinal
(state_vector[it->first],
@@ -273,7 +290,7 @@ namespace hfst { namespace implementations
// ... final states gone through
t->SetInputSymbols(&st);
- return t;
+ return t;
}
#endif // HAVE_OPENFST
diff --git a/libhfst/src/implementations/ConvertXfsmTransducer.cc b/libhfst/src/implementations/ConvertXfsmTransducer.cc
index 2f3d1c8..541dd01 100644
--- a/libhfst/src/implementations/ConvertXfsmTransducer.cc
+++ b/libhfst/src/implementations/ConvertXfsmTransducer.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#if HAVE_CONFIG_H
@@ -12,8 +12,8 @@
#endif
#include "ConvertTransducerFormat.h"
-#include "HfstTransitionGraph.h"
-#include "HfstTransducer.h"
+#include "HfstBasicTransducer.h"
+//#include "HfstTransducer.h"
#ifndef MAIN_TEST
namespace hfst { namespace implementations
@@ -21,7 +21,7 @@ namespace hfst { namespace implementations
/* -----------------------------------------------------------
- Conversion functions between HfstBasicTransducer and xfsm transducer.
+ Conversion functions between HfstBasicTransducer and xfsm transducer.
---------------------------------------------------------- */
@@ -49,12 +49,12 @@ namespace hfst { namespace implementations
/* ----------------------------------------------------------------------
- Create an HfstBasicTransducer equivalent to foma transducer \a t.
+ Create an HfstBasicTransducer equivalent to foma transducer \a t.
---------------------------------------------------------------------- */
HfstBasicTransducer * ConversionFunctions::
- xfsm_to_hfst_basic_transducer(NETptr t)
+ xfsm_to_hfst_basic_transducer(NETptr t)
{
HfstBasicTransducer * result = new HfstBasicTransducer();
@@ -68,7 +68,7 @@ namespace hfst { namespace implementations
while (state_ptr != NULL)
{
// initial state exists already
- if (state_ptr != start_ptr)
+ if (state_ptr != start_ptr)
{
(void)result->add_state();
}
@@ -82,24 +82,24 @@ namespace hfst { namespace implementations
HfstState result_state = result->get_max_state();
while (state_ptr != NULL)
{
- if (state_ptr == start_ptr)
+ if (state_ptr == start_ptr)
{
// initial state exists already in result
xfsm_to_hfst_state.insert(std::pair<STATEptr, HfstState>(state_ptr, 0));
- if (state_ptr->final != 0)
+ if (state_ptr->final != 0)
{
result->set_final_weight(0, 0);
}
- }
+ }
else
{
xfsm_to_hfst_state.insert(std::pair<STATEptr, HfstState>(state_ptr, result_state));
- if (state_ptr->final != 0)
+ if (state_ptr->final != 0)
{
result->set_final_weight(result_state, 0);
}
--result_state;
- }
+ }
state_ptr = state_ptr->next;
}
@@ -133,12 +133,12 @@ namespace hfst { namespace implementations
/* ------------------------------------------------------------------------
- Create an xfsm transducer equivalent to HfstBasicTransducer \a hfst_fsm.
+ Create an xfsm transducer equivalent to HfstBasicTransducer \a hfst_fsm.
------------------------------------------------------------------------ */
NETptr ConversionFunctions::
- hfst_basic_transducer_to_xfsm(const HfstBasicTransducer * hfst_fsm)
+ hfst_basic_transducer_to_xfsm(const HfstBasicTransducer * hfst_fsm)
{
NETptr result = null_net();
@@ -225,7 +225,7 @@ namespace hfst { namespace implementations
if (hfst::is_epsilon(*it) || hfst::is_unknown(*it) || hfst::is_identity(*it))
continue;
(void) alph_add_to(ap, XfsmTransducer::hfst_symbol_to_xfsm_symbol(it->c_str()), DONT_KEEP);
- }
+ }
return result;
}
diff --git a/libhfst/src/implementations/FomaTransducer.cc b/libhfst/src/implementations/FomaTransducer.cc
index dc56822..3b034e9 100644
--- a/libhfst/src/implementations/FomaTransducer.cc
+++ b/libhfst/src/implementations/FomaTransducer.cc
@@ -1,12 +1,17 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
+#ifndef _FOMALIB_H_
+#define _FOMALIB_H_
+#include "back-ends/foma/fomalib.h"
+#endif
+
#include "FomaTransducer.h"
#ifndef MAIN_TEST
@@ -27,9 +32,9 @@ namespace hfst { namespace implementations {
if (filename == std::string())
{ input_file = stdin; }
else {
- input_file = fopen(filename.c_str(),"r");
+ input_file = hfst::hfst_fopen(filename.c_str(),"r");
if (input_file == NULL)
- {
+ {
HFST_THROW(StreamNotReadableException); }
}
}
@@ -49,9 +54,9 @@ namespace hfst { namespace implementations {
char FomaInputStream::stream_get() {
return (char) fgetc(input_file); }
- short FomaInputStream::stream_get_short()
+ short FomaInputStream::stream_get_short()
{
- short i;
+ short i = 0;
assert(1 == fread(&i,sizeof(i),1,input_file));
return i;
}
@@ -99,11 +104,11 @@ namespace hfst { namespace implementations {
/* Skip the identifier string "FOMA_TYPE" */
void FomaInputStream::skip_identifier_version_3_0(void)
- {
+ {
char foma_identifier[10];
- int foma_id_count = fread(foma_identifier,10,1,input_file);
+ int foma_id_count = hfst::size_t_to_int(fread(foma_identifier,10,1,input_file));
if (foma_id_count != 1)
- {
+ {
HFST_THROW(NotTransducerStreamException); }
if (0 != strcmp(foma_identifier,"FOMA_TYPE"))
{
@@ -113,9 +118,9 @@ namespace hfst { namespace implementations {
void FomaInputStream::skip_hfst_header(void)
{
char hfst_header[6];
- int header_count = fread(hfst_header,6,1,input_file);
+ int header_count = hfst::size_t_to_int(fread(hfst_header,6,1,input_file));
if (header_count != 1)
- {
+ {
HFST_THROW(NotTransducerStreamException); }
try { skip_identifier_version_3_0(); }
catch (const HfstException e)
@@ -123,7 +128,7 @@ namespace hfst { namespace implementations {
}
void FomaInputStream::ignore(unsigned int n)
- {
+ {
for (unsigned int i=0; i<n; i++)
fgetc(input_file);
}
@@ -132,7 +137,7 @@ namespace hfst { namespace implementations {
{
if (is_eof())
return NULL;
- struct fsm * t = FomaTransducer::read_net(input_file);
+ fsm * t = FomaTransducer::read_net(input_file);
if (t == NULL) {
HFST_THROW(NotTransducerStreamException);
}
@@ -148,17 +153,17 @@ namespace hfst { namespace implementations {
filename(std::string(str))
{
if (filename != std::string()) {
- ofile = fopen(filename.c_str(), "wb");
+ ofile = hfst::hfst_fopen(filename.c_str(), "wb");
if (ofile == NULL) {
HFST_THROW(StreamNotReadableException);
}
- }
+ }
else {
ofile = stdout;
}
}
- void FomaOutputStream::close(void)
+ void FomaOutputStream::close(void)
{
if (filename != std::string())
{ fclose(ofile); }
@@ -169,8 +174,8 @@ namespace hfst { namespace implementations {
fputc(c,ofile);
}
- void FomaOutputStream::write_transducer(struct fsm * transducer)
- {
+ void FomaOutputStream::write_transducer(fsm * transducer)
+ {
if (1 != FomaTransducer::write_net(transducer, ofile)) {
HFST_THROW_MESSAGE
(HfstFatalException,
@@ -180,8 +185,8 @@ namespace hfst { namespace implementations {
fsm * FomaTransducer::create_empty_transducer(void)
- {
- struct fsm * retval = fsm_empty_set();
+ {
+ fsm * retval = fsm_empty_set();
return retval;
}
@@ -190,18 +195,18 @@ namespace hfst { namespace implementations {
return fsm_empty_string();
}
- void FomaTransducer::harmonize(struct fsm *net1, struct fsm *net2)
+ void FomaTransducer::harmonize(fsm *net1, fsm *net2)
{
fsm_merge_sigma(net1, net2);
}
- void FomaTransducer::delete_foma(struct fsm * net)
+ void FomaTransducer::delete_foma(fsm * net)
{
fsm_destroy(net);
}
fsm * FomaTransducer::define_transducer(const std::string &symbol)
- {
+ {
/*if (symbol == hfst::internal_unknown) {
return define_transducer(symbol, symbol);
}*/
@@ -212,12 +217,12 @@ namespace hfst { namespace implementations {
fsm * FomaTransducer::define_transducer
(const std::string &isymbol, const std::string &osymbol)
- {
+ {
// identity-to-identity gives wrong result if cross product is used
if (isymbol == osymbol && isymbol == hfst::internal_identity) {
return define_transducer(isymbol);
}
- return fsm_cross_product( fsm_symbol(const_cast<char*>(isymbol.c_str())),
+ return fsm_cross_product( fsm_symbol(const_cast<char*>(isymbol.c_str())),
fsm_symbol(const_cast<char*>(osymbol.c_str())) );
// should either argument be deleted?
}
@@ -230,20 +235,23 @@ namespace hfst { namespace implementations {
int state_number=0;
struct fsm_construct_handle *h;
- struct fsm *net;
- h = fsm_construct_init(strdup(std::string("").c_str()));
+ fsm *net;
+ char * empty = strdup(std::string("").c_str());
+ h = fsm_construct_init(empty);
+ free(empty);
- for (StringPairVector::const_iterator it = spv.begin();
- it != spv.end(); it++)
+ for (StringPairVector::const_iterator it = spv.begin();
+ it != spv.end(); it++)
{
char *in = strdup(it->first.c_str());
char *out = strdup(it->second.c_str());
fsm_construct_add_arc(h, state_number, state_number+1, in, out);
+ free(in); free(out);
state_number++;
}
fsm_construct_set_initial(h, 0);
- fsm_construct_set_final(h, state_number);
+ fsm_construct_set_final(h, state_number);
net = fsm_construct_done(h);
fsm_count(net);
@@ -252,7 +260,7 @@ namespace hfst { namespace implementations {
sigma_add_special (1, net->sigma);
sigma_add_special (2, net->sigma);
- return net;
+ return net;
}
fsm * FomaTransducer::define_transducer(const StringPairSet &sps, bool cyclic)
@@ -264,10 +272,10 @@ namespace hfst { namespace implementations {
int target = (cyclic) ? 0 : 1;
struct fsm_construct_handle *h;
- struct fsm *net;
+ fsm *net;
h = fsm_construct_init(strdup(std::string("").c_str()));
- for (StringPairSet::const_iterator it = sps.begin(); it != sps.end(); it++)
+ for (StringPairSet::const_iterator it = sps.begin(); it != sps.end(); it++)
{
char *in = strdup(it->first.c_str());
char *out = strdup(it->second.c_str());
@@ -275,12 +283,12 @@ namespace hfst { namespace implementations {
}
fsm_construct_set_initial(h, source);
- fsm_construct_set_final(h, target);
+ fsm_construct_set_final(h, target);
net = fsm_construct_done(h);
fsm_count(net);
- return net;
+ return net;
}
fsm * FomaTransducer::define_transducer
@@ -292,13 +300,13 @@ namespace hfst { namespace implementations {
int state_number=0;
struct fsm_construct_handle *h;
- struct fsm *net;
+ fsm *net;
h = fsm_construct_init(strdup(std::string("").c_str()));
- for (std::vector<StringPairSet>::const_iterator it = spsv.begin();
- it != spsv.end(); it++)
+ for (std::vector<StringPairSet>::const_iterator it = spsv.begin();
+ it != spsv.end(); it++)
{
- for (StringPairSet::const_iterator it2 = (*it).begin();
+ for (StringPairSet::const_iterator it2 = (*it).begin();
it2 != (*it).end(); it2++ ) {
char *in = strdup(it2->first.c_str());
char *out = strdup(it2->second.c_str());
@@ -308,27 +316,27 @@ namespace hfst { namespace implementations {
}
fsm_construct_set_initial(h, 0);
- fsm_construct_set_final(h, state_number);
+ fsm_construct_set_final(h, state_number);
net = fsm_construct_done(h);
fsm_count(net);
- return net;
+ return net;
}
fsm * FomaTransducer::copy(fsm * t)
- {
+ {
return fsm_copy(t);
}
fsm * FomaTransducer::determinize(fsm * t)
- {
+ {
// returns the argument, so a copy is taken here
return fsm_determinize(fsm_copy(t));
}
fsm * FomaTransducer::minimize(fsm * t)
- {
+ {
// returns the argument, so a copy is taken here
return fsm_minimize(fsm_copy(t));
}
@@ -340,27 +348,27 @@ namespace hfst { namespace implementations {
}
fsm * FomaTransducer::repeat_star(fsm * t)
- {
+ {
return fsm_kleene_star(fsm_copy(t));
}
fsm * FomaTransducer::repeat_plus(fsm * t)
- {
+ {
return fsm_kleene_plus(fsm_copy(t));
}
fsm * FomaTransducer::repeat_n(fsm * t, unsigned int n)
- {
+ {
return fsm_concat_n(fsm_copy(t), n);
}
fsm * FomaTransducer::repeat_le_n(fsm * t, unsigned int n)
- {
+ {
return fsm_concat_m_n(fsm_copy(t),0,n);
}
fsm * FomaTransducer::optionalize(fsm * t)
- {
+ {
return fsm_optionality(fsm_copy(t));
}
@@ -370,12 +378,12 @@ namespace hfst { namespace implementations {
}
fsm * FomaTransducer::reverse(fsm * t)
- {
+ {
return fsm_reverse(fsm_copy(t));
}
fsm * FomaTransducer::extract_input_language(fsm * t)
- {
+ {
// foma does not handle epsilon transducer properly..
return fsm_upper(fsm_copy(t));
}
@@ -388,8 +396,8 @@ namespace hfst { namespace implementations {
fsm * FomaTransducer::substitute(fsm * t,String old_symbol,String new_symbol)
{
- return fsm_substitute_symbol(t,
- strdup(old_symbol.c_str()),
+ return fsm_substitute_symbol(t,
+ strdup(old_symbol.c_str()),
strdup(new_symbol.c_str()));
}
@@ -399,13 +407,13 @@ namespace hfst { namespace implementations {
const char * epsilon = internal_epsilon.c_str();
char * epsilon_marker = strdup("@_EPSILON_SYMBOL_MARKER_@");
const char * identity = internal_identity.c_str();
- fsm * eps_marked =
- fsm_substitute_symbol(t, const_cast<char*>(epsilon),
+ fsm * eps_marked =
+ fsm_substitute_symbol(t, const_cast<char*>(epsilon),
epsilon_marker);
fsm * ins = fsm_kleene_star
- (
- fsm_union(
- fsm_symbol(const_cast<char*>(identity)),
+ (
+ fsm_union(
+ fsm_symbol(const_cast<char*>(identity)),
fsm_cross_product
( fsm_symbol(const_cast<char*>(epsilon)),
fsm_symbol(const_cast<char*>(symbol_pair.second.c_str())
@@ -418,7 +426,7 @@ namespace hfst { namespace implementations {
epsilon_marker,
const_cast<char*>(epsilon));
free(epsilon_marker);
- // marker should be removed from sigma..
+ // marker should be removed from sigma..
// (HfstBasicTransducer is now used instead)
}
@@ -455,8 +463,12 @@ namespace hfst { namespace implementations {
bool FomaTransducer::are_equivalent
(fsm * t1, fsm * t2)
{
- return fsm_isempty(fsm_union(fsm_minus(fsm_copy(t1),fsm_copy(t2)),
- fsm_minus(fsm_copy(t2),fsm_copy(t1))));
+ fsm * test =
+ fsm_union(fsm_minus(fsm_copy(t1),fsm_copy(t2)),
+ fsm_minus(fsm_copy(t2),fsm_copy(t1)));
+ int eq = fsm_isempty(test);
+ //fsm_destroy(test);
+ return (eq == 1);
}
bool FomaTransducer::is_cyclic(fsm * t)
@@ -492,11 +504,11 @@ namespace hfst { namespace implementations {
static bool extract_paths
(fsm * t, int state,
- std::map<int,unsigned short> all_visitations,
+ std::map<int,unsigned short> all_visitations,
std::map<int, unsigned short> path_visitations,
ExtractStringsCb& callback, int cycles,
- std::vector<hfst::FdState<int> >* fd_state_stack,
- bool filter_fd,
+ std::vector<hfst::FdState<int> >* fd_state_stack,
+ bool filter_fd,
StringPairVector &spv)
{
@@ -541,7 +553,7 @@ namespace hfst { namespace implementations {
{
size_t j;
for(j=0; j<sorted_arcs.size(); j++)
- if (all_visitations[s->target]
+ if (all_visitations[s->target]
< all_visitations[sorted_arcs[j]->target])
break;
sorted_arcs.push_back(NULL);
@@ -570,7 +582,7 @@ namespace hfst { namespace implementations {
}
}
- /* Handle spv here. Special symbols (flags, epsilons)
+ /* Handle spv here. Special symbols (flags, epsilons)
are always inserted. */
std::string istring("");
@@ -578,7 +590,7 @@ namespace hfst { namespace implementations {
//find the key in sigma
char* c_in=NULL;
- for(struct sigma* sig=t->sigma; sig!=NULL&&sig->symbol!=NULL;
+ for(struct sigma* sig=t->sigma; sig!=NULL&&sig->symbol!=NULL;
sig=sig->next)
{ if(sig->number == arc->in) {
c_in = sig->symbol;
@@ -587,7 +599,7 @@ namespace hfst { namespace implementations {
//find the key in sigma
char* c_out=NULL;
- for(struct sigma* sig=t->sigma; sig!=NULL&&sig->symbol!=NULL;
+ for(struct sigma* sig=t->sigma; sig!=NULL&&sig->symbol!=NULL;
sig=sig->next) {
if(sig->number == arc->out) {
c_out = sig->symbol;
@@ -600,7 +612,7 @@ namespace hfst { namespace implementations {
istring = strdup(c_in);
}
- if (!filter_fd ||
+ if (!filter_fd ||
fd_state_stack->back().get_table().get_operation(arc->out)==NULL) {
assert(c_out != NULL);
ostring = strdup(c_out);
@@ -627,11 +639,11 @@ namespace hfst { namespace implementations {
void FomaTransducer::extract_paths
(fsm * t, ExtractStringsCb& callback,
- int cycles, FdTable<int>* fd, bool filter_fd)
+ int cycles, FdTable<int>* fd, bool filter_fd)
{
std::map<int, unsigned short> all_visitations;
std::map<int, unsigned short> path_visitations;
- std::vector<hfst::FdState<int> >* fd_state_stack
+ std::vector<hfst::FdState<int> >* fd_state_stack
= (fd==NULL) ? NULL : new std::vector<hfst::FdState<int> >
(1, hfst::FdState<int>(*fd));
@@ -640,7 +652,7 @@ namespace hfst { namespace implementations {
StringPairVector spv;
bool res = true;
for (int i=0; ((t->states)+i)->state_no != -1 && res == true; i++) {
- if (((t->states)+i)->start_state == 1 &&
+ if (((t->states)+i)->start_state == 1 &&
(initial_states.find(((t->states)+i)->state_no)
== initial_states.end()) ) {
@@ -648,12 +660,12 @@ namespace hfst { namespace implementations {
res = hfst::implementations::extract_paths
(t, ((t->states)+i)->state_no, all_visitations, path_visitations,
- callback, cycles, fd_state_stack,
+ callback, cycles, fd_state_stack,
filter_fd, spv);
}
}
// add epsilon path, if needed
- for (std::set<int>::const_iterator it = initial_states.begin();
+ for (std::set<int>::const_iterator it = initial_states.begin();
it != initial_states.end(); it++)
{
bool final_initial = false;
@@ -687,7 +699,7 @@ namespace hfst { namespace implementations {
void FomaTransducer::insert_to_alphabet(fsm * t, const std::string &symbol)
{
sigma_add(strdup(symbol.c_str()), t->sigma);
- }
+ }
void FomaTransducer::remove_from_alphabet
(fsm * t, const std::string &symbol)
@@ -713,7 +725,7 @@ namespace hfst { namespace implementations {
}
unsigned int FomaTransducer::get_symbol_number
- (fsm *t,
+ (fsm *t,
const std::string &symbol)
{
if (symbol == internal_epsilon)
@@ -746,7 +758,7 @@ namespace hfst { namespace implementations {
// epsilon, unknown and identity are always included and
// get_symbol_number always returns a value for them
if (biggest_number < 2)
- return 2;
+ return 2;
return biggest_number;
}
@@ -822,15 +834,15 @@ namespace hfst { namespace implementations {
}
#endif
- struct fsm * FomaTransducer::eliminate_flags(struct fsm * t)
+ fsm * FomaTransducer::eliminate_flags(fsm * t)
{
return flag_eliminate(t, NULL);
}
- struct fsm * FomaTransducer::eliminate_flag(struct fsm * t, const std::string & flag)
+ fsm * FomaTransducer::eliminate_flag(fsm * t, const std::string & flag)
{
char * flag_ = strdup(flag.c_str());
- struct fsm * retval = flag_eliminate(t, flag_);
+ fsm * retval = flag_eliminate(t, flag_);
free(flag_);
return retval;
}
@@ -846,12 +858,12 @@ namespace hfst { namespace implementations {
static inline int explode_line (char *buf, int *values);
/* Read foma transducer . */
- struct fsm * FomaTransducer::read_net(FILE *infile) {
+ fsm * FomaTransducer::read_net(FILE *infile) {
- const unsigned int READ_BUF_SIZE=4096;
+ const unsigned int READ_BUF_SIZE=4096;
char buf[READ_BUF_SIZE];
- struct fsm *net;
- struct fsm_state *fsm;
+ fsm *net;
+ struct fsm_state * fsm_;
char *new_symbol;
int i, items, new_symbol_number, laststate, lineint[5], *cm;
@@ -876,24 +888,24 @@ namespace hfst { namespace implementations {
}
/* Properties */
io_gets(infile, buf);
- sscanf(buf, "%i %i %i %i %i " LONG_LONG_SPECIFIER " %i %i %i %i %i %i %s",
- &net->arity,
- &net->arccount,
- &net->statecount,
- &net->linecount,
- &net->finalcount,
- &net->pathcount,
- &net->is_deterministic,
- &net->is_pruned,
- &net->is_minimized,
- &net->is_epsilon_free,
- &net->is_loop_free,
- &net->is_completed,
+ sscanf(buf, "%i %i %i %i %i " LONG_LONG_SPECIFIER " %i %i %i %i %i %i %s",
+ &net->arity,
+ &net->arccount,
+ &net->statecount,
+ &net->linecount,
+ &net->finalcount,
+ &net->pathcount,
+ &net->is_deterministic,
+ &net->is_pruned,
+ &net->is_minimized,
+ &net->is_epsilon_free,
+ &net->is_loop_free,
+ &net->is_completed,
buf);
// The following strcpy is commented out because we want to leave the
// empty name we've written earlier and use HFST3's name scheme
// for everything. NB: There's a limit of 40 chars for net->name.
- //strcpy(net->name, buf);
+ //strcpy(net->name, buf);
io_gets(infile, buf);
/* Sigma */
@@ -919,48 +931,48 @@ namespace hfst { namespace implementations {
net->states = (fsm_state*) malloc(net->linecount*sizeof(struct fsm_state)); // error
// DEBUG
//fprintf(stderr, "Reserved space for %i state lines\n", net->linecount);
- fsm = net->states;
+ fsm_ = net->states;
laststate = -1;
for (i=0; ;i++) {
io_gets(infile, buf);
if (buf[0] == '#') break;
/* scanf is just too slow here */
- //items = sscanf(buf, "%i %i %i %i %i",&lineint[0],
+ //items = sscanf(buf, "%i %i %i %i %i",&lineint[0],
//&lineint[1], &lineint[2], &lineint[3], &lineint[4]);
items = explode_line(buf, &lineint[0]);
switch (items) {
case 2:
- (fsm+i)->state_no = laststate;
- (fsm+i)->in = lineint[0];
- (fsm+i)->out = lineint[0];
- (fsm+i)->target = lineint[1];
- (fsm+i)->final_state = last_final;
+ (fsm_+i)->state_no = laststate;
+ (fsm_+i)->in = lineint[0];
+ (fsm_+i)->out = lineint[0];
+ (fsm_+i)->target = lineint[1];
+ (fsm_+i)->final_state = last_final;
break;
case 3:
- (fsm+i)->state_no = laststate;
- (fsm+i)->in = lineint[0];
- (fsm+i)->out = lineint[1];
- (fsm+i)->target = lineint[2];
- (fsm+i)->final_state = last_final;
+ (fsm_+i)->state_no = laststate;
+ (fsm_+i)->in = lineint[0];
+ (fsm_+i)->out = lineint[1];
+ (fsm_+i)->target = lineint[2];
+ (fsm_+i)->final_state = last_final;
break;
case 4:
- (fsm+i)->state_no = lineint[0];
- (fsm+i)->in = lineint[1];
- (fsm+i)->out = lineint[1];
- (fsm+i)->target = lineint[2];
- (fsm+i)->final_state = lineint[3];
+ (fsm_+i)->state_no = lineint[0];
+ (fsm_+i)->in = lineint[1];
+ (fsm_+i)->out = lineint[1];
+ (fsm_+i)->target = lineint[2];
+ (fsm_+i)->final_state = lineint[3];
laststate = lineint[0];
last_final = lineint[3];
break;
case 5:
- (fsm+i)->state_no = lineint[0]; // error
- (fsm+i)->in = lineint[1];
- (fsm+i)->out = lineint[2];
- (fsm+i)->target = lineint[3];
- (fsm+i)->final_state = lineint[4];
+ (fsm_+i)->state_no = lineint[0]; // error
+ (fsm_+i)->in = lineint[1];
+ (fsm_+i)->out = lineint[2];
+ (fsm_+i)->target = lineint[3];
+ (fsm_+i)->final_state = lineint[4];
laststate = lineint[0];
last_final = lineint[4];
break;
@@ -969,11 +981,11 @@ namespace hfst { namespace implementations {
return NULL;
}
if (laststate > 0) {
- (fsm+i)->start_state = 0;
+ (fsm_+i)->start_state = 0;
} else if (laststate == -1) {
- (fsm+i)->start_state = -1;
+ (fsm_+i)->start_state = -1;
} else {
- (fsm+i)->start_state = 1;
+ (fsm_+i)->start_state = 1;
}
}
@@ -986,7 +998,7 @@ namespace hfst { namespace implementations {
sscanf(buf,"%i", &i);
*cm = i;
cm++;
- }
+ }
}
if (strcmp(buf, "##end##") != 0) {
printf("File format error!\n");
@@ -1001,7 +1013,7 @@ static int io_gets(FILE *infile, char *target) {
for (i = 0; c != '\n' && c != '\0'; i++) {
*(target+i) = c;
c = getc(infile);
- }
+ }
*(target+i) = '\0';
// Windows...
@@ -1037,7 +1049,7 @@ static inline int explode_line (char *buf, int *values) {
int FomaTransducer::write_net(fsm * net, FILE * outfile) {
struct sigma *sigma;
- struct fsm_state *fsm;
+ struct fsm_state * fsm_;
int i, maxsigma, laststate, *cm;
// If this is not done, linecount can return a false value.
@@ -1049,50 +1061,50 @@ static inline int explode_line (char *buf, int *values) {
/* Properties */
fprintf(outfile, "%s","##props##\n");
fprintf(outfile, "%i %i %i %i %i " LONG_LONG_SPECIFIER " %i %i %i %i %i %i %s\n",
- net->arity,
- net->arccount,
- net->statecount,
- net->linecount,
- net->finalcount,
- net->pathcount,
- net->is_deterministic,
- net->is_pruned,
- net->is_minimized,
- net->is_epsilon_free,
- net->is_loop_free,
- net->is_completed,
+ net->arity,
+ net->arccount,
+ net->statecount,
+ net->linecount,
+ net->finalcount,
+ net->pathcount,
+ net->is_deterministic,
+ net->is_pruned,
+ net->is_minimized,
+ net->is_epsilon_free,
+ net->is_loop_free,
+ net->is_completed,
net->name);
/* Sigma */
fprintf(outfile, "%s","##sigma##\n");
- for (sigma = net->sigma; sigma != NULL && sigma->number != -1;
+ for (sigma = net->sigma; sigma != NULL && sigma->number != -1;
sigma = sigma->next) {
fprintf(outfile, "%i %s\n",sigma->number, sigma->symbol);
}
/* State array */
laststate = -1;
- fsm = net->states;
+ fsm_ = net->states;
fprintf(outfile, "%s","##states##\n");
- for (fsm = net->states; fsm->state_no !=-1; fsm++) {
- if (fsm->state_no != laststate) {
- if (fsm->in != fsm->out) {
+ for (fsm_ = net->states; fsm_->state_no !=-1; fsm_++) {
+ if (fsm_->state_no != laststate) {
+ if (fsm_->in != fsm_->out) {
fprintf(outfile, "%i %i %i %i %i\n",
- fsm->state_no, fsm->in, fsm->out, fsm->target,
- fsm->final_state);
+ fsm_->state_no, fsm_->in, fsm_->out, fsm_->target,
+ fsm_->final_state);
} else {
- fprintf(outfile, "%i %i %i %i\n",fsm->state_no, fsm->in,
- fsm->target, fsm->final_state);
+ fprintf(outfile, "%i %i %i %i\n",fsm_->state_no, fsm_->in,
+ fsm_->target, fsm_->final_state);
}
} else {
- if (fsm->in != fsm->out) {
- fprintf(outfile, "%i %i %i\n", fsm->in, fsm->out, fsm->target);
+ if (fsm_->in != fsm_->out) {
+ fprintf(outfile, "%i %i %i\n", fsm_->in, fsm_->out, fsm_->target);
} else {
- fprintf(outfile, "%i %i\n", fsm->in, fsm->target);
+ fprintf(outfile, "%i %i\n", fsm_->in, fsm_->target);
}
}
- laststate = fsm->state_no;
+ laststate = fsm_->state_no;
}
/* Sentinel for states */
fprintf(outfile, "-1 -1 -1 -1 -1\n");
@@ -1129,11 +1141,11 @@ static inline int explode_line (char *buf, int *values) {
#include <iostream>
using namespace hfst::implementations;
-int main(int argc, char * argv[])
+int main(int argc, char * argv[])
{
std::cout << "Unit tests for " __FILE__ ":";
- fsm * epsilon
+ fsm * epsilon
= FomaTransducer::define_transducer("@_EPSILON_SYMBOL_@");
fsm * epsilon_i = FomaTransducer::extract_input_language(epsilon);
fsm * epsilon_i_min = FomaTransducer::minimize(fsm_copy(epsilon_i));
diff --git a/libhfst/src/implementations/FomaTransducer.h b/libhfst/src/implementations/FomaTransducer.h
index e9d72c2..c881cc0 100644
--- a/libhfst/src/implementations/FomaTransducer.h
+++ b/libhfst/src/implementations/FomaTransducer.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _FOMA_TRANSDUCER_H_
@@ -16,15 +16,17 @@
#include "HfstFlagDiacritics.h"
#include <stdlib.h>
-#ifndef _FOMALIB_H_
-#define _FOMALIB_H_
-#include "back-ends/foma/fomalib.h"
-#endif
+//#ifndef _FOMALIB_H_
+//#define _FOMALIB_H_
+//#include "back-ends/foma/fomalib.h"
+//#endif
+
+struct fsm;
#include <cstdio>
#include <string>
#include <sstream>
-#include <iostream>
+#include <iosfwd>
#include "../FormatSpecifiers.h"
@@ -39,7 +41,7 @@ namespace hfst {
using std::ostream;
using std::ostringstream;
- class FomaInputStream
+ class FomaInputStream
{
private:
std::string filename;
@@ -65,14 +67,14 @@ namespace hfst {
static bool is_fst(std::istream &s);
};
- class FomaOutputStream
+ class FomaOutputStream
{
private:
std::string filename;
FILE *ofile;
//void write_3_0_library_header(FILE *file);
public:
- FomaOutputStream(void);
+ FomaOutputStream(void);
FomaOutputStream(const std::string &filename);
void close(void);
void write(const char &c);
@@ -89,9 +91,9 @@ namespace hfst {
(const hfst::StringPairSet &sps, bool cyclic=false);
static fsm * define_transducer(const std::vector<StringPairSet> &spsv);
static fsm * define_transducer
- (const std::string &symbol);
+ (const std::string &symbol);
static fsm * define_transducer
- (const std::string &isymbol, const std::string &osymbol);
+ (const std::string &isymbol, const std::string &osymbol);
static fsm * copy(fsm * t);
static fsm * determinize(fsm * t);
static fsm * minimize(fsm * t);
@@ -126,7 +128,7 @@ namespace hfst {
fsm * t2);
static void extract_paths(fsm * t, hfst::ExtractStringsCb& callback,
- int cycles=-1, FdTable<int>* fd=NULL,
+ int cycles=-1, FdTable<int>* fd=NULL,
bool filter_fd=false);
static void extract_random_paths
(const fsm *t, HfstTwoLevelPaths &results, int max_num);
@@ -140,8 +142,8 @@ namespace hfst {
static void insert_to_alphabet(fsm *t, const std::string &symbol);
static void remove_from_alphabet(fsm *t, const std::string &symbol);
static StringSet get_alphabet(fsm *t);
- static unsigned int get_symbol_number(fsm *t,
- const std::string &symbol);
+ static unsigned int get_symbol_number(fsm *t,
+ const std::string &symbol);
static void harmonize(fsm *net1, fsm *net2);
diff --git a/libhfst/src/implementations/HfstTransitionGraph.h b/libhfst/src/implementations/HfstBasicTransducer.cc
similarity index 70%
copy from libhfst/src/implementations/HfstTransitionGraph.h
copy to libhfst/src/implementations/HfstBasicTransducer.cc
index 1b157db..d823b7e 100644
--- a/libhfst/src/implementations/HfstTransitionGraph.h
+++ b/libhfst/src/implementations/HfstBasicTransducer.cc
@@ -1,194 +1,25 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
- #ifndef _HFST_TRANSITION_GRAPH_H_
- #define _HFST_TRANSITION_GRAPH_H_
-
- /** @file HfstTransitionGraph.h
- @brief Class HfstTransitionGraph */
-
- #include <cstdio>
- #include <string>
- #include <set>
- #include <cassert>
- #include <iostream>
- #include <algorithm>
- #include <stack>
-
- #include "../HfstSymbolDefs.h"
- #include "../HfstExceptionDefs.h"
- #include "../HfstDataTypes.h"
- #include "../HarmonizeUnknownAndIdentitySymbols.h"
- #include "../HfstFlagDiacritics.h"
- #include "../HfstEpsilonHandler.h"
- #include "ConvertTransducerFormat.h"
- #include "HfstTransition.h"
- #include "HfstTropicalTransducerTransitionData.h"
-//#include "HfstFastTransitionData.h"
- #include "../string-utils.h"
-
- #include "../hfstdll.h"
+#include "HfstBasicTransducer.h"
- namespace hfst {
+#ifndef MAIN_TEST
- /** @brief A namespace for all code that forms a bridge between
- backend libraries and HFST.
+ namespace hfst {
- Most code in this namespace is not intended to be accessed
- by the user of the HFST interface. An exception is HFST's
- own transducer class hfst::HfstTransitionGraph and classes that
- are needed to use it.
- */
namespace implementations {
- /** @brief The number of a state in an HfstTransitionGraph. */
- typedef unsigned int HfstState;
-
- typedef std::pair<HfstState, std::vector<std::pair<std::string, std::string> > > HfstReplacement;
- typedef std::vector<HfstReplacement> HfstReplacements;
- typedef std::map<HfstState, HfstReplacements > HfstReplacementsMap;
-
- typedef std::vector<std::vector<hfst::implementations::HfstBasicTransition> > HfstBasicStates;
-
- /** @brief A simple transition graph format that consists of
- states and transitions between those states.
-
- Probably the easiest way to use this template is to choose
- the implementations #HfstBasicTransducer
- (HfstTransitionGraph<HfstTropicalTransducerTransitionData>)
- and #HfstBasicTransition
- (HfstTransition<HfstTropicalTransducerTransitionData>).
- The class HfstTropicalTransducerTransitionData contains an input string,
- an output string and a float weight. HfstBasicTransducer is the
- implementation that is used as an example in this documentation.
-
- An example of creating a HfstBasicTransducer [foo:bar baz:baz]
- with weight 0.4 from scratch:
-
- \verbatim
- // Create an empty transducer
- // The transducer has initially one start state (number zero)
- // that is not final
- HfstBasicTransducer fsm;
- // Add two states to the transducer
- fsm.add_state(1);
- fsm.add_state(2);
- // Create a transition [foo:bar] leading to state 1 with weight 0.1 ...
- HfstBasicTransition tr(1, "foo", "bar", 0.1);
- // ... and add it to state zero
- fsm.add_transition(0, tr);
- // Add a transition [baz:baz] with weight 0 from state 1 to state 2
- fsm.add_transition(1, HfstBasicTransition(2, "baz", "baz", 0.0));
- // Set state 2 as final with weight 0.3
- fsm.set_final_weight(2, 0.3);
- \endverbatim
-
- An example of iterating through a HfstBasicTransducer's states
- and transitions when printing it in AT&T format to stderr:
-
- \verbatim
- // The first state is always number zero.
- unsigned int source_state=0;
-
- // Go through all states
- for (HfstBasicTransducer::const_iterator it = fsm.begin();
- it != fsm.end(); it++ )
- {
- // Go through all transitions
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
- = it->begin(); tr_it != it->end(); tr_it++)
- {
- std::cerr << source_state << "\t"
- << tr_it->get_target_state() << "\t"
- << tr_it->get_input_symbol() << "\t"
- << tr_it->get_output_symbol() << "\t"
- << tr_it->get_weight() << std::endl;
- }
-
- if (fsm.is_final_state(source_state))
- {
- std::cerr << source_state << "\t"
- << fsm.get_final_weight(source_state) << std::endl;
- }
-
- // the next state is numbered source_state + 1
- source_state++;
- }
- \endverbatim
-
- @see #HfstBasicTransducer HfstBasicTransition */
- template <class C> class HfstTransitionGraph
- {
-
- // --- Datatypes and variables ---
-
- public:
- /** @brief Datatype for a symbol in a transition. */
- typedef typename C::SymbolType HfstSymbol;
- /** @brief Datatype for a symbol pair in a transition. */
- typedef std::pair<HfstSymbol, HfstSymbol>
- HfstSymbolPair;
- /** @brief A set of symbol pairs. */
- typedef std::set<HfstSymbolPair> HfstSymbolPairSet;
- /** @brief A set of symbol pairs. */
- typedef std::set<HfstSymbol> HfstSymbolSet;
- /** @brief A vector of symbol pairs. */
- typedef std::vector<HfstSymbolPair> HfstSymbolPairVector;
- /** @brief Datatype for the alphabet of a graph. */
- typedef std::set<HfstSymbol> HfstTransitionGraphAlphabet;
-
- /** @brief Datatype for the states of a transition in a graph. */
- typedef std::vector<HfstTransition<C> > HfstTransitions;
-
- /* Datatype for the states of a graph and their transitions.
- Each index of the vector is a state and the transitions
- on that index are the transitions of that state. */
- typedef std::vector<HfstTransitions> HfstStates;
-
- /* States of the graph and their transitions. */
- HfstStates state_vector;
-
- protected:
- /* The initial state number. */
- static const HfstState INITIAL_STATE = 0;
-
- /* Datatype for the final states and their weights in a graph. */
- typedef std::map<HfstState,typename C::WeightType> FinalWeightMap;
- /* The final states and their weights in the graph. */
- FinalWeightMap final_weight_map;
-
- /* The alphabet of the graph. */
- HfstTransitionGraphAlphabet alphabet;
-
- /* Used by substitute function. */
- typedef unsigned int HfstNumber;
- typedef std::vector<HfstNumber> HfstNumberVector;
- typedef std::pair<HfstNumber, HfstNumber> HfstNumberPair;
- typedef std::map<HfstNumberPair, HfstNumberPair> HfstNumberPairSubstitutions;
-
- protected:
- /* @brief An iterator type that points to a state in a graph.
-
- The value pointed by the iterator is of type HfstTransitions. */
- typedef typename HfstStates::iterator iterator;
-
- public:
- /** @brief A const iterator type that points a state in a graph.
-
- The value pointed by the iterator is of type HfstTransitions. */
- typedef typename HfstStates::const_iterator const_iterator;
-
- /** @brief The name of the graph. */
- std::string name;
+ // static const HfstState INITIAL_STATE = 0;
/** @brief The states of the graph. */
- std::vector<HfstState> states() const {
+ std::vector<HfstState> HfstBasicTransducer::states() const {
std::vector<HfstState> retval(this->get_max_state()+1, 0);
for (unsigned int i=0; i<(this->get_max_state()+1); i++)
retval[i] = i;
@@ -196,7 +27,11 @@
}
/** @brief The states of the graph and their transitions. */
- HfstBasicStates states_and_transitions() const {
+ const HfstBasicStates & HfstBasicTransducer::states_and_transitions() const {
+ return state_vector;
+ }
+
+ HfstBasicStates & HfstBasicTransducer::states_and_transitions() {
return state_vector;
}
@@ -206,23 +41,25 @@
/** @brief Create a graph with one initial state that has state number
zero and is not a final state, i.e. create an empty graph. */
- HFSTDLL HfstTransitionGraph(void) {
+ HfstBasicTransducer::HfstBasicTransducer(void) {
initialize_alphabet(alphabet);
- HfstTransitions tr;
+ HfstBasicTransitions tr;
state_vector.push_back(tr);
+ name = std::string("");
}
- HFSTDLL HfstTransitionGraph(FILE *file) {
+ HfstBasicTransducer::HfstBasicTransducer(FILE *file) {
initialize_alphabet(alphabet);
- HfstTransitions tr;
+ HfstBasicTransitions tr;
state_vector.push_back(tr);
unsigned int linecount=0;
this->assign(read_in_att_format(file, "@0@", linecount));
+ name = std::string("");
}
/** @brief The assignment operator. */
- HFSTDLL HfstTransitionGraph &operator=(const HfstTransitionGraph &graph)
+ HfstBasicTransducer & HfstBasicTransducer::operator=(const HfstBasicTransducer &graph)
{
if (this == &graph)
return *this;
@@ -230,26 +67,28 @@
final_weight_map = graph.final_weight_map;
alphabet = graph.alphabet;
assert(alphabet.count(HfstSymbol()) == 0);
+ name = graph.name;
return *this;
}
- HFSTDLL HfstTransitionGraph &assign(const HfstTransitionGraph &graph)
+ HfstBasicTransducer & HfstBasicTransducer::assign(const HfstBasicTransducer &graph)
{
return this->operator=(graph);
}
- /** @brief Create a deep copy of HfstTransitionGraph \a graph. */
- HFSTDLL HfstTransitionGraph(const HfstTransitionGraph &graph) {
+ /** @brief Create a deep copy of HfstBasicTransducer \a graph. */
+ HfstBasicTransducer::HfstBasicTransducer(const HfstBasicTransducer &graph) {
state_vector = graph.state_vector;
final_weight_map = graph.final_weight_map;
alphabet = graph.alphabet;
+ name = graph.name;
assert(alphabet.count(HfstSymbol()) == 0);
}
- /** @brief Create an HfstTransitionGraph equivalent to HfstTransducer
+ /** @brief Create an HfstBasicTransducer equivalent to HfstTransducer
\a transducer. FIXME: move to a separate file */
- HFSTDLL HfstTransitionGraph(const hfst::HfstTransducer &transducer) {
- HfstTransitionGraph<HfstTropicalTransducerTransitionData>
+ HfstBasicTransducer::HfstBasicTransducer(const hfst::HfstTransducer &transducer) {
+ HfstBasicTransducer
*fsm = ConversionFunctions::
hfst_transducer_to_hfst_basic_transducer(transducer);
state_vector = fsm->state_vector;
@@ -263,32 +102,32 @@
// --- Initialization, optimization and debugging ---
// --------------------------------------------------
- protected:
- /* Add epsilon, unknown and identity symbols to the alphabet
+
+ /* Add epsilon, unknown and identity symbols to the alphabet
\a alpha. */
- void initialize_alphabet(HfstTransitionGraphAlphabet &alpha) {
- alpha.insert(C::get_epsilon());
- alpha.insert(C::get_unknown());
- alpha.insert(C::get_identity());
+ void HfstBasicTransducer::initialize_alphabet(HfstBasicTransducer::HfstAlphabet &alpha) {
+ alpha.insert(HfstTropicalTransducerTransitionData::get_epsilon());
+ alpha.insert(HfstTropicalTransducerTransitionData::get_unknown());
+ alpha.insert(HfstTropicalTransducerTransitionData::get_identity());
}
/* Check that all symbols that occur in the transitions of the graph
are also in the alphabet. */
- bool check_alphabet()
+ bool HfstBasicTransducer::check_alphabet()
{
for (iterator it = begin(); it != end(); it++)
{
- for (typename HfstTransitions::iterator tr_it
+ for (HfstBasicTransitions::iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
- if(alphabet.find(data.get_input_symbol())
+ if(alphabet.find(data.get_input_symbol())
== alphabet.end()) {
return false;
}
- if(alphabet.find(data.get_output_symbol())
+ if(alphabet.find(data.get_output_symbol())
== alphabet.end()) {
return false;
}
@@ -297,11 +136,10 @@
return true;
}
- public:
/* Print the alphabet of the graph to standard error stream. */
- HFSTDLL void print_alphabet() const
+ void HfstBasicTransducer::print_alphabet() const
{
- for (typename HfstTransitionGraphAlphabet::const_iterator it
+ for (HfstBasicTransducer::HfstAlphabet::const_iterator it
= alphabet.begin(); it != alphabet.end(); it++)
{
if (it != alphabet.begin())
@@ -311,25 +149,25 @@
std::cerr << std::endl;
}
- protected:
+
/* Get the number of the \a symbol. */
- unsigned int get_symbol_number
+ unsigned int HfstBasicTransducer::get_symbol_number
(const HfstSymbol &symbol) const {
- return C::get_number(symbol);
+ return HfstTropicalTransducerTransitionData::get_number(symbol);
}
- /* For internal optimization: Reserve space for
+ /* For internal optimization: Reserve space for
\a number_of_states states. */
- void initialize_state_vector
+ void HfstBasicTransducer::initialize_state_vector
(unsigned int number_of_states)
{
state_vector.reserve(number_of_states);
}
/* For internal optimization: Reserve space for
- \a number_of_transitions transitions for state number
+ \a number_of_transitions transitions for state number
\a state_number. */
- void initialize_transition_vector
+ void HfstBasicTransducer::initialize_transition_vector
(unsigned int state_number, unsigned int number_of_transitions)
{
add_state(state_number);
@@ -341,25 +179,25 @@
// ---------- The alphabet -----------
// -----------------------------------
- public:
+
/** @brief Explicitly add \a symbol to the alphabet of the graph.
@note Usually the user does not have to take care of the alphabet
of a graph. This function can be useful in some special cases. */
- HFSTDLL void add_symbol_to_alphabet(const HfstSymbol &symbol) {
+ void HfstBasicTransducer::add_symbol_to_alphabet(const HfstSymbol &symbol) {
alphabet.insert(symbol);
}
- /** @brief Remove symbol \a symbol from the alphabet of the graph.
+ /** @brief Remove symbol \a symbol from the alphabet of the graph.
@note Use with care, removing symbols that occur in the transitions
of the graph can have unexpected results. */
- HFSTDLL void remove_symbol_from_alphabet(const HfstSymbol &symbol) {
+ void HfstBasicTransducer::remove_symbol_from_alphabet(const HfstSymbol &symbol) {
alphabet.erase(symbol);
}
- HFSTDLL void remove_symbols_from_alphabet(const HfstSymbolSet &symbols) {
- for (typename HfstSymbolSet::const_iterator it = symbols.begin();
+ void HfstBasicTransducer::remove_symbols_from_alphabet(const HfstSymbolSet &symbols) {
+ for (HfstSymbolSet::const_iterator it = symbols.begin();
it != symbols.end(); it++)
{
alphabet.erase(*it);
@@ -368,18 +206,18 @@
/** @brief Same as #add_symbol_to_alphabet for each symbol in
\a symbols. */
- HFSTDLL void add_symbols_to_alphabet(const HfstSymbolSet &symbols)
+ void HfstBasicTransducer::add_symbols_to_alphabet(const HfstSymbolSet &symbols)
{
- for (typename HfstSymbolSet::const_iterator it = symbols.begin();
+ for (HfstSymbolSet::const_iterator it = symbols.begin();
it != symbols.end(); it++)
{
alphabet.insert(*it);
}
}
- HFSTDLL void add_symbols_to_alphabet(const HfstSymbolPairSet &symbols)
+ void HfstBasicTransducer::add_symbols_to_alphabet(const HfstSymbolPairSet &symbols)
{
- for (typename HfstSymbolPairSet::const_iterator it = symbols.begin();
+ for (HfstSymbolPairSet::const_iterator it = symbols.begin();
it != symbols.end(); it++)
{
alphabet.insert(it->first);
@@ -387,25 +225,25 @@
}
}
- /* Remove all symbols that are given in \a symbols but do not occur
+ /* Remove all symbols that are given in \a symbols but do not occur
in transitions of the graph from its alphabet. */
- HFSTDLL void prune_alphabet_after_substitution(const std::set<unsigned int> &symbols)
+ void HfstBasicTransducer::prune_alphabet_after_substitution(const std::set<unsigned int> &symbols)
{
if (symbols.size() == 0)
return;
std::vector<bool> symbols_found;
symbols_found.resize
- (C::get_max_number()+1, false);
+ (HfstTropicalTransducerTransitionData::get_max_number()+1, false);
// Go through all transitions
for (iterator it = begin(); it != end(); it++)
{
- for (typename HfstTransitions::iterator tr_it
+ for (HfstBasicTransitions::iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
- const C & data = tr_it->get_transition_data();
+ const HfstTropicalTransducerTransitionData & data = tr_it->get_transition_data();
symbols_found.at(data.get_input_number()) = true;
symbols_found.at(data.get_output_number()) = true;
}
@@ -417,21 +255,21 @@
it != symbols.end(); it++)
{
if (! symbols_found.at(*it))
- alphabet.erase(C::get_symbol(*it));
+ alphabet.erase(HfstTropicalTransducerTransitionData::get_symbol(*it));
}
}
- HFSTDLL HfstTransitionGraphAlphabet symbols_used()
+ HfstBasicTransducer::HfstAlphabet HfstBasicTransducer::symbols_used()
{
- HfstTransitionGraphAlphabet retval;
+ HfstBasicTransducer::HfstAlphabet retval;
for (iterator it = begin(); it != end(); it++)
{
- for (typename HfstTransitions::iterator tr_it
+ for (HfstBasicTransitions::iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
retval.insert(data.get_input_symbol());
retval.insert(data.get_output_symbol());
@@ -441,20 +279,20 @@
}
/** @brief Remove all symbols that do not occur in transitions of
- the graph from its alphabet.
+ the graph from its alphabet.
@param force Whether unused symbols are removed even if
unknown or identity symbols occur in transitions.
Epsilon, unknown and identity \link hfst::String symbols\endlink
are always included in the alphabet. */
- HFSTDLL void prune_alphabet(bool force=true) {
+ void HfstBasicTransducer::prune_alphabet(bool force/*=true*/) {
// Which symbols occur in the graph
- HfstTransitionGraphAlphabet symbols_found = symbols_used();
+ HfstBasicTransducer::HfstAlphabet symbols_found = symbols_used();
// Whether unknown or identity symbols are used
- bool unknowns_or_identities_used =
+ bool unknowns_or_identities_used =
( (symbols_found.find("@_UNKNOWN_SYMBOL_@") != symbols_found.end()) ||
(symbols_found.find("@_IDENTITY_SYMBOL_@") != symbols_found.end()) );
@@ -464,17 +302,17 @@
return;
// Special symbols are always known
- symbols_found.insert("@_EPSILON_SYMBOL_@");
- symbols_found.insert("@_UNKNOWN_SYMBOL_@");
- symbols_found.insert("@_IDENTITY_SYMBOL_@");
+ symbols_found.insert("@_EPSILON_SYMBOL_@");
+ symbols_found.insert("@_UNKNOWN_SYMBOL_@");
+ symbols_found.insert("@_IDENTITY_SYMBOL_@");
- // Which symbols in the graph's alphabet did not occur in
+ // Which symbols in the graph's alphabet did not occur in
// the graph
- HfstTransitionGraphAlphabet symbols_not_found;
+ HfstBasicTransducer::HfstAlphabet symbols_not_found;
- for (typename HfstTransitionGraphAlphabet::iterator it
+ for (HfstBasicTransducer::HfstAlphabet::iterator it
= alphabet.begin();
- it != alphabet.end(); it++)
+ it != alphabet.end(); it++)
{
if (symbols_found.find(*it) == symbols_found.end())
symbols_not_found.insert(*it);
@@ -482,7 +320,7 @@
// Remove the symbols that did not occur in the graph
// from its alphabet
- for (typename HfstTransitionGraphAlphabet::iterator it
+ for (HfstBasicTransducer::HfstAlphabet::iterator it
= symbols_not_found.begin();
it != symbols_not_found.end(); it++)
{
@@ -490,26 +328,26 @@
}
}
- /** @brief Get the set of HfstSymbols in the alphabet
- of the graph.
+ /** @brief Get the set of HfstSymbols in the alphabet
+ of the graph.
The HfstSymbols do not necessarily occur in any transitions
- of the graph. Epsilon, unknown and identity \link
+ of the graph. Epsilon, unknown and identity \link
hfst::String symbols\endlink are always included in the alphabet. */
- HFSTDLL const HfstTransitionGraphAlphabet &get_alphabet() const {
+ const HfstBasicTransducer::HfstAlphabet & HfstBasicTransducer::get_alphabet() const {
return alphabet;
}
- HFSTDLL StringPairSet get_transition_pairs() const {
+ StringPairSet HfstBasicTransducer::get_transition_pairs() const {
StringPairSet retval;
for (const_iterator it = begin(); it != end(); it++)
{
- for (typename HfstTransitions::const_iterator tr_it
+ for (HfstBasicTransitions::const_iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
retval.insert(StringPair(data.get_input_symbol(),
data.get_output_symbol()));
@@ -526,10 +364,10 @@
/** @brief Add a new state to this graph and return its number.
@return The next (smallest) free state number. */
- HFSTDLL HfstState add_state(void) {
- HfstTransitions tr;
+ HfstState HfstBasicTransducer::add_state(void) {
+ HfstBasicTransitions tr;
state_vector.push_back(tr);
- return state_vector.size()-1;
+ return (HfstState)(state_vector.size()-1);
}
/** @brief Add a state \a s to this graph.
@@ -538,26 +376,26 @@
All states with state number smaller than \a s are also
added to the graph if they did not exist before.
@return \a s*/
- HFSTDLL HfstState add_state(HfstState s) {
+ HfstState HfstBasicTransducer::add_state(HfstState s) {
while(state_vector.size() <= s) {
- HfstTransitions tr;
+ HfstBasicTransitions tr;
state_vector.push_back(tr);
}
return s;
}
/** @brief Get the biggest state number in use. */
- HFSTDLL HfstState get_max_state() const {
- return state_vector.size()-1;
+ HfstState HfstBasicTransducer::get_max_state() const {
+ return (HfstState)(state_vector.size()-1);
}
- /** @brief Add a transition \a transition to state \a s.
+ /** @brief Add a transition \a transition to state \a s.
If state \a s does not exist, it is created. */
- HFSTDLL void add_transition(HfstState s, const HfstTransition<C> & transition,
- bool add_symbols_to_alphabet=true) {
+ void HfstBasicTransducer::add_transition(HfstState s, const HfstBasicTransition & transition,
+ bool add_symbols_to_alphabet/*=true*/) {
- C data = transition.get_transition_data();
+ HfstTropicalTransducerTransitionData data = transition.get_transition_data();
add_state(s);
add_state(transition.get_target_state());
@@ -574,22 +412,22 @@
if they are no longer used in the graph.
If \a state or \a transition does not exist, nothing is done. */
- HFSTDLL void remove_transition(HfstState s, const HfstTransition<C> & transition,
- bool remove_symbols_from_alphabet=false)
+ void HfstBasicTransducer::remove_transition(HfstState s, const HfstBasicTransition & transition,
+ bool remove_symbols_from_alphabet/*=false*/)
{
if (! (state_vector.size() > s))
{
return;
}
- HfstTransitions & transitions = state_vector[s];
+ HfstBasicTransitions & transitions = state_vector[s];
// iterators to transitions to be removed
// transitions must be removed in reverse order so that iterators
// are not invalidated
- std::stack<typename HfstTransitions::iterator> elements_to_remove;
+ std::stack<HfstBasicTransitions::iterator> elements_to_remove;
// find the transitions to be removed
- for (typename HfstTransitions::iterator it = transitions.begin();
+ for (HfstBasicTransitions::iterator it = transitions.begin();
it != transitions.end(); it++)
{
// weight is ignored
@@ -598,7 +436,7 @@
it->get_target_state() == transition.get_target_state())
{
// schedule transition to be removed
- elements_to_remove.push(it);
+ elements_to_remove.push(it);
}
}
// remove the transitions in reverse order
@@ -606,11 +444,11 @@
{
state_vector[s].erase(elements_to_remove.top());
elements_to_remove.pop();
- }
+ }
if (remove_symbols_from_alphabet)
{
- HfstTransitionGraphAlphabet alpha = this->symbols_used();
+ HfstBasicTransducer::HfstAlphabet alpha = this->symbols_used();
if (alpha.find(transition.get_input_symbol()) == alpha.end())
this->remove_symbol_from_alphabet(transition.get_input_symbol());
if (alpha.find(transition.get_output_symbol()) == alpha.end())
@@ -618,14 +456,14 @@
}
}
- /** @brief Whether state \a s is final.
+ /** @brief Whether state \a s is final.
FIXME: return positive infinity instead if not final. */
- HFSTDLL bool is_final_state(HfstState s) const {
+ bool HfstBasicTransducer::is_final_state(HfstState s) const {
return (final_weight_map.find(s) != final_weight_map.end());
}
/** Get the final weight of state \a s in this graph. */
- HFSTDLL typename C::WeightType get_final_weight(HfstState s) const {
+ HfstTropicalTransducerTransitionData::WeightType HfstBasicTransducer::get_final_weight(HfstState s) const {
if (s > this->get_max_state())
HFST_THROW(StateIndexOutOfBoundsException);
if (final_weight_map.find(s) != final_weight_map.end())
@@ -633,61 +471,61 @@
HFST_THROW(StateIsNotFinalException);
}
- /** @brief Set the final weight of state \a s in this graph
- to \a weight.
+ /** @brief Set the final weight of state \a s in this graph
+ to \a weight.
If the state does not exist, it is created. */
- HFSTDLL void set_final_weight(HfstState s,
- const typename C::WeightType & weight) {
+ void HfstBasicTransducer::set_final_weight(HfstState s,
+ const HfstTropicalTransducerTransitionData::WeightType & weight) {
add_state(s);
final_weight_map[s] = weight;
}
/** @brief Sort the arcs of this transducer according to input and
output symbols. */
- HFSTDLL HfstTransitionGraph &sort_arcs(void)
+ HfstBasicTransducer & HfstBasicTransducer::sort_arcs(void)
{
- for (typename HfstStates::iterator it = state_vector.begin();
+ for (HfstBasicStates::iterator it = state_vector.begin();
it != state_vector.end();
++it)
{
- HfstTransitions &transitions = *it;
- std::sort<typename HfstTransitions::iterator>
+ HfstBasicTransitions &transitions = *it;
+ std::sort<HfstBasicTransitions::iterator>
(transitions.begin(),transitions.end());
}
return *this;
}
- /** @brief Get an iterator to the beginning of the states in
- the graph.
+ /** @brief Get an iterator to the beginning of the states in
+ the graph.
For an example, see #HfstTransitionGraph */
- HFSTDLL iterator begin() { return state_vector.begin(); }
+ HfstBasicTransducer::iterator HfstBasicTransducer::begin() { return state_vector.begin(); }
- /** @brief Get a const iterator to the beginning of
+ /** @brief Get a const iterator to the beginning of
states in the graph. */
- HFSTDLL const_iterator begin() const { return state_vector.begin(); }
+ HfstBasicTransducer::const_iterator HfstBasicTransducer::begin() const { return state_vector.begin(); }
- /** @brief Get an iterator to the end of states (last state + 1)
+ /** @brief Get an iterator to the end of states (last state + 1)
in the graph. */
- HFSTDLL iterator end() { return state_vector.end(); }
+ HfstBasicTransducer::iterator HfstBasicTransducer::end() { return state_vector.end(); }
/** @brief Get a const iterator to the end of states (last state + 1)
in the graph. */
- HFSTDLL const_iterator end() const { return state_vector.end(); }
+ HfstBasicTransducer::const_iterator HfstBasicTransducer::end() const { return state_vector.end(); }
- /** @brief Get the set of transitions of state \a s in this graph.
+ /** @brief Get the set of transitions of state \a s in this graph.
If the state does not exist, a @a StateIndexOutOfBoundsException
is thrown.
*/
- HFSTDLL const HfstTransitions & operator[](HfstState s) const
+ const HfstBasicTransitions & HfstBasicTransducer::operator[](HfstState s) const
{
- if (s >= state_vector.size()) {
+ if (s >= state_vector.size()) {
HFST_THROW(StateIndexOutOfBoundsException); }
return state_vector[s];
- }
+ }
/** @brief Alternative name for operator[].
@@ -695,16 +533,16 @@
@see operator[]
*/
- HFSTDLL const HfstTransitions & transitions(HfstState s) const
+ const HfstBasicTransitions & HfstBasicTransducer::transitions(HfstState s) const
{
return this->operator[](s);
}
/** @brief Get mutable transitions.
*/
- HFSTDLL HfstTransitions & transitions(HfstState s)
+ HfstBasicTransitions & HfstBasicTransducer::transitions(HfstState s)
{
- if (s >= state_vector.size()) {
+ if (s >= state_vector.size()) {
HFST_THROW(StateIndexOutOfBoundsException); }
return state_vector[s];
}
@@ -713,11 +551,11 @@
// ----- Reading and writing in AT&T format -----
// --------------------------------------------------
- protected:
+
/* Change state numbers s1 to s2 and vice versa. */
- void swap_state_numbers(HfstState s1, HfstState s2) {
+ void HfstBasicTransducer::swap_state_numbers(HfstState s1, HfstState s2) {
- HfstTransitions s1_copy = state_vector[s1];
+ HfstBasicTransitions s1_copy = state_vector[s1];
state_vector[s1] = state_vector[s2];
state_vector[s2] = s1_copy;
@@ -727,7 +565,7 @@
// Go through all transitions
for (unsigned int i=0; i < it->size(); i++)
{
- HfstTransition<C> &tr_it = it->operator[](i);
+ HfstBasicTransition &tr_it = it->operator[](i);
HfstState new_target=tr_it.get_target_state();
if (tr_it.get_target_state() == s1)
@@ -737,7 +575,7 @@
if (new_target != tr_it.get_target_state())
{
- HfstTransition<C> tr
+ HfstBasicTransition tr
(new_target,
tr_it.get_input_symbol(),
tr_it.get_output_symbol(),
@@ -751,22 +589,22 @@
} // ----- all states gone through -----
// Swap final states, if needed
- typename FinalWeightMap::iterator s1_it = final_weight_map.find(s1);
- typename FinalWeightMap::iterator s2_it = final_weight_map.find(s2);
- typename FinalWeightMap::iterator end_it = final_weight_map.end();
+ FinalWeightMap::iterator s1_it = final_weight_map.find(s1);
+ FinalWeightMap::iterator s2_it = final_weight_map.find(s2);
+ FinalWeightMap::iterator end_it = final_weight_map.end();
if (s1_it != end_it && s2_it != end_it) {
- typename C::WeightType s1_weight = s1_it->second;
+ HfstTropicalTransducerTransitionData::WeightType s1_weight = s1_it->second;
final_weight_map[s1] = s2_it->second;
final_weight_map[s2] = s1_weight;
}
if (s1_it != end_it) {
- typename C::WeightType w = s1_it->second;
+ HfstTropicalTransducerTransitionData::WeightType w = s1_it->second;
final_weight_map.erase(s1);
final_weight_map[s2] = w;
}
if (s2_it != end_it) {
- typename C::WeightType w = s2_it->second;
+ HfstTropicalTransducerTransitionData::WeightType w = s2_it->second;
final_weight_map.erase(s2);
final_weight_map[s1] = w;
}
@@ -775,23 +613,23 @@
}
- static void write_weight(FILE * file, float weight)
+ void HfstBasicTransducer::write_weight(FILE * file, float weight)
{
//if (weight == 0) // avoid unnecessary 0.000000's
- // fprintf(file, "%i", 0);
+ // fprintf(file, "%i", 0);
//else
fprintf(file, "%f", weight);
}
- static void write_weight(std::ostream & os, float weight)
+ void HfstBasicTransducer::write_weight(std::ostream & os, float weight)
{
//if (weight == 0) // avoid unnecessary 0.000000's
- // os << 0;
+ // os << 0;
//else
os << weight;
}
- static void xfstize(std::string & symbol)
+ void HfstBasicTransducer::xfstize(std::string & symbol)
{
std::string escaped_symbol;
for (size_t pos = 0; pos < symbol.size(); pos++)
@@ -808,7 +646,7 @@
symbol = escaped_symbol;
}
- static void xfstize_symbol(std::string & symbol)
+ void HfstBasicTransducer::xfstize_symbol(std::string & symbol)
{
xfstize(symbol);
replace_all(symbol, "@_EPSILON_SYMBOL_@", "0");
@@ -817,28 +655,28 @@
replace_all(symbol, "\t", "@_TAB_@");
}
- void print_xfst_state(std::ostream & os, HfstState state)
+ void HfstBasicTransducer::print_xfst_state(std::ostream & os, HfstState state)
{
if (state == INITIAL_STATE) { os << "S"; }
if (is_final_state(state)) { os << "f"; }
os << "s" << state;
}
- void print_xfst_state(FILE * file, HfstState state)
+ void HfstBasicTransducer::print_xfst_state(FILE * file, HfstState state)
{
if (state == INITIAL_STATE) { fprintf(file, "S"); }
if (is_final_state(state)) { fprintf(file, "f"); }
fprintf(file, "s%i", state);
}
- void print_xfst_arc(std::ostream & os, C data)
+ void HfstBasicTransducer::print_xfst_arc(std::ostream & os, HfstTropicalTransducerTransitionData data)
{
// replace all spaces, epsilons and tabs
if (data.get_input_symbol() !=
- data.get_output_symbol())
+ data.get_output_symbol())
{
os << "<";
- }
+ }
std::string s = data.get_input_symbol();
xfstize_symbol(s);
os << s;
@@ -848,7 +686,7 @@
{
s = data.get_output_symbol();
xfstize_symbol(s);
- os << ":" << s;
+ os << ":" << s;
}
if (data.get_input_symbol() !=
data.get_output_symbol())
@@ -857,7 +695,7 @@
}
}
- void print_xfst_arc(FILE * file, C data)
+ void HfstBasicTransducer::print_xfst_arc(FILE * file, HfstTropicalTransducerTransitionData data)
{
if (data.get_input_symbol() !=
data.get_output_symbol())
@@ -884,11 +722,11 @@
}
}
- public:
+
/** @brief Write the graph in xfst text format to ostream \a os.
\a write_weights defines whether weights are printed (todo). */
- HFSTDLL void write_in_xfst_format(std::ostream &os, bool write_weights=true)
+ void HfstBasicTransducer::write_in_xfst_format(std::ostream &os, bool write_weights/*=true*/)
{
(void)write_weights; // todo
unsigned int source_state=0;
@@ -903,7 +741,7 @@
}
else
{
- for (typename HfstTransitions::iterator tr_it
+ for (HfstBasicTransitions::iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
@@ -911,7 +749,7 @@
{
os << ", ";
}
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
print_xfst_arc(os, data);
os << " -> ";
@@ -920,11 +758,11 @@
}
os << "." << std::endl;
source_state++;
- }
+ }
}
// note: unknown and identity are both '?'
- HFSTDLL static std::string prologize_symbol(const std::string & symbol)
+ std::string HfstBasicTransducer::prologize_symbol(const std::string & symbol)
{
if (symbol == "0")
return "%0";
@@ -944,7 +782,7 @@
}
// caveat: '?' is always unknown
- HFSTDLL static std::string deprologize_symbol(const std::string & symbol)
+ std::string HfstBasicTransducer::deprologize_symbol(const std::string & symbol)
{
if (symbol == "%0")
return "0";
@@ -962,13 +800,13 @@
return retval;
}
- HFSTDLL static void print_prolog_arc_symbols(FILE * file, C data)
+ void HfstBasicTransducer::print_prolog_arc_symbols(FILE * file, HfstTropicalTransducerTransitionData data)
{
std::string symbol = prologize_symbol(data.get_input_symbol());
fprintf(file, "\"%s\"", symbol.c_str());
if (data.get_input_symbol() !=
- data.get_output_symbol() ||
+ data.get_output_symbol() ||
data.get_input_symbol() == "@_UNKNOWN_SYMBOL_@")
{
symbol = prologize_symbol(data.get_output_symbol());
@@ -976,13 +814,13 @@
}
}
- HFSTDLL static void print_prolog_arc_symbols(std::ostream & os, C data)
+ void HfstBasicTransducer::print_prolog_arc_symbols(std::ostream & os, HfstTropicalTransducerTransitionData data)
{
std::string symbol = prologize_symbol(data.get_input_symbol());
os << "\"" << symbol << "\"";
if (data.get_input_symbol() !=
- data.get_output_symbol() ||
+ data.get_output_symbol() ||
data.get_input_symbol() == "@_UNKNOWN_SYMBOL_@")
{
symbol = prologize_symbol(data.get_output_symbol());
@@ -992,8 +830,8 @@
/** @brief Write the graph in prolog format to FILE \a file.
\a write_weights defines whether weights are printed (todo). */
- HFSTDLL void write_in_prolog_format(FILE * file, const std::string & name,
- bool write_weights=true)
+ void HfstBasicTransducer::write_in_prolog_format(FILE * file, const std::string & name,
+ bool write_weights/*=true*/)
{
unsigned int source_state=0;
const char * identifier = name.c_str();
@@ -1006,9 +844,9 @@
fprintf(file, "network(%s).\n", identifier);
// Print symbols that are in the alphabet but not used in arcs.
- HfstTransitionGraphAlphabet symbols_used_ = symbols_used();
+ HfstBasicTransducer::HfstAlphabet symbols_used_ = symbols_used();
initialize_alphabet(symbols_used_); // exclude special symbols
- for (typename HfstTransitionGraphAlphabet::const_iterator it
+ for (HfstBasicTransducer::HfstAlphabet::const_iterator it
= alphabet.begin(); it != alphabet.end(); it++)
{
if (symbols_used_.find(*it) == symbols_used_.end())
@@ -1020,13 +858,13 @@
// Print arcs.
for (iterator it = begin(); it != end(); it++)
{
- for (typename HfstTransitions::iterator tr_it
+ for (HfstBasicTransitions::iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
fprintf(file, "arc(%s, %i, %i, ",
identifier, source_state, tr_it->get_target_state());
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
print_prolog_arc_symbols(file, data);
if (write_weights) {
fprintf(file, ", ");
@@ -1038,12 +876,12 @@
}
// Print final states.
- for (typename FinalWeightMap::const_iterator it
+ for (FinalWeightMap::const_iterator it
= this->final_weight_map.begin();
it != this->final_weight_map.end(); it++)
{
fprintf(file, "final(%s, %i", identifier, it->first);
- if (write_weights)
+ if (write_weights)
{
fprintf(file, ", ");
write_weight(file, it->second);
@@ -1054,8 +892,8 @@
/** @brief Write the graph in prolog format to ostream \a os.
\a write_weights defines whether weights are printed (todo). */
- HFSTDLL void write_in_prolog_format(std::ostream & os, const std::string & name,
- bool write_weights=true)
+ void HfstBasicTransducer::write_in_prolog_format(std::ostream & os, const std::string & name,
+ bool write_weights/*=true*/)
{
unsigned int source_state=0;
@@ -1068,9 +906,9 @@
os << "network(" << name << ")." << std::endl;
// Print symbols that are in the alphabet but not used in arcs.
- HfstTransitionGraphAlphabet symbols_used_ = symbols_used();
+ HfstBasicTransducer::HfstAlphabet symbols_used_ = symbols_used();
initialize_alphabet(symbols_used_); // exclude special symbols
- for (typename HfstTransitionGraphAlphabet::const_iterator it
+ for (HfstBasicTransducer::HfstAlphabet::const_iterator it
= alphabet.begin(); it != alphabet.end(); it++)
{
if (symbols_used_.find(*it) == symbols_used_.end())
@@ -1082,16 +920,16 @@
// Print arcs.
for (iterator it = begin(); it != end(); it++)
{
- for (typename HfstTransitions::iterator tr_it
+ for (HfstBasicTransitions::iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
os << "arc(" << name << ", " << source_state << ", " << tr_it->get_target_state() << ", ";
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
print_prolog_arc_symbols(os, data);
if (write_weights) {
os << ", ";
- write_weight(os, data.get_weight());
+ write_weight(os, data.get_weight());
}
os << ")." << std::endl;
}
@@ -1099,7 +937,7 @@
}
// Print final states.
- for (typename FinalWeightMap::const_iterator it
+ for (FinalWeightMap::const_iterator it
= this->final_weight_map.begin();
it != this->final_weight_map.end(); it++)
{
@@ -1114,7 +952,7 @@
// If \a str is of format ".+", change it to .+ and return true.
// Else, return false.
- HFSTDLL static bool strip_quotes_from_both_sides(std::string & str)
+ bool HfstBasicTransducer::strip_quotes_from_both_sides(std::string & str)
{
if (str.size() < 3)
return false;
@@ -1127,7 +965,7 @@
// If \a str is of format .+)\.", change it to .+ and return true.
// Else, return false.
- HFSTDLL static bool strip_ending_parenthesis_and_comma(std::string & str)
+ bool HfstBasicTransducer::strip_ending_parenthesis_and_comma(std::string & str)
{
if (str.size() < 3)
return false;
@@ -1137,7 +975,7 @@
return true;
}
- HFSTDLL static bool parse_prolog_network_line(const std::string & line, std::string & name)
+ bool HfstBasicTransducer::parse_prolog_network_line(const std::string & line, HfstBasicTransducer & graph)
{
// 'network(NAME).'
char namearr[100];
@@ -1150,17 +988,17 @@
if (!strip_ending_parenthesis_and_comma(namestr))
return false;
- name = namestr;
+ graph.name = namestr;
return true;
}
// Get positions of \a c in \a str. If \a esc is precedes
// \a c, \a c is not included.
- HFSTDLL static std::vector<unsigned int> get_positions_of_unescaped_char
+ std::vector<unsigned int> HfstBasicTransducer::get_positions_of_unescaped_char
(const std::string & str, char c, char esc)
{
std::vector<unsigned int> retval;
- for (size_t i=0; i < str.length(); i++)
+ for (unsigned int i=0; i < (unsigned int)str.length(); i++)
{
if (str[i] == c)
{
@@ -1175,11 +1013,11 @@
return retval;
}
- // Extract input and output symbols, if possible, from prolog arc
- // \a str and store them to \a isymbol and \a osymbol.
+ // Extract input and output symbols, if possible, from prolog arc
+ // \a str and store them to \a isymbol and \a osymbol.
// Return whether symbols were successfully extracted.
// \a str must be of format "foo":"bar" or "foo"
- HFSTDLL static bool get_prolog_arc_symbols
+ bool HfstBasicTransducer::get_prolog_arc_symbols
(const std::string & str, std::string & isymbol, std::string & osymbol)
{
// find positions of non-escaped double quotes (todo: double double-quote?)
@@ -1197,7 +1035,7 @@
else if (quote_positions.size() == 4)
{
if (quote_positions[0] != 0 ||
- quote_positions[3] != str.length()-1)
+ quote_positions[3] != str.length()-1)
{
return false; // extra characters outside quotes
}
@@ -1239,7 +1077,7 @@
return true;
}
- HFSTDLL static bool extract_weight(std::string & symbol, float & weight)
+ bool HfstBasicTransducer::extract_weight(std::string & symbol, float & weight)
{
size_t last_double_quote = symbol.find_last_of('"');
size_t last_space = symbol.find_last_of(' ');
@@ -1249,7 +1087,7 @@
{ return false; }
if (last_space == std::string::npos) {
- ; // no weight
+ ; // no weight
}
else if (last_double_quote > last_space) {
; // no weight, last space is part of a symbol
@@ -1268,19 +1106,19 @@
return true;
}
- HFSTDLL static bool parse_prolog_arc_line(const std::string & line, HfstTransitionGraph & graph)
+ bool HfstBasicTransducer::parse_prolog_arc_line(const std::string & line, HfstBasicTransducer & graph)
{
// symbolstr can also contain the weight
char namestr[100]; char sourcestr[100];
char targetstr[100]; char symbolstr[100];
- int n = sscanf(line.c_str(), "arc(%[^,], %[^,], %[^,], %[^\t\n]",
+ int n = sscanf(line.c_str(), "arc(%[^,], %[^,], %[^,], %[^\t\n]",
namestr, sourcestr, targetstr, symbolstr);
std::string symbol(symbolstr);
// strip the ending ")." from symbolstr
- if (!strip_ending_parenthesis_and_comma(symbol))
+ if (!strip_ending_parenthesis_and_comma(symbol))
{ return false; }
if (n != 4)
@@ -1302,11 +1140,11 @@
if (!get_prolog_arc_symbols(symbol, isymbol, osymbol))
return false;
- graph.add_transition(source, HfstTransition<C>(target, isymbol, osymbol, weight));
+ graph.add_transition(source, HfstBasicTransition(target, isymbol, osymbol, weight));
return true;
}
- HFSTDLL static bool parse_prolog_final_line(const std::string & line, HfstTransitionGraph & graph)
+ bool HfstBasicTransducer::parse_prolog_final_line(const std::string & line, HfstBasicTransducer & graph)
{
// 'final(NAME, number).' or 'final(NAME, number, weight).'
char namestr[100];
@@ -1350,7 +1188,7 @@
return true;
}
- HFSTDLL static bool parse_prolog_symbol_line(const std::string & line, HfstTransitionGraph & graph)
+ bool HfstBasicTransducer::parse_prolog_symbol_line(const std::string & line, HfstBasicTransducer & graph)
{
// 'symbol(NAME, "foo").'
char namearr[100];
@@ -1377,7 +1215,7 @@
}
// Erase newlines from the end of \a str and return \a str.
- HFSTDLL static std::string strip_newlines(std::string & str)
+ std::string HfstBasicTransducer::strip_newlines(std::string & str)
{
for (signed int i=(signed int)str.length()-1; i >= 0; --i)
{
@@ -1393,7 +1231,7 @@
// or from \a file. If successfull, strip the line from newlines,
// increment \a linecount by one and return the line.
// Else, throw an EndOfStreamException.
- HFSTDLL static std::string get_stripped_line
+ std::string HfstBasicTransducer::get_stripped_line
(std::istream & is, FILE * file, unsigned int & linecount)
{
char line [255];
@@ -1402,40 +1240,41 @@
if (! is.getline(line,255).eof())
HFST_THROW(EndOfStreamException);
}
- else { /* we use FILEs */
+ else { /* we use FILEs */
if (NULL == fgets(line, 255, file))
HFST_THROW(EndOfStreamException);
}
linecount++;
std::string linestr(line);
- return strip_newlines(linestr);
+ return strip_newlines(linestr);
}
- /* Create an HfstTransitionGraph as defined in prolog format
+ /* Create an HfstTransitionGraph as defined in prolog format
in istream \a is or FILE \a file.
- The functions is called by functions
+ The functions is called by functions
read_in_prolog_format(istream&) and
- read_in_prolog_format(FILE*).
+ read_in_prolog_format(FILE*).
If \a file is NULL, it is ignored and \a is is used.
If \a file is not NULL, it is used and \a is is ignored. */
- HFSTDLL static HfstTransitionGraph read_in_prolog_format
- (std::istream &is, FILE *file, unsigned int & linecount)
+ HfstBasicTransducer HfstBasicTransducer::read_in_prolog_format
+ (std::istream &is, FILE *file, unsigned int & linecount)
{
- HfstTransitionGraph retval;
+ HfstBasicTransducer retval;
std::string linestr;
while(true)
{
- try
+ try
{
linestr = get_stripped_line(is, file, linecount);
- }
- catch (const EndOfStreamException & e)
+ }
+ catch (const EndOfStreamException & e)
{
- HFST_THROW(NotValidPrologFormatException);
+ (void)e;
+ HFST_THROW(NotValidPrologFormatException);
}
if (linestr.length() != 0 && linestr[0] == '#')
@@ -1449,7 +1288,7 @@
}
- if (! parse_prolog_network_line(linestr, retval.name))
+ if (! parse_prolog_network_line(linestr, retval))
{
std::string message("first line not valid prolog: ");
message.append(linestr);
@@ -1458,16 +1297,17 @@
while(true)
{
- try
+ try
{
linestr = get_stripped_line(is, file, linecount);
- if (linestr == "") // prolog separator
+ if (linestr == "") // prolog separator
{
return retval;
}
- }
- catch (const EndOfStreamException & e)
+ }
+ catch (const EndOfStreamException & e)
{
+ (void)e;
return retval;
}
@@ -1483,28 +1323,28 @@
HFST_THROW(NotValidPrologFormatException); // this should not happen
}
- HFSTDLL static HfstTransitionGraph read_in_prolog_format
+ HfstBasicTransducer HfstBasicTransducer::read_in_prolog_format
(std::istream &is,
- unsigned int & linecount)
+ unsigned int & linecount)
{
return read_in_prolog_format
(is, NULL /* a dummy variable */,
linecount);
}
- HFSTDLL static HfstTransitionGraph read_in_prolog_format
- (FILE *file,
- unsigned int & linecount)
+ HfstBasicTransducer HfstBasicTransducer::read_in_prolog_format
+ (FILE *file,
+ unsigned int & linecount)
{
return read_in_prolog_format
(std::cin /* a dummy variable */,
file, linecount);
- }
+ }
/** @brief Write the graph in xfst text format to FILE \a file.
\a write_weights defines whether weights are printed (todo). */
- HFSTDLL void write_in_xfst_format(FILE * file, bool write_weights=true)
+ void HfstBasicTransducer::write_in_xfst_format(FILE * file, bool write_weights/*=true*/)
{
(void)write_weights;
unsigned int source_state=0;
@@ -1519,7 +1359,7 @@
}
else
{
- for (typename HfstTransitions::iterator tr_it
+ for (HfstBasicTransitions::iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
@@ -1527,7 +1367,7 @@
{
fprintf(file, ", ");
}
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
print_xfst_arc(file, data);
@@ -1537,7 +1377,7 @@
}
fprintf(file, ".\n");
source_state++;
- }
+ }
}
@@ -1545,16 +1385,16 @@
/** @brief Write the graph in AT&T format to ostream \a os.
\a write_weights defines whether weights are printed. */
- HFSTDLL void write_in_att_format(std::ostream &os, bool write_weights=true)
+ void HfstBasicTransducer::write_in_att_format(std::ostream &os, bool write_weights/*=true*/)
{
unsigned int source_state=0;
for (iterator it = begin(); it != end(); it++)
{
- for (typename HfstTransitions::iterator tr_it
+ for (HfstBasicTransitions::iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
std::string isymbol = data.get_input_symbol();
replace_all(isymbol, " ", "@_SPACE_@");
@@ -1566,7 +1406,7 @@
replace_all(osymbol, "@_EPSILON_SYMBOL_@", "@0@");
replace_all(osymbol, "\t", "@_TAB_@");
- os << source_state << "\t"
+ os << source_state << "\t"
<< tr_it->get_target_state() << "\t"
<< isymbol << "\t"
<< osymbol;
@@ -1587,21 +1427,21 @@
os << "\n";
}
source_state++;
- }
+ }
}
/** @brief Write the graph in AT&T format to FILE \a file.
\a write_weights defines whether weights are printed. */
- HFSTDLL void write_in_att_format(FILE *file, bool write_weights=true)
+ void HfstBasicTransducer::write_in_att_format(FILE *file, bool write_weights/*=true*/)
{
unsigned int source_state=0;
for (iterator it = begin(); it != end(); it++)
{
- for (typename HfstTransitions::iterator tr_it
+ for (HfstBasicTransitions::iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
std::string isymbol = data.get_input_symbol();
replace_all(isymbol, " ", "@_SPACE_@");
@@ -1622,34 +1462,34 @@
if (write_weights) {
fprintf(file, "\t");
write_weight(file, data.get_weight());
- }
+ }
fprintf(file, "\n");
}
if (is_final_state(source_state))
{
fprintf(file, "%i", source_state);
if (write_weights) {
- fprintf(file, "\t");
+ fprintf(file, "\t");
write_weight(file, get_final_weight(source_state));
}
fprintf(file, "\n");
}
source_state++;
- }
+ }
}
- HFSTDLL void write_in_att_format(char * ptr, bool write_weights=true)
+ void HfstBasicTransducer::write_in_att_format(char * ptr, bool write_weights/*=true*/)
{
unsigned int source_state=0;
size_t cwt = 0; // characters written in total
size_t cw = 0; // characters written in latest call to sprintf
for (iterator it = begin(); it != end(); it++)
{
- for (typename HfstTransitions::iterator tr_it
+ for (HfstBasicTransitions::iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
std::string isymbol = data.get_input_symbol();
replace_all(isymbol, " ", "@_SPACE_@");
@@ -1681,30 +1521,30 @@
cw = sprintf(ptr + cwt, "%i", source_state);
cwt = cwt + cw;
if (write_weights)
- cw = sprintf(ptr + cwt, "\t%f",
+ cw = sprintf(ptr + cwt, "\t%f",
get_final_weight(source_state));
cwt = cwt + cw;
cw = sprintf(ptr + cwt, "\n");
cwt = cwt + cw;
}
source_state++;
- }
+ }
}
/** @brief Write the graph in AT&T format to FILE \a file using numbers
instead of symbol names.
\a write_weights defines whether weights are printed. */
- HFSTDLL void write_in_att_format_number(FILE *file, bool write_weights=true)
+ void HfstBasicTransducer::write_in_att_format_number(FILE *file, bool write_weights/*=true*/)
{
unsigned int source_state=0;
for (iterator it = begin(); it != end(); it++)
{
- for (typename HfstTransitions::iterator tr_it
+ for (HfstBasicTransitions::iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
fprintf(file, "%i\t%i\t%i\t%i",
source_state,
@@ -1714,14 +1554,14 @@
if (write_weights)
fprintf(file, "\t%f",
- data.get_weight());
+ data.get_weight());
fprintf(file, "\n");
if (is_final_state(source_state))
{
fprintf(file, "%i", source_state);
if (write_weights)
- fprintf(file, "\t%f",
+ fprintf(file, "\t%f",
get_final_weight(source_state));
fprintf(file, "\n");
}
@@ -1731,16 +1571,72 @@
}
- /* Create an HfstTransitionGraph as defined in AT&T format
+ bool HfstBasicTransducer::add_att_line(char * line, const std::string & epsilon_symbol)
+ {
+ // scan one line that can have a maximum of five fields
+ char a1 [100]; char a2 [100]; char a3 [100];
+ char a4 [100]; char a5 [100];
+ // how many fields could be parsed
+ int n = sscanf(line, "%s%s%s%s%s", a1, a2, a3, a4, a5);
+
+ // set value of weight
+ float weight = 0;
+ if (n == 2) // a final state line with weight
+ {
+ weight = hfst::double_to_float(atof(a2));
+ }
+ if (n == 5) // a transition line with weight
+ {
+ weight = hfst::double_to_float(atof(a5));
+ }
+
+ if (n == 1 || n == 2) // a final state line
+ {
+ set_final_weight( atoi(a1), weight );
+ }
+
+ else if (n == 4 || n == 5) { // a transition line
+ std::string input_symbol=std::string(a3);
+ std::string output_symbol=std::string(a4);
+
+ // replace "@_SPACE_@"s with " " and "@0@"s with
+ // "@_EPSILON_SYMBOL_@"
+ replace_all(input_symbol, "@_SPACE_@", " ");
+ replace_all(input_symbol, "@0@", "@_EPSILON_SYMBOL_@");
+ replace_all(input_symbol, "@_TAB_@", "\t");
+ replace_all(input_symbol, "@_COLON_@", ":");
+
+ replace_all(output_symbol, "@_SPACE_@", " ");
+ replace_all(output_symbol, "@0@", "@_EPSILON_SYMBOL_@");
+ replace_all(output_symbol, "@_TAB_@", "\t");
+ replace_all(output_symbol, "@_COLON_@", ":");
+
+ if (epsilon_symbol.compare(input_symbol) == 0)
+ input_symbol="@_EPSILON_SYMBOL_@";
+ if (epsilon_symbol.compare(output_symbol) == 0)
+ output_symbol="@_EPSILON_SYMBOL_@";
+
+ HfstBasicTransition tr( atoi(a2), input_symbol,
+ output_symbol, weight );
+ add_transition( atoi(a1), tr );
+ }
+
+ else { // line could not be parsed
+ return false;
+ }
+ return true;
+ }
+
+ /* Create an HfstTransitionGraph as defined in AT&T format
in istream \a is or FILE \a file. \a epsilon_symbol defines
- how epsilon is represented.
+ how epsilon is represented.
- The functions is called by functions
+ The functions is called by functions
read_in_att_format(istream&, std::string) and
- read_in_att_format(FILE*, std::string).
+ read_in_att_format(FILE*, std::string).
If \a file is NULL, it is ignored and \a is is used.
If \a file is not NULL, it is used and \a is is ignored. */
- HFSTDLL static HfstTransitionGraph read_in_att_format
+ HfstBasicTransducer HfstBasicTransducer::read_in_att_format
(std::istream &is,
FILE *file,
std::string epsilon_symbol,
@@ -1757,7 +1653,7 @@
}
}
- HfstTransitionGraph retval;
+ HfstBasicTransducer retval;
char line [255];
while(true) {
@@ -1765,7 +1661,7 @@
if (! is.getline(line,255).eof())
break;
}
- else { /* we use FILEs */
+ else { /* we use FILEs */
if (NULL == fgets(line, 255, file))
break;
}
@@ -1775,7 +1671,7 @@
// an empty line signifying an empty transducer,
// a special case that is accepted if it is the only
// transducer in the stream
- if ( // empty line with or without a newline
+ if ( // empty line with or without a newline
(line[0] == '\0') ||
(line[0] == '\n' && line[1] == '\0') ||
// windows newline
@@ -1793,104 +1689,63 @@
if (*line == '-') // transducer separator line is "--"
return retval;
- // scan one line that can have a maximum of five fields
- char a1 [100]; char a2 [100]; char a3 [100];
- char a4 [100]; char a5 [100];
- // how many fields could be parsed
- int n = sscanf(line, "%s%s%s%s%s", a1, a2, a3, a4, a5);
-
- // set value of weight
- float weight = 0;
- if (n == 2) // a final state line with weight
- weight = atof(a2);
- if (n == 5) // a transition line with weight
- weight = atof(a5);
-
- if (n == 1 || n == 2) // a final state line
- retval.set_final_weight( atoi(a1), weight );
-
- else if (n == 4 || n == 5) { // a transition line
- std::string input_symbol=std::string(a3);
- std::string output_symbol=std::string(a4);
-
- // replace "@_SPACE_@"s with " " and "@0@"s with
- // "@_EPSILON_SYMBOL_@"
- replace_all(input_symbol, "@_SPACE_@", " ");
- replace_all(input_symbol, "@0@", "@_EPSILON_SYMBOL_@");
- replace_all(input_symbol, "@_TAB_@", "\t");
- replace_all(input_symbol, "@_COLON_@", ":");
-
- replace_all(output_symbol, "@_SPACE_@", " ");
- replace_all(output_symbol, "@0@", "@_EPSILON_SYMBOL_@");
- replace_all(output_symbol, "@_TAB_@", "\t");
- replace_all(output_symbol, "@_COLON_@", ":");
-
- if (epsilon_symbol.compare(input_symbol) == 0)
- input_symbol="@_EPSILON_SYMBOL_@";
- if (epsilon_symbol.compare(output_symbol) == 0)
- output_symbol="@_EPSILON_SYMBOL_@";
-
- HfstTransition <C> tr( atoi(a2), input_symbol,
- output_symbol, weight );
- retval.add_transition( atoi(a1), tr );
- }
-
- else { // line could not be parsed
- std::string message(line);
- HFST_THROW_MESSAGE
- (NotValidAttFormatException,
- message);
- }
+ if (! retval.add_att_line(line, epsilon_symbol))
+ {
+ std::string message(line);
+ HFST_THROW_MESSAGE
+ (NotValidAttFormatException,
+ message);
+ }
}
return retval;
}
- /** @brief Create an HfstTransitionGraph as defined in AT&T
- transducer format in istream \a is. \a epsilon_symbol
- defines how epsilon is represented.
- @pre \a is not at end, otherwise an exception is thrown.
- @note Multiple AT&T transducer definitions are separated with
+ /** @brief Create an HfstTransitionGraph as defined in AT&T
+ transducer format in istream \a is. \a epsilon_symbol
+ defines how epsilon is represented.
+ @pre \a is not at end, otherwise an exception is thrown.
+ @note Multiple AT&T transducer definitions are separated with
the line "--". */
- HFSTDLL static HfstTransitionGraph read_in_att_format
+ HfstBasicTransducer HfstBasicTransducer::read_in_att_format
(std::istream &is,
std::string epsilon_symbol,
- unsigned int & linecount)
+ unsigned int & linecount)
{
return read_in_att_format
(is, NULL /* a dummy variable */,
epsilon_symbol, linecount);
}
- /** @brief Create an HfstTransitionGraph as defined
- in AT&T transducer format in FILE \a file.
- \a epsilon_symbol defines how epsilon is represented.
- @pre \a is not at end, otherwise an exception is thrown.
- @note Multiple AT&T transducer definitions are separated with
+ /** @brief Create an HfstTransitionGraph as defined
+ in AT&T transducer format in FILE \a file.
+ \a epsilon_symbol defines how epsilon is represented.
+ @pre \a is not at end, otherwise an exception is thrown.
+ @note Multiple AT&T transducer definitions are separated with
the line "--". */
- HFSTDLL static HfstTransitionGraph read_in_att_format
- (FILE *file,
+ HfstBasicTransducer HfstBasicTransducer::read_in_att_format
+ (FILE *file,
std::string epsilon_symbol,
- unsigned int & linecount)
+ unsigned int & linecount)
{
return read_in_att_format
(std::cin /* a dummy variable */,
file, epsilon_symbol, linecount);
- }
+ }
// ----------------------------------------------
// ----- Substitution functions -----
// ----------------------------------------------
- protected:
+
/* A function that performs in-place-substitution in the graph. */
- void substitute_(HfstSymbol old_symbol,
+ void HfstBasicTransducer::substitute_(HfstSymbol old_symbol,
HfstSymbol new_symbol,
- bool input_side=true,
- bool output_side=true)
+ bool input_side/*=true*/,
+ bool output_side/*=true*/)
{
// ----- Go through all states -----
for (iterator it = begin(); it != end(); it++)
@@ -1898,9 +1753,9 @@
// Go through all transitions
for (unsigned int i=0; i < it->size(); i++)
{
- HfstTransition<C> &tr_it = it->operator[](i);
+ HfstBasicTransition &tr_it = it->operator[](i);
- // The substituting input and output symbols for the
+ // The substituting input and output symbols for the
// current transition.
HfstSymbol substituting_input_symbol
= tr_it.get_input_symbol();
@@ -1927,7 +1782,7 @@
add_symbol_to_alphabet(new_symbol);
// change the current transition accordingly.
- HfstTransition<C> tr
+ HfstBasicTransition tr
(tr_it.get_target_state(),
substituting_input_symbol,
substituting_output_symbol,
@@ -1946,9 +1801,9 @@
/* A function that performs in-place substitutions in the graph
as defined in \a substitutions.
- substitutions[from_number] = to_number,
+ substitutions[from_number] = to_number,
if substitutions[from_number] = no_substitution, no substitution is made */
- void substitute_(const HfstNumberVector &substitutions,
+ void HfstBasicTransducer::substitute_(const HfstNumberVector &substitutions,
unsigned int no_substitution)
{
// ----- Go through all states -----
@@ -1957,7 +1812,7 @@
// Go through all transitions
for (unsigned int i=0; i < it->size(); i++)
{
- HfstTransition<C> &tr_it = it->operator[](i);
+ HfstBasicTransition &tr_it = it->operator[](i);
HfstNumber old_inumber = tr_it.get_input_number();
HfstNumber old_onumber = tr_it.get_output_number();
@@ -1967,20 +1822,20 @@
// If a substitution is to be performed,
if (new_inumber != no_substitution ||
- new_onumber != no_substitution)
+ new_onumber != no_substitution)
{
if (new_inumber != no_substitution)
- add_symbol_to_alphabet(C::get_symbol(new_inumber));
+ add_symbol_to_alphabet(HfstTropicalTransducerTransitionData::get_symbol(new_inumber));
else
new_inumber = old_inumber;
if (new_onumber != no_substitution)
- add_symbol_to_alphabet(C::get_symbol(new_onumber));
+ add_symbol_to_alphabet(HfstTropicalTransducerTransitionData::get_symbol(new_onumber));
else
new_onumber = old_onumber;
// change the current transition accordingly.
- HfstTransition<C> tr
+ HfstBasicTransition tr
(tr_it.get_target_state(),
new_inumber,
new_onumber,
@@ -1998,7 +1853,7 @@
/* A function that performs in-place substitutions in the graph
as defined in \a substitutions. */
- void substitute_(const HfstNumberPairSubstitutions &substitutions)
+ void HfstBasicTransducer::substitute_(const HfstNumberPairSubstitutions &substitutions)
{
// ----- Go through all states -----
for (iterator it = begin(); it != end(); it++)
@@ -2006,7 +1861,7 @@
// Go through all transitions
for (unsigned int i=0; i < it->size(); i++)
{
- HfstTransition<C> &tr_it = it->operator[](i);
+ HfstBasicTransition &tr_it = it->operator[](i);
HfstNumberPair old_number_pair
( tr_it.get_input_number(),
@@ -2027,7 +1882,7 @@
get_symbol(new_output_number));
// change the current transition accordingly.
- HfstTransition<C> tr
+ HfstBasicTransition tr
(tr_it.get_target_state(),
new_input_number,
new_output_number,
@@ -2043,14 +1898,14 @@
return;
}
- public:
+
/* A function that performs in-place removal of all transitions
equivalent to \a sp in the graph. */
- HFSTDLL void remove_transitions(const HfstSymbolPair &sp)
+ void HfstBasicTransducer::remove_transitions(const HfstSymbolPair &sp)
{
- unsigned int in_match = C::get_number(sp.first);
- unsigned int out_match = C::get_number(sp.second);
+ unsigned int in_match = HfstTropicalTransducerTransitionData::get_number(sp.first);
+ unsigned int out_match = HfstTropicalTransducerTransitionData::get_number(sp.second);
bool in_match_used = false;
bool out_match_used = false;
@@ -2061,14 +1916,14 @@
// Go through all transitions of the current state
for (unsigned int i=0; i < it->size(); i++)
{
- HfstTransition<C> &tr_it = it->operator[](i);
+ HfstBasicTransition &tr_it = it->operator[](i);
// If a match was found, remove the transition:
unsigned int in_tr = tr_it.get_input_number();
unsigned int out_tr = tr_it.get_output_number();
if (in_tr == in_match && out_tr == out_match) {
it->erase(it->begin()+i); }
- else
+ else
{
if (in_tr == in_match || out_tr == in_match) {
in_match_used=true; }
@@ -2085,10 +1940,10 @@
alphabet.erase(sp.second); }
}
- protected:
+
/* A function that performs in-place-substitution in the graph. */
- void substitute_(const HfstSymbolPair &old_sp,
+ void HfstBasicTransducer::substitute_(const HfstSymbolPair &old_sp,
const HfstSymbolPairSet &new_sps)
{
if (new_sps.empty())
@@ -2096,8 +1951,8 @@
return remove_transitions(old_sp);
}
- unsigned int old_input_number = C::get_number(old_sp.first);
- unsigned int old_output_number = C::get_number(old_sp.second);
+ unsigned int old_input_number = HfstTropicalTransducerTransitionData::get_number(old_sp.first);
+ unsigned int old_output_number = HfstTropicalTransducerTransitionData::get_number(old_sp.second);
// Whether any substitution was performed
bool substitution_performed=false;
@@ -2106,12 +1961,12 @@
for (iterator it = begin(); it != end(); it++)
{
// The transitions to be added to the current state
- HfstTransitions new_transitions;
+ HfstBasicTransitions new_transitions;
// Go through all transitions of the current state
for (unsigned int i=0; i < it->size(); i++)
{
- HfstTransition<C> &tr_it = it->operator[](i);
+ HfstBasicTransition &tr_it = it->operator[](i);
// If a match was found, substitute:
if (tr_it.get_input_number() == old_input_number &&
@@ -2121,13 +1976,13 @@
// change the current transition so that it is equivalent
// to the first substituting transition in new_sps
- typename HfstSymbolPairSet::const_iterator IT
+ HfstSymbolPairSet::const_iterator IT
= new_sps.begin();
- HfstTransition<C> tr
+ HfstBasicTransition tr
(tr_it.get_target_state(),
- C::get_number(IT->first),
- C::get_number(IT->second),
+ HfstTropicalTransducerTransitionData::get_number(IT->first),
+ HfstTropicalTransducerTransitionData::get_number(IT->second),
tr_it.get_weight(),
true);
@@ -2137,10 +1992,10 @@
// in new_sps to be added to the current state.
while (IT != new_sps.end())
{
- HfstTransition<C> TR
+ HfstBasicTransition TR
(tr_it.get_target_state(),
- C::get_number(IT->first),
- C::get_number(IT->second),
+ HfstTropicalTransducerTransitionData::get_number(IT->first),
+ HfstTropicalTransducerTransitionData::get_number(IT->second),
tr_it.get_weight(),
true);
@@ -2148,12 +2003,12 @@
IT++;
}
- } // (substitution and scheduling done)
+ } // (substitution and scheduling done)
} // (all transitions of a state gone through)
// Add the new transitions to the current state
- for (typename HfstTransitions
+ for (HfstBasicTransitions
::const_iterator NIT = new_transitions.begin();
NIT != new_transitions.end(); NIT++)
{
@@ -2172,7 +2027,7 @@
// Remove symbols that were removed because of substitutions
// (or didn't occur in the graph in the first place)
std::set<unsigned int> syms;
- /*for (typename HfstSymbolPairSet::const_iterator it = new_sps.begin();
+ /*for (HfstSymbolPairSet::const_iterator it = new_sps.begin();
it != new_sps.end(); it++) {
syms.insert(C::get_number(it->first));
syms.insert(C::get_number(it->second)); ?????????
@@ -2186,19 +2041,19 @@
/* A function that performs in-place-substitution in the graph. */
- void substitute_(bool (*func)
+ void HfstBasicTransducer::substitute_(bool (*func)
(const HfstSymbolPair &sp, HfstSymbolPairSet &sps))
{
// ----- Go through all states. -----
for (iterator it = begin(); it != end(); it++)
{
// The transitions to be added to the current state.
- HfstTransitions new_transitions;
+ HfstBasicTransitions new_transitions;
// Go through all transitions.
for (unsigned int i=0; i < it->size(); i++)
{
- HfstTransition<C> &tr_it = it->operator[](i);
+ HfstBasicTransition &tr_it = it->operator[](i);
HfstSymbolPair transition_symbol_pair
(tr_it.get_input_symbol(),
@@ -2208,7 +2063,7 @@
// If a substitution is to be performed,
bool perform_substitution=false;
try {
- perform_substitution =
+ perform_substitution =
(*func)(transition_symbol_pair, substituting_transitions);
}
catch (const HfstException & e)
@@ -2216,19 +2071,19 @@
throw e;
}
if (perform_substitution)
- {
+ {
// change the transition to the first element
// in new_sps
- typename HfstSymbolPairSet::const_iterator IT
+ HfstSymbolPairSet::const_iterator IT
= substituting_transitions.begin();
- if (! C::is_valid_symbol(IT->first) ||
- ! C::is_valid_symbol(IT->second) )
+ if (! HfstTropicalTransducerTransitionData::is_valid_symbol(IT->first) ||
+ ! HfstTropicalTransducerTransitionData::is_valid_symbol(IT->second) )
HFST_THROW_MESSAGE
(EmptyStringException,
- "HfstTransitionGraph::substitute");
+ "HfstBasicTransducer::substitute");
- HfstTransition<C> tr
+ HfstBasicTransition tr
(tr_it.get_target_state(),
IT->first,
IT->second,
@@ -2244,13 +2099,13 @@
while (IT != substituting_transitions.end())
{
- if (! C::is_valid_symbol(IT->first) ||
- ! C::is_valid_symbol(IT->second) )
+ if (! HfstTropicalTransducerTransitionData::is_valid_symbol(IT->first) ||
+ ! HfstTropicalTransducerTransitionData::is_valid_symbol(IT->second) )
HFST_THROW_MESSAGE
(EmptyStringException,
- "HfstTransitionGraph::substitute");
+ "HfstBasicTransducer::substitute");
- HfstTransition<C> TR
+ HfstBasicTransition TR
(tr_it.get_target_state(),
IT->first,
IT->second,
@@ -2264,12 +2119,12 @@
IT++;
}
- } // Substitution and scheduling performed.
+ } // Substitution and scheduling performed.
} // All transitions gone through.
// Add the new transitions.
- for (typename HfstTransitions
+ for (HfstBasicTransitions
::const_iterator NIT = new_transitions.begin();
NIT != new_transitions.end(); NIT++)
{
@@ -2281,26 +2136,26 @@
return;
}
- public:
+
/* ----------------------------------------
The public substitution functions.
---------------------------------------- */
- /** @brief Substitute \a old_symbol with \a new_symbol in
+ /** @brief Substitute \a old_symbol with \a new_symbol in
all transitions. \a input_side and \a output_side define
whether the substitution is made on input and output sides. */
- HFSTDLL HfstTransitionGraph &
- substitute(const HfstSymbol &old_symbol,
+ HfstBasicTransducer &
+ HfstBasicTransducer::substitute(const HfstSymbol &old_symbol,
const HfstSymbol &new_symbol,
- bool input_side=true,
- bool output_side=true) {
+ bool input_side/*=true*/,
+ bool output_side/*=true*/) {
- if (! C::is_valid_symbol(old_symbol) ||
- ! C::is_valid_symbol(new_symbol) ) {
+ if (! HfstTropicalTransducerTransitionData::is_valid_symbol(old_symbol) ||
+ ! HfstTropicalTransducerTransitionData::is_valid_symbol(new_symbol) ) {
HFST_THROW_MESSAGE
(EmptyStringException,
- "HfstTransitionGraph::substitute"); }
+ "HfstBasicTransducer::substitute"); }
// If a symbol is substituted with itself, do nothing.
if (old_symbol == new_symbol)
@@ -2313,7 +2168,7 @@
// if the substitution is made on both sides.
if (input_side && output_side) {
/* Special symbols are always included in the alphabet */
- if (! is_epsilon(old_symbol) &&
+ if (! is_epsilon(old_symbol) &&
! is_unknown(old_symbol) &&
! is_identity(old_symbol)) {
alphabet.erase(old_symbol); }
@@ -2326,12 +2181,12 @@
return *this;
}
- HFSTDLL HfstTransitionGraph &substitute_symbols
+ HfstBasicTransducer & HfstBasicTransducer::substitute_symbols
(const HfstSymbolSubstitutions &substitutions)
{ return this->substitute(substitutions); }
/** @brief Substitute all transitions as defined in \a substitutions */
- HfstTransitionGraph &substitute
+ HfstBasicTransducer & HfstBasicTransducer::substitute
(const HfstSymbolSubstitutions &substitutions)
{
// add symbols to the global HfstTransition alphabet
@@ -2347,9 +2202,11 @@
// substitutions_[from_symbol] = to_symbol
std::vector<unsigned int> substitutions_;
// marker that means that no substitution is made
- unsigned int no_substitution = C::get_max_number()+substitutions.size()+1;
+ size_t st = HfstTropicalTransducerTransitionData::get_max_number()+substitutions.size()+1;
+ unsigned int no_substitution = hfst::size_t_to_uint(st);
+
substitutions_.resize
- (C::get_max_number()+1, no_substitution);
+ (HfstTropicalTransducerTransitionData::get_max_number()+1, no_substitution);
for (HfstSymbolSubstitutions::const_iterator it
= substitutions.begin();
it != substitutions.end(); it++)
@@ -2365,30 +2222,30 @@
return *this;
}
- HFSTDLL HfstTransitionGraph &substitute_symbol_pairs
+ HfstBasicTransducer & HfstBasicTransducer::substitute_symbol_pairs
(const HfstSymbolPairSubstitutions &substitutions)
{ return this->substitute(substitutions); }
/** @brief Substitute all transitions as defined in \a substitutions.
- For each transition x:y, \a substitutions is searched and if
+ For each transition x:y, \a substitutions is searched and if
a mapping x:y -> X:Y is found, the transition x:y is replaced
with X:Y. If no mapping is found, the transition remains the same.
*/
- HFSTDLL HfstTransitionGraph &substitute
+ HfstBasicTransducer & HfstBasicTransducer::substitute
(const HfstSymbolPairSubstitutions &substitutions)
{
// Convert from symbols to numbers
HfstNumberPairSubstitutions substitutions_;
- for (HfstSymbolPairSubstitutions::const_iterator it
+ for (HfstSymbolPairSubstitutions::const_iterator it
= substitutions.begin();
it != substitutions.end(); it++)
{
HfstNumberPair from_transition
- (get_symbol_number(it->first.first),
+ (get_symbol_number(it->first.first),
get_symbol_number(it->first.second));
HfstNumberPair to_transition
- (get_symbol_number(it->second.first),
+ (get_symbol_number(it->second.first),
get_symbol_number(it->second.second));
substitutions_[from_transition] = to_transition;
}
@@ -2400,23 +2257,23 @@
/** @brief Substitute all transitions \a sp with a set of transitions
\a sps. */
- HFSTDLL HfstTransitionGraph &substitute
- (const HfstSymbolPair &sp, const HfstSymbolPairSet &sps)
+ HfstBasicTransducer & HfstBasicTransducer::substitute
+ (const HfstSymbolPair &sp, const HfstSymbolPairSet &sps)
{
- if (! C::is_valid_symbol(sp.first) ||
- ! C::is_valid_symbol(sp.second) ) {
+ if (! HfstTropicalTransducerTransitionData::is_valid_symbol(sp.first) ||
+ ! HfstTropicalTransducerTransitionData::is_valid_symbol(sp.second) ) {
HFST_THROW_MESSAGE
(EmptyStringException,
- "HfstTransitionGraph::substitute"); }
+ "HfstBasicTransducer::substitute"); }
- for (typename HfstSymbolPairSet::const_iterator it = sps.begin();
+ for (HfstSymbolPairSet::const_iterator it = sps.begin();
it != sps.end(); it++)
{
- if (! C::is_valid_symbol(it->first) ||
- ! C::is_valid_symbol(it->second) ) {
+ if (! HfstTropicalTransducerTransitionData::is_valid_symbol(it->first) ||
+ ! HfstTropicalTransducerTransitionData::is_valid_symbol(it->second) ) {
HFST_THROW_MESSAGE
(EmptyStringException,
- "HfstTransitionGraph::substitute"); }
+ "HfstBasicTransducer::substitute"); }
}
substitute_(sp, sps);
@@ -2424,158 +2281,58 @@
return *this;
}
- /** @brief Substitute all transitions \a old_pair with
+ /** @brief Substitute all transitions \a old_pair with
\a new_pair. */
- HFSTDLL HfstTransitionGraph &substitute
- (const HfstSymbolPair &old_pair,
- const HfstSymbolPair &new_pair)
+ HfstBasicTransducer & HfstBasicTransducer::substitute
+ (const HfstSymbolPair &old_pair,
+ const HfstSymbolPair &new_pair)
{
- if (! C::is_valid_symbol(old_pair.first) ||
- ! C::is_valid_symbol(new_pair.first) ||
- ! C::is_valid_symbol(old_pair.second) ||
- ! C::is_valid_symbol(new_pair.second) ) {
+ if (! HfstTropicalTransducerTransitionData::is_valid_symbol(old_pair.first) ||
+ ! HfstTropicalTransducerTransitionData::is_valid_symbol(new_pair.first) ||
+ ! HfstTropicalTransducerTransitionData::is_valid_symbol(old_pair.second) ||
+ ! HfstTropicalTransducerTransitionData::is_valid_symbol(new_pair.second) ) {
HFST_THROW_MESSAGE
(EmptyStringException,
- "HfstTransitionGraph::substitute"); }
+ "HfstBasicTransducer::substitute"); }
StringPairSet new_pair_set;
new_pair_set.insert(new_pair);
substitute_(old_pair, new_pair_set);
return *this;
- }
+ }
/** @brief Substitute all transitions with a set of transitions as
- defined by function \a func.
+ defined by function \a func.
\a func takes as its argument a transition \a sp and inserts
into the set of transitions \a sps the transitions with which
the original transition \a sp must be replaced. \a func returns
a value indicating whether any substitution must be made, i.e.
whether any transition was inserted into \a sps. */
- HFSTDLL HfstTransitionGraph &
- substitute(bool (*func)
- (const HfstSymbolPair &sp, HfstSymbolPairSet &sps) )
- {
+ HfstBasicTransducer &
+ HfstBasicTransducer::substitute(bool (*func)
+ (const HfstSymbolPair &sp, HfstSymbolPairSet &sps) )
+ {
substitute_(func);
return *this;
}
+ hfst::implementations::HfstBasicTransducer& HfstBasicTransducer::substitute(const HfstSymbolPair &sp,
+ const HfstBasicTransducer &graph) {
-
- /* ----------------------------------------------------
- Substitute string pair with a transition graph
- ---------------------------------------------------- */
-
- protected:
- /* Used in function
- substitute(const StringPair&, HfstTransitionGraph&) */
- struct substitution_data
- {
- HfstState origin_state;
- HfstState target_state;
- typename C::WeightType weight;
- HfstTransitionGraph * substituting_graph;
-
- substitution_data(HfstState origin,
- HfstState target,
- typename C::WeightType weight,
- HfstTransitionGraph * substituting)
- {
- origin_state=origin;
- target_state=target;
- this->weight=weight;
- substituting_graph=substituting;
- }
- };
-
- /* Used in function substitute(const StringPair&,
- HfstTransitionGraph&)
- Add a copy of substituting graph with epsilon transitions between
- states and with weight as defined in \a sub. */
- void add_substitution(const substitution_data &sub) {
- // Epsilon transition to initial state of \a graph
- HfstState s = add_state();
- HfstTransition <C> epsilon_transition
- (s, C::get_epsilon(), C::get_epsilon(),
- sub.weight);
- add_transition(sub.origin_state, epsilon_transition);
-
- /* Offset between state numbers */
- unsigned int offset = s;
-
- // Copy \a graph
- const HfstTransitionGraph * graph = sub.substituting_graph;
- HfstState source_state=0;
- for (const_iterator it = graph->begin();
- it != graph->end(); it++)
- {
- for (typename HfstTransitions::const_iterator tr_it
- = it->begin();
- tr_it != it->end(); tr_it++)
- {
- C data = tr_it->get_transition_data();
-
- HfstTransition <C> transition
- (tr_it->get_target_state() + offset,
- data.get_input_symbol(),
- data.get_output_symbol(),
- data.get_weight());
-
- add_transition(source_state + offset, transition);
- }
- source_state++;
- }
-
- // Epsilon transitions from final states of \a graph
- for (typename FinalWeightMap::const_iterator it
- = graph->final_weight_map.begin();
- it != graph->final_weight_map.end(); it++)
- {
- HfstTransition <C> epsilon_transition
- (sub.target_state, C::get_epsilon(), C::get_epsilon(),
- it->second);
- add_transition(it->first + offset, epsilon_transition);
- }
- }
-
-
- public:
-
- /** @brief Substitute all transitions \a old_symbol : \a new_symbol
- with a copy of \a graph.
-
- Copies of \a graph are attached to this graph with
- epsilon transitions.
-
- The weights of the transitions to be substituted are copied
- to epsilon transitions leaving from the source state of
- the transitions to be substituted to the initial state
- of a copy of \a graph.
-
- The final weights in \a
- graph are copied to epsilon transitions leading from
- the final states (after substitution non-final states)
- of \a graph to target states of transitions
- \a old_symbol : \a new_symbol (that are substituted)
- in this graph.
- */
- HFSTDLL HfstTransitionGraph &
- substitute(const HfstSymbolPair &sp,
- const HfstTransitionGraph &graph) {
-
- if ( ! ( C::is_valid_symbol(sp.first) &&
- C::is_valid_symbol(sp.second) ) ) {
+ if ( ! ( HfstTropicalTransducerTransitionData::is_valid_symbol(sp.first) &&
+ HfstTropicalTransducerTransitionData::is_valid_symbol(sp.second) ) ) {
HFST_THROW_MESSAGE
- (EmptyStringException,
- "HfstTransitionGraph::substitute(const HfstSymbolPair&, "
- "const HfstTransitionGraph&)");
+ (EmptyStringException,
+ "HfstBasicTransducer::substitute(const HfstSymbolPair&, "
+ "const HfstBasicTransducer&)");
}
// If neither symbol to be substituted is known to the graph,
// do nothing.
- if (alphabet.find(sp.first) == alphabet.end() &&
+ if (alphabet.find(sp.first) == alphabet.end() &&
alphabet.find(sp.second) == alphabet.end())
return *this;
@@ -2589,38 +2346,38 @@
{
// The transitions that are substituted, i.e. removed
- std::vector<typename HfstTransitions::iterator>
+ std::vector<HfstBasicTransitions::iterator>
old_transitions;
// Go through all transitions
- for (typename HfstTransitions::iterator tr_it
+ for (HfstBasicTransitions::iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
- // Whether there is anything to substitute
+ // Whether there is anything to substitute
// in this transition
if (data.get_input_symbol() == sp.first &&
- data.get_output_symbol() == sp.second)
+ data.get_output_symbol() == sp.second)
{
// schedule a substitution
substitutions.push_back(substitution_data
- (source_state,
- tr_it->get_target_state(),
+ (source_state,
+ tr_it->get_target_state(),
data.get_weight(),
- const_cast<HfstTransitionGraph *>(&graph)));
+ const_cast<HfstBasicTransducer *>(&graph)));
// schedule the old transition to be deleted
old_transitions.push_back(tr_it);
}
// (one transition gone through)
- }
+ }
// (all transitions in a state gone through)
// Remove the substituted transitions
- for (typename std::vector<typename
- HfstTransitions::iterator>::iterator IT =
- old_transitions.begin();
+ for (std::vector<
+ HfstBasicTransitions::iterator>::iterator IT =
+ old_transitions.begin();
IT != old_transitions.end(); IT++) {
it->erase(*IT);
}
@@ -2630,7 +2387,7 @@
// (all states gone trough)
// Add the substitutions
- for (typename std::vector<substitution_data>::iterator IT
+ for (std::vector<substitution_data>::iterator IT
= substitutions.begin();
IT != substitutions.end(); IT++)
{
@@ -2639,60 +2396,98 @@
return *this;
}
+ /* Used in function substitute(const StringPair&,
+ HfstTransitionGraph&)
+ Add a copy of substituting graph with epsilon transitions between
+ states and with weight as defined in \a sub. */
+ void HfstBasicTransducer::add_substitution(const HfstBasicTransducer::substitution_data &sub) {
+ // Epsilon transition to initial state of \a graph
+ HfstState s = add_state();
+ HfstBasicTransition epsilon_transition
+ (s, HfstTropicalTransducerTransitionData::get_epsilon(), HfstTropicalTransducerTransitionData::get_epsilon(),
+ sub.weight);
+ add_transition(sub.origin_state, epsilon_transition);
+
+ /* Offset between state numbers */
+ unsigned int offset = s;
+
+ // Copy \a graph
+ const HfstBasicTransducer * graph = sub.substituting_graph;
+ HfstState source_state=0;
+ for (const_iterator it = graph->begin();
+ it != graph->end(); it++)
+ {
+ for (HfstBasicTransitions::const_iterator tr_it
+ = it->begin();
+ tr_it != it->end(); tr_it++)
+ {
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
+ HfstBasicTransition transition
+ (tr_it->get_target_state() + offset,
+ data.get_input_symbol(),
+ data.get_output_symbol(),
+ data.get_weight());
+ add_transition(source_state + offset, transition);
+ }
+ source_state++;
+ }
+ // Epsilon transitions from final states of \a graph
+ for (FinalWeightMap::const_iterator it
+ = graph->final_weight_map.begin();
+ it != graph->final_weight_map.end(); it++)
+ {
+ HfstBasicTransition epsilon_transition
+ (sub.target_state, HfstTropicalTransducerTransitionData::get_epsilon(), HfstTropicalTransducerTransitionData::get_epsilon(),
+ it->second);
+ add_transition(it->first + offset, epsilon_transition);
+ }
+ }
-
-
-
-
-
-
-
-
- HFSTDLL std::string weight2marker(float weight)
+ std::string HfstBasicTransducer::weight2marker(float weight)
{
std::ostringstream o;
o << weight;
return std::string("@") + o.str() + std::string("@");
}
- HFSTDLL HfstTransitionGraph & substitute_weights_with_markers() {
+ HfstBasicTransducer & HfstBasicTransducer::substitute_weights_with_markers() {
// Go through all current states (we are going to add them)
- HfstState limit = state_vector.size();
- for (HfstState state = 0; state < limit; state++)
+ size_t limit = state_vector.size();
+ for (size_t state = 0; state < limit; state++)
{
// The transitions that are substituted
- std::stack<typename HfstTransitions::iterator>
+ std::stack<HfstBasicTransitions::iterator>
old_transitions;
// The transitions that will substitute
- std::vector<HfstTransition <C> > new_transitions;
+ std::vector<HfstBasicTransition> new_transitions;
// Go through all transitions
- for (typename HfstTransitions::iterator tr_it
+ for (HfstBasicTransitions::iterator tr_it
= state_vector[state].begin();
tr_it != state_vector[state].end(); tr_it++)
{
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
- // Whether there is anything to substitute
+ // Whether there is anything to substitute
// in this transition
if (data.get_weight() != 0 )
{
// schedule a substitution
new_transitions.push_back
- (HfstTransition <C> (tr_it->get_target_state(),
- data.get_input_symbol(),
- data.get_output_symbol(),
- data.get_weight()));
+ (HfstBasicTransition (tr_it->get_target_state(),
+ data.get_input_symbol(),
+ data.get_output_symbol(),
+ data.get_weight()));
// schedule the old transition to be deleted
old_transitions.push(tr_it);
}
// (one transition gone through)
- }
+ }
// (all transitions in a state gone through)
// Remove the substituted transitions
@@ -2702,22 +2497,24 @@
}
// Add the substituting transitions
- for (typename std::vector<HfstTransition <C> >::iterator IT
+ for (std::vector<HfstBasicTransition>::iterator IT
= new_transitions.begin();
IT != new_transitions.end(); IT++)
{
HfstState new_state = add_state();
std::string marker = weight2marker(IT->get_weight());
//std::cerr << "got marker '" << marker << "'" << std::endl;
- HfstTransition <C> marker_transition(IT->get_target_state(),
+ HfstBasicTransition marker_transition(IT->get_target_state(),
marker,
marker,
0);
- HfstTransition <C> new_transition(new_state,
+ HfstBasicTransition new_transition(new_state,
IT->get_input_symbol(),
IT->get_output_symbol(),
0);
- add_transition(state, new_transition);
+
+ unsigned int source_state = hfst::size_t_to_uint(state);
+ add_transition(source_state, new_transition);
add_transition(new_state, marker_transition);
}
@@ -2727,7 +2524,7 @@
// Go through the final states
std::set<HfstState> final_states_to_remove;
- for (typename FinalWeightMap::iterator fin_it = final_weight_map.begin();
+ for (FinalWeightMap::iterator fin_it = final_weight_map.begin();
fin_it != final_weight_map.end(); fin_it++)
{
if (fin_it->second != 0)
@@ -2735,7 +2532,7 @@
HfstState new_state = add_state();
set_final_weight(new_state, 0);
std::string marker = weight2marker(fin_it->second);
- HfstTransition <C> epsilon_transition(new_state,
+ HfstBasicTransition epsilon_transition(new_state,
marker,
marker,
0);
@@ -2751,27 +2548,22 @@
return *this;
}
-
- // ####
- // another version of substitute for internal use..
- // ####
- typedef std::map<HfstSymbol, HfstTransitionGraph> SubstMap;
- HFSTDLL HfstTransitionGraph &
- substitute(SubstMap & substitution_map,
+ HfstBasicTransducer &
+ HfstBasicTransducer::substitute(SubstMap & substitution_map,
bool harmonize) {
bool symbol_found = false;
- for (typename SubstMap::const_iterator it = substitution_map.begin();
+ for (SubstMap::const_iterator it = substitution_map.begin();
it != substitution_map.end(); it++)
{
- if ( ! ( C::is_valid_symbol(it->first) ))
+ if ( ! ( HfstTropicalTransducerTransitionData::is_valid_symbol(it->first) ))
{
- HFST_THROW_MESSAGE(EmptyStringException,
- "HfstTransitionGraph::substitute "
- "(const std::map<HfstSymbol, HfstTransitionGraph> &)");
+ HFST_THROW_MESSAGE(EmptyStringException,
+ "HfstBasicTransducer::substitute "
+ "(const std::map<HfstSymbol, HfstBasicTransducer> &)");
}
- if (!symbol_found && alphabet.find(it->first) != alphabet.end())
+ if (!symbol_found && alphabet.find(it->first) != alphabet.end())
{
symbol_found = true;
}
@@ -2796,22 +2588,22 @@
{
// The transitions that are substituted, i.e. removed
- std::stack<typename HfstTransitions::iterator>
+ std::stack<HfstBasicTransitions::iterator>
old_transitions;
// Go through all transitions
- for (typename HfstTransitions::iterator tr_it
+ for (HfstBasicTransitions::iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
- // Whether there is anything to substitute
+ // Whether there is anything to substitute
// in this transition
String istr = data.get_input_symbol();
String ostr = data.get_output_symbol();
- typename SubstMap::iterator map_it_input = substitution_map.find(istr);
- typename SubstMap::iterator map_it_output = substitution_map.find(ostr);
+ SubstMap::iterator map_it_input = substitution_map.find(istr);
+ SubstMap::iterator map_it_output = substitution_map.find(ostr);
if (map_it_input == substitution_map.end() &&
map_it_output == substitution_map.end())
@@ -2828,7 +2620,7 @@
// schedule a substitution
substitution_data sd
(source_state,
- tr_it->get_target_state(),
+ tr_it->get_target_state(),
data.get_weight(),
&(map_it_input->second));
substitutions.push_back(sd);
@@ -2838,11 +2630,11 @@
substitutions_performed_for_symbols.insert(istr);
}
// (one transition gone through)
- }
+ }
// (all transitions in a state gone through)
// Remove the substituted transitions
- while (!old_transitions.empty())
+ while (!old_transitions.empty())
{
it->erase(old_transitions.top());
old_transitions.pop();
@@ -2871,7 +2663,7 @@
}
// Add the substitutions
- for (typename std::vector<substitution_data>::iterator IT
+ for (std::vector<substitution_data>::iterator IT
= substitutions.begin();
IT != substitutions.end(); IT++)
{
@@ -2883,7 +2675,7 @@
- HFSTDLL bool marker2weight(const std::string & str, float & weight)
+ bool HfstBasicTransducer::marker2weight(const std::string & str, float & weight)
{
if (str.size() < 3)
return false;
@@ -2895,42 +2687,42 @@
sstream >> weight;
if (sstream.fail()) {
return false;
- }
+ }
return true;
- }
+ }
- HFSTDLL HfstTransitionGraph & substitute_markers_with_weights() {
+ HfstBasicTransducer & HfstBasicTransducer::substitute_markers_with_weights() {
// Go through all states
- HfstState limit = state_vector.size();
- for (HfstState state = 0; state < limit; state++)
+ size_t limit = state_vector.size();
+ for (size_t state = 0; state < limit; state++)
{
// The transitions that are substituted
- std::stack<typename HfstTransitions::iterator>
+ std::stack<HfstBasicTransitions::iterator>
old_transitions;
// The transitions that will substitute
- std::vector<HfstTransition <C> > new_transitions;
+ std::vector<HfstBasicTransition> new_transitions;
// Go through all transitions
- for (typename HfstTransitions::iterator tr_it
+ for (HfstBasicTransitions::iterator tr_it
= state_vector[state].begin();
tr_it != state_vector[state].end(); tr_it++)
{
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
float weight = 0;
- // Whether there is anything to substitute
+ // Whether there is anything to substitute
// in this transition
- if ( (!marker2weight(data.get_input_symbol(), weight)) &&
+ if ( (!marker2weight(data.get_input_symbol(), weight)) &&
marker2weight(data.get_output_symbol(), weight) )
{
//std::cerr << "got weight '" << weight << "'" << std::endl;
// schedule a substitution
new_transitions.push_back
- (HfstTransition <C> (tr_it->get_target_state(),
- data.get_input_symbol(),
- hfst::internal_epsilon,
- weight));
+ (HfstBasicTransition (tr_it->get_target_state(),
+ data.get_input_symbol(),
+ hfst::internal_epsilon,
+ weight));
// schedule the old transition to be deleted
old_transitions.push(tr_it);
}
@@ -2944,7 +2736,7 @@
}
else {}
// (one transition gone through)
- }
+ }
// (all transitions in a state gone through)
// Remove the substituted transitions
@@ -2954,7 +2746,7 @@
}
// Add the substituting transitions
- for (typename std::vector<HfstTransition <C> >::iterator IT
+ for (std::vector<HfstBasicTransition>::iterator IT
= new_transitions.begin();
IT != new_transitions.end(); IT++)
{
@@ -2980,21 +2772,21 @@
weight_markers.pop();
}
- return *this;
+ return *this;
}
// aliases
- HFSTDLL HfstTransitionGraph & substitute_symbol(const std::string &old_symbol, const std::string &new_symbol, bool input_side=true, bool output_side=true)
+ HfstBasicTransducer & HfstBasicTransducer::substitute_symbol(const std::string &old_symbol, const std::string &new_symbol, bool input_side/*=true*/, bool output_side/*=true*/)
{ return this->substitute(old_symbol, new_symbol, input_side, output_side); }
- HFSTDLL HfstTransitionGraph & substitute_symbol_pair(const StringPair &old_symbol_pair, const StringPair &new_symbol_pair)
+ HfstBasicTransducer & HfstBasicTransducer::substitute_symbol_pair(const StringPair &old_symbol_pair, const StringPair &new_symbol_pair)
{ return this->substitute(old_symbol_pair, new_symbol_pair); }
- HFSTDLL HfstTransitionGraph & substitute_symbol_pair_with_set(const StringPair &old_symbol_pair, const hfst::StringPairSet &new_symbol_pair_set)
+ HfstBasicTransducer & HfstBasicTransducer::substitute_symbol_pair_with_set(const StringPair &old_symbol_pair, const hfst::StringPairSet &new_symbol_pair_set)
{ return this->substitute(old_symbol_pair, new_symbol_pair_set); }
- HFSTDLL HfstTransitionGraph & substitute_symbol_pair_with_transducer(const StringPair &symbol_pair, HfstTransitionGraph &transducer)
+ HfstBasicTransducer & HfstBasicTransducer::substitute_symbol_pair_with_transducer(const StringPair &symbol_pair, HfstBasicTransducer &transducer)
{ return this->substitute(symbol_pair, transducer); }
@@ -3003,16 +2795,16 @@
---------------------------- */
- /** @brief Insert freely any number of \a symbol_pair in
+ /** @brief Insert freely any number of \a symbol_pair in
the graph with weight \a weight. */
- HFSTDLL HfstTransitionGraph &insert_freely
- (const HfstSymbolPair &symbol_pair, typename C::WeightType weight)
- {
- if ( ! ( C::is_valid_symbol(symbol_pair.first) &&
- C::is_valid_symbol(symbol_pair.second) ) ) {
+ HfstBasicTransducer & HfstBasicTransducer::insert_freely
+ (const HfstSymbolPair &symbol_pair, HfstTropicalTransducerTransitionData::WeightType weight)
+ {
+ if ( ! ( HfstTropicalTransducerTransitionData::is_valid_symbol(symbol_pair.first) &&
+ HfstTropicalTransducerTransitionData::is_valid_symbol(symbol_pair.second) ) ) {
HFST_THROW_MESSAGE
- (EmptyStringException,
- "HfstTransitionGraph::insert_freely"
+ (EmptyStringException,
+ "HfstBasicTransducer::insert_freely"
"(const HfstSymbolPair&, W)");
}
@@ -3021,8 +2813,8 @@
HfstState source_state=0;
for (iterator it = begin(); it != end(); it++) {
- HfstTransition <C> tr( source_state, symbol_pair.first,
- symbol_pair.second, weight );
+ HfstBasicTransition tr( source_state, symbol_pair.first,
+ symbol_pair.second, weight );
it->push_back(tr);
source_state++;
}
@@ -3030,21 +2822,21 @@
return *this;
}
- /** @brief Insert freely any number of any symbol in \a symbol_pairs in
+ /** @brief Insert freely any number of any symbol in \a symbol_pairs in
the graph with weight \a weight. */
- HFSTDLL HfstTransitionGraph &insert_freely
- (const HfstSymbolPairSet &symbol_pairs,
- typename C::WeightType weight)
+ HfstBasicTransducer & HfstBasicTransducer::insert_freely
+ (const HfstSymbolPairSet &symbol_pairs,
+ HfstTropicalTransducerTransitionData::WeightType weight)
{
- for (typename HfstSymbolPairSet::const_iterator symbols_it
+ for (HfstSymbolPairSet::const_iterator symbols_it
= symbol_pairs.begin();
symbols_it != symbol_pairs.end(); symbols_it++)
{
- if ( ! ( C::is_valid_symbol(symbols_it->first) &&
- C::is_valid_symbol(symbols_it->second) ) ) {
+ if ( ! ( HfstTropicalTransducerTransitionData::is_valid_symbol(symbols_it->first) &&
+ HfstTropicalTransducerTransitionData::is_valid_symbol(symbols_it->second) ) ) {
HFST_THROW_MESSAGE
- (EmptyStringException,
- "HfstTransitionGraph::insert_freely"
+ (EmptyStringException,
+ "HfstBasicTransducer::insert_freely"
"(const HfstSymbolPairSet&, W)");
}
@@ -3053,14 +2845,14 @@
}
HfstState source_state=0;
- for (iterator it = begin(); it != end(); it++)
+ for (iterator it = begin(); it != end(); it++)
{
- for (typename HfstSymbolPairSet::const_iterator symbols_it
+ for (HfstSymbolPairSet::const_iterator symbols_it
= symbol_pairs.begin();
symbols_it != symbol_pairs.end(); symbols_it++)
{
- HfstTransition <C> tr( source_state, symbols_it->first,
- symbols_it->second, weight );
+ HfstBasicTransition tr( source_state, symbols_it->first,
+ symbols_it->second, weight );
it->push_back(tr);
}
source_state++;
@@ -3071,11 +2863,11 @@
/** @brief Insert freely any number of \a graph in this
graph. */
- HFSTDLL HfstTransitionGraph &insert_freely
- (const HfstTransitionGraph &graph)
+ HfstBasicTransducer & HfstBasicTransducer::insert_freely
+ (const HfstBasicTransducer &graph)
{
- HfstSymbol marker_this = C::get_marker(alphabet);
- HfstSymbol marker_graph = C::get_marker(alphabet);
+ HfstSymbol marker_this = HfstTropicalTransducerTransitionData::get_marker(alphabet);
+ HfstSymbol marker_graph = HfstTropicalTransducerTransitionData::get_marker(alphabet);
HfstSymbol marker = marker_this;
if (marker_graph > marker)
marker = marker_graph;
@@ -3095,23 +2887,23 @@
/** @brief Harmonize this HfstTransitionGraph and \a another.
- In harmonization the unknown and identity symbols in
+ In harmonization the unknown and identity symbols in
transitions of both graphs are expanded according to
- the symbols that are previously unknown to the graph.
+ the symbols that are previously unknown to the graph.
For example the graphs
- \verbatim
+ \verbatim
[a:b ?:?]
[c:d ? ?:c]
\endverbatim
are expanded to
\verbatim
- [ a:b [?:? | ?:c | ?:d | c:d | d:c | c:? | d:?] ]
+ [ a:b [?:? | ?:c | ?:d | c:d | d:c | c:? | d:?] ]
[ c:d [? | a | b] [?:c| a:c | b:?] ]
\endverbatim
when harmonized.
- The symbol "?" means \@_UNKNOWN_SYMBOL_\@ in either or
- both sides of a transition
+ The symbol "?" means \@_UNKNOWN_SYMBOL_\@ in either or
+ both sides of a transition
(transitions of type [?:x], [x:?] and [?:?]).
The transition [?] means [\@_IDENTITY_SYMBOL_\@].
@@ -3119,7 +2911,7 @@
that take two or more graphs as their arguments, unless otherwise
said.
*/
- HFSTDLL HfstTransitionGraph &harmonize(HfstTransitionGraph &another)
+ HfstBasicTransducer & HfstBasicTransducer::harmonize(HfstBasicTransducer &another)
{
HarmonizeUnknownAndIdentitySymbols foo(*this, another);
return *this;
@@ -3130,13 +2922,13 @@
Disjunction functions
------------------------------- */
- protected:
+
/* Disjunct the transition of path \a spv pointed by \a it
to state \a s. If the transition does not exist in the graph,
it is created as well as its target state.
@return The final state of path \a spv, when \a it is at end. */
- HfstState disjunct(const StringPairVector &spv,
+ HfstState HfstBasicTransducer::disjunct(const StringPairVector &spv,
StringPairVector::const_iterator &it,
HfstState s)
{
@@ -3145,17 +2937,17 @@
return s;
}
- HfstTransitions tr = state_vector[s];
+ HfstBasicTransitions tr = state_vector[s];
bool transition_found=false;
/* The target state of the transition followed or added */
- HfstState next_state;
+ HfstState next_state;
// Find the transition
// (Searching is slow?)
- for (typename HfstTransitions::iterator tr_it = tr.begin();
+ for (HfstBasicTransitions::iterator tr_it = tr.begin();
tr_it != tr.end(); tr_it++)
{
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
if (data.get_input_symbol().compare(it->first) == 0 &&
data.get_output_symbol().compare(it->second) == 0)
{
@@ -3169,8 +2961,8 @@
if (! transition_found)
{
next_state = add_state();
- HfstTransition <C> transition(next_state, it->first,
- it->second, 0);
+ HfstBasicTransition transition(next_state, it->first,
+ it->second, 0);
add_transition(s, transition);
}
@@ -3179,17 +2971,17 @@
return disjunct(spv, it, next_state);
}
- public:
+
- /** @brief Disjunct this graph with a one-path graph
- defined by string pair vector \a spv with weight \a weight.
+ /** @brief Disjunct this graph with a one-path graph
+ defined by string pair vector \a spv with weight \a weight.
@pre This graph must be a trie where all weights are in
- final states, i.e. all transitions have a zero weight.
+ final states, i.e. all transitions have a zero weight.
There is no way to test whether a graph is a trie, so the use
- of this function is probably limited to fast construction
- of a lexicon. Here is an example:
+ of this function is probably limited to fast construction
+ of a lexicon. Here is an example:
\verbatim
HfstBasicTransducer lexicon;
@@ -3200,24 +2992,24 @@
\endverbatim
*/
- HFSTDLL HfstTransitionGraph &disjunct
- (const StringPairVector &spv, typename C::WeightType weight)
+ HfstBasicTransducer & HfstBasicTransducer::disjunct
+ (const StringPairVector &spv, HfstTropicalTransducerTransitionData::WeightType weight)
{
StringPairVector::const_iterator it = spv.begin();
HfstState final_state = disjunct(spv, it, INITIAL_STATE);
// Set the weight of final state
- if (is_final_state(final_state))
+ if (is_final_state(final_state))
{
float old_weight = get_final_weight(final_state);
- if (old_weight < weight)
+ if (old_weight < weight)
return *this; /* The same path with smaller weight remains */
}
set_final_weight(final_state, weight);
return *this;
}
- HFSTDLL bool is_special_symbol(const std::string & symbol)
+ bool HfstBasicTransducer::is_special_symbol(const std::string & symbol)
{
if (symbol.size() < 2)
return false;
@@ -3226,7 +3018,7 @@
return false;
}
- HFSTDLL HfstTransitionGraph &complete()
+ HfstBasicTransducer & HfstBasicTransducer::complete()
{
HfstState failure_state = add_state();
HfstState current_state = 0;
@@ -3235,11 +3027,11 @@
{
std::set<HfstSymbol> symbols_present;
- for (typename HfstTransitions::iterator tr_it
+ for (HfstBasicTransitions::iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
if (data.get_input_symbol() != data.get_output_symbol())
{
@@ -3250,8 +3042,8 @@
for (std::set<std::string>::const_iterator alpha_it = alphabet.begin();
alpha_it != alphabet.end(); alpha_it++)
{
- if (symbols_present.find(*alpha_it) ==
- symbols_present.end() &&
+ if (symbols_present.find(*alpha_it) ==
+ symbols_present.end() &&
! is_special_symbol(*alpha_it))
{
add_transition
@@ -3264,11 +3056,11 @@
return *this;
}
- HFSTDLL StringSet get_flags() const
+ StringSet HfstBasicTransducer::get_flags() const
{
StringSet flags;
for (StringSet::const_iterator it = alphabet.begin();
- it != alphabet.end(); it++)
+ it != alphabet.end(); it++)
{
if (FdOperation::is_diacritic(*it)) {
flags.insert(*it);
@@ -3280,8 +3072,8 @@
// Whether symbol \a symbol must be purged from transitions and alphabet
// of a transducer after \a flag has been eliminated from the transducer.
// If \a flag is the empty string, all flags have been eliminated.
- HFSTDLL bool purge_symbol(const std::string & symbol, const std::string & flag)
- {
+ bool HfstBasicTransducer::purge_symbol(const std::string & symbol, const std::string & flag)
+ {
if (! FdOperation::is_diacritic(symbol))
return false;
if (flag == "")
@@ -3292,22 +3084,22 @@
}
// Replace arcs in \a transducer that use flag \a flag with epsilon arcs
- // and remove \a flag from alphabet of \a transducer. If \a flag is the empty
+ // and remove \a flag from alphabet of \a transducer. If \a flag is the empty
// string, replace/remove all flags.
- HFSTDLL void flag_purge(const std::string & flag)
+ void HfstBasicTransducer::flag_purge(const std::string & flag)
{
// (1) Go through all states and transitions
for (iterator it = begin(); it != end(); it++)
{
for (unsigned int i=0; i < it->size(); i++)
{
- HfstTransition<C> &tr_it = it->operator[](i);
+ HfstBasicTransition &tr_it = it->operator[](i);
if ( purge_symbol(tr_it.get_input_symbol(), flag) ||
purge_symbol(tr_it.get_output_symbol(), flag) )
{
// change the current transition
- HfstTransition<C> tr
+ HfstBasicTransition tr
(tr_it.get_target_state(), "@_EPSILON_SYMBOL_@",
"@_EPSILON_SYMBOL_@", tr_it.get_weight());
it->operator[](i) = tr;
@@ -3317,7 +3109,7 @@
// (2) Go through the alphabet
StringSet extra_symbols;
for (StringSet::const_iterator it = alphabet.begin();
- it != alphabet.end(); it++)
+ it != alphabet.end(); it++)
{
if (purge_symbol(*it, flag))
extra_symbols.insert(*it);
@@ -3326,76 +3118,28 @@
remove_symbols_from_alphabet(extra_symbols);
}
- /* A topological sort. */
- struct TopologicalSort
- {
- std::vector<int> distance_of_state;
- std::vector<std::set<HfstState> > states_at_distance;
-
- /* Initialize the TopologicalSort by reserving space for a transducer
- with biggest state number \a biggest_state_number, */
- HFSTDLL void set_biggest_state_number(unsigned int biggest_state_number)
- {
- distance_of_state = std::vector<int>(biggest_state_number+1, -1);
- }
-
- /* Set the maximum distance of \a state to \a distance, */
- HFSTDLL void set_state_at_distance(HfstState state, unsigned int distance,
- bool overwrite)
- {
- // see that 'state' does not exceed the maximum state number given in initialization
- if (state > (distance_of_state.size() - 1))
- {
- std::cerr << "ERROR in TopologicalSort::set_state_at_distance: first argument ("
- << state << ") is out of range (should be < " << distance_of_state.size()
- << ")" << std::endl;
- }
- // if there is nothing on index 'state',
- // push back empty sets of states up to index 'state', including
- while (distance + 1 > (unsigned int)states_at_distance.size())
- {
- std::set<HfstState> empty_set;
- states_at_distance.push_back(empty_set);
- }
- // if there was previous distance defined for 'state', erase it, if needed
- int previous_distance = distance_of_state.at(state);
- if (previous_distance != -1 && previous_distance != (int)distance && overwrite)
- {
- states_at_distance.at(previous_distance).erase(state);
- }
- // set state and distance
- states_at_distance.at(distance).insert(state);
- distance_of_state.at(state) = distance;
- }
-
- /* The states that have a maximum distance of \a distance. */
- HFSTDLL const std::set<HfstState> & get_states_at_distance(unsigned int distance)
- {
- // if there is nothing on index 'state',
- // push back empty sets of states up to index 'state', including
- while (distance > (states_at_distance.size() - 1))
- {
- std::set<HfstState> empty_set;
- states_at_distance.push_back(empty_set);
- }
- return states_at_distance.at(distance);
- }
- };
-
- enum SortDistance { MaximumDistance, MinimumDistance };
-
- /*
+ /*
Get a topological (maximum distance) sort of this graph.
@return A vector of sets of states. At each vector index ind, the
result contains the set of all states whose (maximum) distance from
the start state is ind.
*/
- HFSTDLL std::vector<std::set<HfstState> > topsort(SortDistance dist) const
+ std::vector<std::set<HfstState> > HfstBasicTransducer::topsort(SortDistance dist) const
{
typedef std::set<HfstState>::const_iterator StateIt;
unsigned int current_distance = 0; // topological distance
TopologicalSort TopSort;
- TopSort.set_biggest_state_number(state_vector.size()-1);
+
+ size_t st = state_vector.size();
+ if (st == 0)
+ {
+ std::vector<std::set<HfstState> > empty;
+ return empty;
+ }
+ st = st-1;
+ unsigned int biggest_state_number = hfst::size_t_to_uint(st);
+ TopSort.set_biggest_state_number(biggest_state_number);
+
TopSort.set_state_at_distance(0,current_distance,(dist == MaximumDistance));
bool new_states_found = false; // end condition for do-while loop
@@ -3406,31 +3150,31 @@
std::set<HfstState> new_states;
// go through all states at current distance
- const std::set<HfstState> & states =
+ const std::set<HfstState> & states =
TopSort.get_states_at_distance(current_distance);
for (StateIt state_it = states.begin();
state_it != states.end(); state_it++)
{
// go through all transitions of each state
- const HfstTransitions & transitions
+ const HfstBasicTransitions & transitions
= this->state_vector.at(*state_it);
- for (typename HfstTransitions::const_iterator transition_it
+ for (HfstBasicTransitions::const_iterator transition_it
= transitions.begin();
transition_it != transitions.end(); transition_it++)
{
new_states_found = true;
new_states.insert(transition_it->get_target_state());
- }
+ }
// all transitions gone through
}
// all states gone through
// set each accessible state at distance one higher than the
// current distance
- for (StateIt it = new_states.begin();
+ for (StateIt it = new_states.begin();
it != new_states.end(); it++)
{
- TopSort.set_state_at_distance(*it, current_distance + 1, (dist == MaximumDistance));
+ TopSort.set_state_at_distance(*it, current_distance + 1, (dist == MaximumDistance));
}
current_distance++;
}
@@ -3439,26 +3183,30 @@
return TopSort.states_at_distance;
}
- /** The length of longest string accepted by this graph.
+ /** The length of longest string accepted by this graph.
If no string is accepted, return -1. */
- HFSTDLL int longest_path_size()
+ int HfstBasicTransducer::longest_path_size()
{
// get topological maximum distance sort
std::vector<std::set<HfstState> > states_sorted = this->topsort(MaximumDistance);
// go through all sets of states in descending order
- for (int distance = states_sorted.size() - 1; distance >= 0; distance--)
+ size_t st = states_sorted.size();
+ if (st > 0)
{
- const std::set<HfstState> & states
- = states_sorted.at((unsigned int)distance);
- // go through all states in a set
- for (std::set<HfstState>::const_iterator it = states.begin();
- it != states.end(); it++)
+ for (int distance = hfst::size_t_to_int(st-1); distance >= 0; distance--)
{
- // if a final state is encountered, return the distance
- // of that state
- if (is_final_state(*it))
+ const std::set<HfstState> & states
+ = states_sorted.at((unsigned int)distance);
+ // go through all states in a set
+ for (std::set<HfstState>::const_iterator it = states.begin();
+ it != states.end(); it++)
{
- return distance;
+ // if a final state is encountered, return the distance
+ // of that state
+ if (is_final_state(*it))
+ {
+ return distance;
+ }
}
}
}
@@ -3466,35 +3214,39 @@
return -1;
}
- /** The lengths of strings accepted by this graph, in descending order.
+ /** The lengths of strings accepted by this graph, in descending order.
If not string is accepted, return an empty vector. */
- HFSTDLL std::vector<unsigned int> path_sizes()
+ std::vector<unsigned int> HfstBasicTransducer::path_sizes()
{
std::vector<unsigned int> result;
// get topological maximum distance sort
std::vector<std::set<HfstState> > states_sorted = this->topsort(MinimumDistance);
// go through all sets of states in descending order
- for (int distance = states_sorted.size() - 1; distance >= 0; distance--)
+ size_t st = states_sorted.size();
+ if (st > 0)
{
- const std::set<HfstState> & states
- = states_sorted.at((unsigned int)distance);
- // go through all states in a set
- for (std::set<HfstState>::const_iterator it = states.begin();
- it != states.end(); it++)
+ for (int distance = hfst::size_t_to_int(st-1); distance >= 0; distance--)
{
- // if a final state is encountered, add its distance
- // to result
- if (is_final_state(*it))
+ const std::set<HfstState> & states
+ = states_sorted.at((unsigned int)distance);
+ // go through all states in a set
+ for (std::set<HfstState>::const_iterator it = states.begin();
+ it != states.end(); it++)
{
- result.push_back((unsigned int)distance);
- break; // go to next set of states
+ // if a final state is encountered, add its distance
+ // to result
+ if (is_final_state(*it))
+ {
+ result.push_back((unsigned int)distance);
+ break; // go to next set of states
+ }
}
}
}
return result;
}
- bool has_negative_epsilon_cycles
+ bool HfstBasicTransducer::has_negative_epsilon_cycles
(HfstState state,
float total_weight,
std::map<HfstState, float> & state_weights)
@@ -3510,10 +3262,10 @@
}
state_weights[state] = total_weight;
- // Go through all transitions in this state
- const HfstBasicTransducer::HfstTransitions &transitions
+ // Go through all transitions in this state
+ const HfstBasicTransitions &transitions
= this->operator[](state);
- for (HfstBasicTransducer::HfstTransitions::const_iterator it
+ for (HfstBasicTransitions::const_iterator it
= transitions.begin();
it != transitions.end(); it++)
{
@@ -3528,12 +3280,12 @@
return false;
}
- bool has_negative_epsilon_cycles()
+ bool HfstBasicTransducer::has_negative_epsilon_cycles()
{
bool has_negative_epsilon_transitions = false;
for (iterator it = begin(); it != end(); it++)
{
- for (typename HfstTransitions::iterator tr_it
+ for (HfstBasicTransitions::iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
@@ -3555,27 +3307,27 @@
if (has_negative_epsilon_cycles(state, 0, state_weights))
return true;
}
- return false;
+ return false;
}
- HFSTDLL bool is_infinitely_ambiguous
- (HfstState state,
+ bool HfstBasicTransducer::is_infinitely_ambiguous
+ (HfstState state,
std::set<HfstState> &epsilon_path_states,
std::vector<unsigned int> &states_handled)
{
if (states_handled[state] != 0)
return false;
- // Go through all transitions in this state
- const HfstBasicTransducer::HfstTransitions &transitions
+ // Go through all transitions in this state
+ const HfstBasicTransitions &transitions
= this->operator[](state);
- for (HfstBasicTransducer::HfstTransitions::const_iterator it
+ for (HfstBasicTransitions::const_iterator it
= transitions.begin();
it != transitions.end(); it++)
{
- // (Diacritics are also treated as epsilons, although it might cause false
- // positive results, because loops with diacritics can be invalidated by
- // other diacritics.)
+ // (Diacritics are also treated as epsilons, although it might cause false
+ // positive results, because loops with diacritics can be invalidated by
+ // other diacritics.)
if ( is_epsilon(it->get_input_symbol()) ||
FdOperation::is_diacritic(it->get_input_symbol()) )
{
@@ -3591,14 +3343,14 @@
return true;
}
epsilon_path_states.erase(state);
- }
+ }
}
// mark state as handled
states_handled[state] = 1;
return false;
}
- HFSTDLL bool is_infinitely_ambiguous()
+ bool HfstBasicTransducer::is_infinitely_ambiguous()
{
std::set<HfstState> epsilon_path_states;
HfstState max_state = this->get_max_state();
@@ -3612,33 +3364,51 @@
return false;
}
- HFSTDLL bool is_lookup_infinitely_ambiguous
+ bool HfstBasicTransducer::is_possible_flag(std::string symbol, StringVector & fds, bool obey_flags)
+ {
+ if (FdOperation::is_diacritic(symbol))
+ {
+ FlagDiacriticTable FdT;
+ fds.push_back(symbol);
+ if ((!obey_flags) || FdT.is_valid_string(fds))
+ { return true; }
+ else
+ {
+ fds.pop_back();
+ return false;
+ }
+ }
+ return false;
+ }
+
+ bool HfstBasicTransducer::is_lookup_infinitely_ambiguous
(const HfstOneLevelPath& s,
unsigned int& index, HfstState state,
- std::set<HfstState> &epsilon_path_states
- )
+ std::set<HfstState> &epsilon_path_states,
+ StringVector & fds, bool obey_flags)
{
- // Whether the end of the lookup path s has been reached
+ // Whether the end of the lookup path s has been reached
bool only_epsilons=false;
if ((unsigned int)s.second.size() == index)
{
only_epsilons=true;
}
- // Go through all transitions in this state
- const HfstBasicTransducer::HfstTransitions &transitions
+ // Go through all transitions in this state
+ const HfstBasicTransitions &transitions
= this->operator[](state);
- for (HfstBasicTransducer::HfstTransitions::const_iterator it
+ for (HfstBasicTransitions::const_iterator it
= transitions.begin();
it != transitions.end(); it++)
{
- // CASE 1: Input epsilons do not consume a symbol in the lookup path s,
- // so they can be added freely.
- // (Diacritics are also treated as epsilons, although it might cause false
- // positive results, because loops with diacritics can be invalidated by
- // other diacritics.)
+ // CASE 1: Input epsilons do not consume a symbol in the lookup path s,
+ // so they can be added freely.
+ // (Diacritics are also treated as epsilons, although it might cause false
+ // positive results, because loops with diacritics can be invalidated by
+ // other diacritics.)
+ bool possible_flag = is_possible_flag(it->get_input_symbol(), fds, obey_flags);
if ( is_epsilon(it->get_input_symbol()) ||
- FdOperation::is_diacritic(it->get_input_symbol()) )
+ possible_flag )
{
epsilon_path_states.insert(state);
if (epsilon_path_states.find(it->get_target_state())
@@ -3647,23 +3417,25 @@
return true;
}
if (is_lookup_infinitely_ambiguous
- (s, index, it->get_target_state(), epsilon_path_states))
+ (s, index, it->get_target_state(), epsilon_path_states, fds, obey_flags))
{
return true;
}
epsilon_path_states.erase(state);
+ if (possible_flag)
+ { fds.pop_back(); }
}
- /* CASE 2: Other input symbols consume a symbol in the lookup path s,
- so they can be added only if the end of the lookup path s has not
+ /* CASE 2: Other input symbols consume a symbol in the lookup path s,
+ so they can be added only if the end of the lookup path s has not
been reached. */
else if (! only_epsilons)
{
bool continu = false;
if (it->get_input_symbol().compare(s.second.at(index)) == 0)
continu = true;
- else if ((it->get_input_symbol().compare("@_UNKNOWN_SYMBOL_@") ||
- it->get_input_symbol().compare("@_IDENTITY_SYMBOL_@"))
+ else if (((it->get_input_symbol().compare("@_UNKNOWN_SYMBOL_@") == 0) ||
+ (it->get_input_symbol().compare("@_IDENTITY_SYMBOL_@") == 0))
&&
(alphabet.find(s.second.at(index)) == alphabet.end()))
{
@@ -3672,61 +3444,76 @@
if (continu)
{
- index++; // consume an input symbol in the lookup path s
+ index++; // consume an input symbol in the lookup path s
std::set<HfstState> empty_set;
if (is_lookup_infinitely_ambiguous
- (s, index, it->get_target_state(), empty_set))
+ (s, index, it->get_target_state(), empty_set, fds, obey_flags))
{
return true;
}
- index--; // add the input symbol back to the lookup path s.
+ index--; // add the input symbol back to the lookup path s.
}
}
}
return false;
}
- HFSTDLL bool is_lookup_infinitely_ambiguous(const HfstOneLevelPath & s)
+ bool HfstBasicTransducer::is_lookup_infinitely_ambiguous(const HfstOneLevelPath & s, bool obey_flags/*=false*/)
{
std::set<HfstState> epsilon_path_states;
epsilon_path_states.insert(0);
unsigned int index=0;
+ StringVector fds;
return is_lookup_infinitely_ambiguous(s, index, INITIAL_STATE,
- epsilon_path_states);
+ epsilon_path_states, fds, obey_flags);
}
- HFSTDLL bool is_lookup_infinitely_ambiguous(const StringVector & s)
+ bool HfstBasicTransducer::is_lookup_infinitely_ambiguous(const StringVector & s, bool obey_flags/*=false*/)
{
std::set<HfstState> epsilon_path_states;
epsilon_path_states.insert(0);
unsigned int index=0;
HfstOneLevelPath path((float)0, s);
+ StringVector fds;
return is_lookup_infinitely_ambiguous(path, index, INITIAL_STATE,
- epsilon_path_states);
+ epsilon_path_states, fds, obey_flags);
}
- HFSTDLL static void push_back_to_two_level_path
+ void HfstBasicTransducer::push_back_to_two_level_path
(HfstTwoLevelPath &path,
const StringPair &sp,
- const float &weight)
+ const float &weight,
+ StringVector * fds_so_far /*= NULL*/)
{
path.second.push_back(sp);
path.first = path.first + weight;
+ if (fds_so_far != NULL)
+ {
+ if (FdOperation::is_diacritic(sp.first))
+ { fds_so_far->push_back(sp.first); }
+ }
}
- HFSTDLL static void pop_back_from_two_level_path
+ void HfstBasicTransducer::pop_back_from_two_level_path
(HfstTwoLevelPath &path,
- const float &weight)
+ const float &weight,
+ StringVector * fds_so_far /*= NULL*/)
{
+ if (fds_so_far != NULL)
+ {
+ StringPair sp = path.second.back();
+ if (FdOperation::is_diacritic(sp.first))
+ { fds_so_far->pop_back(); }
+ }
path.second.pop_back();
path.first = path.first - weight;
}
- HFSTDLL static void add_to_results
+ void HfstBasicTransducer::add_to_results
(HfstTwoLevelPaths &results,
HfstTwoLevelPath &path_so_far,
const float &final_weight,
@@ -3749,12 +3536,13 @@
path_so_far.first = path_so_far.first - final_weight;
}
- HFSTDLL static bool is_possible_transition
+ bool HfstBasicTransducer::is_possible_transition
(const HfstBasicTransition &transition,
const StringVector &lookup_path,
const unsigned int &lookup_index,
const StringSet &alphabet,
- bool &input_symbol_consumed)
+ bool &input_symbol_consumed,
+ StringVector * fds_so_far /*= NULL*/)
{
std::string isymbol = transition.get_input_symbol();
@@ -3769,8 +3557,8 @@
// the current symbol is not found in the alphabet
// of the transducer.
( (is_identity(isymbol) || is_unknown(isymbol)) &&
- (alphabet.find(lookup_path.at(lookup_index))
- == alphabet.end()) )
+ (alphabet.find(lookup_path.at(lookup_index))
+ == alphabet.end()) )
)
{
input_symbol_consumed=true;
@@ -3780,18 +3568,37 @@
// Whether there are more symbols in lookup_path or not,
// we can always go further if the input symbol of the transition
// is an epsilon or a flag diacritic.
- if ( is_epsilon(isymbol) ||
- FdOperation::is_diacritic(isymbol) )
+ if ( is_epsilon(isymbol) )
{
input_symbol_consumed=false;
return true;
}
+ if ( FdOperation::is_diacritic(isymbol) )
+ {
+ if (fds_so_far == NULL)
+ {
+ input_symbol_consumed=false;
+ return true;
+ }
+ else
+ {
+ FlagDiacriticTable FdT;
+ fds_so_far->push_back(isymbol);
+ bool valid = FdT.is_valid_string(*fds_so_far);
+ fds_so_far->pop_back();
+ if (valid)
+ {
+ input_symbol_consumed=false;
+ return true;
+ }
+ }
+ }
// No matches.
return false;
}
- HFSTDLL void lookup_fd
+ void HfstBasicTransducer::lookup
(const StringVector &lookup_path,
HfstTwoLevelPaths &results,
HfstState state,
@@ -3799,8 +3606,9 @@
HfstTwoLevelPath &path_so_far,
StringSet &alphabet,
HfstEpsilonHandler Eh,
- size_t infinite_cutoff,
- float * max_weight = NULL)
+ size_t max_epsilon_cycles,
+ float * max_weight /*= NULL*/,
+ StringVector * flag_diacritic_path /*= NULL*/)
{
// Check whether the number of input epsilon cycles is exceeded
if (! Eh.can_continue(state)) {
@@ -3814,8 +3622,8 @@
// If we are at the end of lookup_path,
if (lookup_index == lookup_path.size())
{
- // and if the current state is final,
- if (this->is_final_state(state))
+ // and if the current state is final,
+ if (this->is_final_state(state))
{
// path_so_far is a valid result if max_weight is not exceeded
add_to_results
@@ -3825,25 +3633,25 @@
// Whether there are more symbols in lookup_path or not,
// go through all transitions in the current state.
- const HfstBasicTransducer::HfstTransitions &transitions
+ const HfstBasicTransitions &transitions
= this->operator[](state);
- for (HfstBasicTransducer::HfstTransitions::const_iterator it
+ for (HfstBasicTransitions::const_iterator it
= transitions.begin();
it != transitions.end(); it++)
{
bool input_symbol_consumed=false;
if ( is_possible_transition
- (*it, lookup_path, lookup_index, alphabet,
- input_symbol_consumed) )
+ (*it, lookup_path, lookup_index, alphabet,
+ input_symbol_consumed, flag_diacritic_path) )
{
// update path_so_far and lookup_index
std::string istr;
std::string ostr;
// identity symbol is replaced with the lookup symbol
- if (is_identity(it->get_input_symbol()))
+ if (is_identity(it->get_input_symbol()))
{
- istr = lookup_path.at(lookup_index);
+ istr = lookup_path.at(lookup_index);
ostr = istr;
}
else
@@ -3857,17 +3665,17 @@
ostr = std::string("?");
else*/
ostr = it->get_output_symbol();
- }
+ }
push_back_to_two_level_path
(path_so_far,
StringPair(istr, ostr),
- it->get_weight());
-
+ it->get_weight(), flag_diacritic_path);
+
HfstEpsilonHandler * Ehp = NULL;
if (input_symbol_consumed) {
lookup_index++;
- Ehp = new HfstEpsilonHandler(infinite_cutoff);
+ Ehp = new HfstEpsilonHandler(max_epsilon_cycles);
}
else {
Eh.push_back(state);
@@ -3875,10 +3683,10 @@
}
// call lookup for the target state of the transition
- lookup_fd(lookup_path, results, it->get_target_state(),
- lookup_index, path_so_far, alphabet, *Ehp, infinite_cutoff, max_weight);
+ lookup(lookup_path, results, it->get_target_state(),
+ lookup_index, path_so_far, alphabet, *Ehp, max_epsilon_cycles, max_weight, flag_diacritic_path);
- // return to the original values of path_so_far
+ // return to the original values of path_so_far
// and lookup_index
if (input_symbol_consumed) {
lookup_index--;
@@ -3889,38 +3697,44 @@
// of Eh is automatically called next
}
- pop_back_from_two_level_path(path_so_far, it->get_weight());
+ pop_back_from_two_level_path(path_so_far, it->get_weight(), flag_diacritic_path);
}
}
}
- HFSTDLL void lookup_fd
+ void HfstBasicTransducer::lookup
(const StringVector &lookup_path,
HfstTwoLevelPaths &results,
- size_t * infinite_cutoff = NULL,
- float * max_weight = NULL)
+ size_t * max_epsilon_cycles /*= NULL*/,
+ float * max_weight /*= NULL*/,
+ bool obey_flags /*= false*/)
{
HfstState state = 0;
unsigned int lookup_index = 0;
HfstTwoLevelPath path_so_far;
StringSet alphabet = this->get_alphabet();
- if (infinite_cutoff != NULL)
+ StringVector * flag_diacritic_path = (obey_flags)? new StringVector() : NULL;
+
+ if (max_epsilon_cycles != NULL)
{
- HfstEpsilonHandler Eh(*infinite_cutoff);
- lookup_fd(lookup_path, results, state, lookup_index, path_so_far,
- alphabet, Eh, *infinite_cutoff, max_weight);
+ HfstEpsilonHandler Eh(*max_epsilon_cycles);
+ lookup(lookup_path, results, state, lookup_index, path_so_far,
+ alphabet, Eh, *max_epsilon_cycles, max_weight, flag_diacritic_path);
}
else
{
HfstEpsilonHandler Eh(100000);
- lookup_fd(lookup_path, results, state, lookup_index, path_so_far,
- alphabet, Eh, 100000, max_weight);
+ lookup(lookup_path, results, state, lookup_index, path_so_far,
+ alphabet, Eh, 100000, max_weight, flag_diacritic_path);
}
+
+ if (flag_diacritic_path != NULL)
+ delete flag_diacritic_path;
}
- HFSTDLL void check_regexp_state_for_cycle(HfstState s, const std::set<HfstState> & states_visited)
+ void HfstBasicTransducer::check_regexp_state_for_cycle(HfstState s, const std::set<HfstState> & states_visited)
{
if (states_visited.find(s) != states_visited.end())
{
@@ -3929,7 +3743,7 @@
}
// Returns whether tr is "^]":"^]". If tr is not allowed, throws an error message.
- HFSTDLL bool check_regexp_transition_end(const HfstBasicTransition & tr, bool input_side)
+ bool HfstBasicTransducer::check_regexp_transition_end(const HfstBasicTransition & tr, bool input_side)
{
std::string istr = tr.get_input_symbol();
std::string ostr = tr.get_output_symbol();
@@ -3941,8 +3755,8 @@
else if ((input_side && is_special_symbol(istr)) || (!input_side && is_special_symbol(ostr)))
{
throw "error: special symbol detected in compile-replace regular expression";
- }
- else
+ }
+ else
{}
if ((input_side && ("^[" == istr)) || (!input_side && ("^[" == ostr)))
@@ -3968,10 +3782,10 @@
// [x:y]* "^]" (x and y cannot be "^]" or "^[") starting from state \a s. The resulting
// paths are stored in \a full_paths. \a path is used to keep track of each path so
// far. Weights are currently ignored.
- HFSTDLL void find_regexp_paths
- (HfstState s,
- std::set<HfstState> & states_visited,
- std::vector<std::pair<std::string, std::string> > & path,
+ void HfstBasicTransducer::find_regexp_paths
+ (HfstState s,
+ std::set<HfstState> & states_visited,
+ std::vector<std::pair<std::string, std::string> > & path,
HfstReplacements & full_paths, bool input_side)
{
// no cycles allowed inside "^[" and "^]"
@@ -3979,9 +3793,9 @@
states_visited.insert(s);
// go through all transitions
- const HfstBasicTransducer::HfstTransitions &transitions
+ const HfstBasicTransitions &transitions
= this->operator[](s);
- for (HfstBasicTransducer::HfstTransitions::const_iterator it
+ for (HfstBasicTransitions::const_iterator it
= transitions.begin();
it != transitions.end(); it++)
{
@@ -4006,7 +3820,7 @@
path,
full_paths, input_side);
path.pop_back();
- }
+ }
}
// all transitions gone through
states_visited.erase(s);
@@ -4021,15 +3835,15 @@
// or loops are encountered on a regexp path. Final states are allowed on regexp paths as they are also
// allowed by Xerox tools.
// Weights are currently ignored.
- HFSTDLL void find_regexp_paths
+ void HfstBasicTransducer::find_regexp_paths
(HfstState s,
- std::vector<std::pair<HfstState, std::vector<std::pair<std::string, std::string> > > > & full_paths,
+ std::vector<std::pair<HfstState, std::vector<std::pair<std::string, std::string> > > > & full_paths,
bool input_side)
{
// go through all transitions
- const HfstBasicTransducer::HfstTransitions &transitions
+ const HfstBasicTransitions &transitions
= this->operator[](s);
- for (HfstBasicTransducer::HfstTransitions::const_iterator it
+ for (HfstBasicTransitions::const_iterator it
= transitions.begin();
it != transitions.end(); it++)
{
@@ -4043,7 +3857,7 @@
}*/
std::set<HfstState> states_visited;
states_visited.insert(s);
- std::vector<std::pair<std::string, std::string> > path;
+ std::vector<std::pair<std::string, std::string> > path;
path.push_back(std::pair<std::string, std::string>(istr, ostr));
find_regexp_paths(it->get_target_state(), states_visited, path, full_paths, input_side);
//fprintf(stderr, "%u regexp paths found for state %u\n", (unsigned int)full_paths.size(), s); // debug
@@ -4054,7 +3868,7 @@
// Find all subpaths of form "^[" [x:y]* "^]" (x and y cannot be "^[" or "^]") and return them.
// retval[start_state] == vector(pair(end_state, vector(pair(isymbol,osymbol) ) ) )
// Weights are currently ignored.
- HFSTDLL HfstReplacementsMap find_replacements(bool input_side)
+ HfstReplacementsMap HfstBasicTransducer::find_replacements(bool input_side)
{
HfstReplacementsMap replacements;
unsigned int state = 0;
@@ -4078,20 +3892,20 @@
// \a graph to state \a state2 with a weight of that state's final weight. Final states of
// \a graph as well as its initial state are made normal non-final states when copying \a graph.
// Todo: copy alphabet? harmonize graphs?
- HFSTDLL void insert_transducer(HfstState state1, HfstState state2, const HfstTransitionGraph & graph)
+ void HfstBasicTransducer::insert_transducer(HfstState state1, HfstState state2, const HfstBasicTransducer & graph)
{
- HfstState offset = add_state();
+ HfstState offset = add_state();
HfstState source_state=0;
for (const_iterator it = graph.begin();
it != graph.end(); it++)
{
- for (typename HfstTransitions::const_iterator tr_it
+ for (HfstBasicTransitions::const_iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
- C data = tr_it->get_transition_data();
+ HfstTropicalTransducerTransitionData data = tr_it->get_transition_data();
- HfstTransition <C> transition
+ HfstBasicTransition transition
(tr_it->get_target_state() + offset,
data.get_input_symbol(),
data.get_output_symbol(),
@@ -4103,19 +3917,19 @@
}
// Epsilon transitions
- for (typename FinalWeightMap::const_iterator it
+ for (FinalWeightMap::const_iterator it
= graph.final_weight_map.begin();
it != graph.final_weight_map.end(); it++)
{
- HfstTransition <C> epsilon_transition
- (state2, C::get_epsilon(), C::get_epsilon(),
+ HfstBasicTransition epsilon_transition
+ (state2, HfstTropicalTransducerTransitionData::get_epsilon(), HfstTropicalTransducerTransitionData::get_epsilon(),
it->second);
add_transition(it->first + offset, epsilon_transition);
}
// Initial transition
- HfstTransition <C> epsilon_transition
- (offset, C::get_epsilon(), C::get_epsilon(), 0);
+ HfstBasicTransition epsilon_transition
+ (offset, HfstTropicalTransducerTransitionData::get_epsilon(), HfstTropicalTransducerTransitionData::get_epsilon(), 0);
add_transition(state1, epsilon_transition);
}
@@ -4125,9 +3939,9 @@
// Find target state corresponding to state pair \a target1, \a target2 in \a state_map and return that state.
// If not found, add a new state to \a intersection, add it to \a state_map and return it.
// \a was_new_state specifies whether a new state was added.
- static HfstState find_target_state
- (HfstState target1, HfstState target2, StateMap & state_map,
- HfstTransitionGraph & intersection, bool & was_new_state)
+ HfstState HfstBasicTransducer::find_target_state
+ (HfstState target1, HfstState target2, StateMap & state_map,
+ HfstBasicTransducer & intersection, bool & was_new_state)
{
StatePair state_pair(target1, target2);
StateMap::const_iterator it = state_map.find(state_pair);
@@ -4145,9 +3959,9 @@
// A function used by find_matches.
// Copy matching transition tr1/tr2 to state \a state in \a intersection and return
// the target state of that transition. Also make that state final, if needed.
- HFSTDLL static HfstState handle_match(const HfstTransitionGraph & graph1, const HfstTransition <C> & tr1,
- const HfstTransitionGraph & graph2, const HfstTransition <C> & tr2,
- HfstTransitionGraph & intersection, HfstState state, StateMap & state_map)
+ HfstState HfstBasicTransducer::handle_match(const HfstBasicTransducer & graph1, const HfstBasicTransition & tr1,
+ const HfstBasicTransducer & graph2, const HfstBasicTransition & tr2,
+ HfstBasicTransducer & intersection, HfstState state, StateMap & state_map)
{
HfstState target1 = tr1.get_target_state();
@@ -4158,8 +3972,8 @@
// The sum of weight is copied to the resulting intersection.
float transition_weight = tr1.get_weight() + tr2.get_weight();
intersection.add_transition
- (state, HfstTransition <C>
- (retval, tr1.get_input_symbol(), tr1.get_output_symbol(), transition_weight));
+ (state, HfstBasicTransition
+ (retval, tr1.get_input_symbol(), tr1.get_output_symbol(), transition_weight));
// For each new state added, check if the corresponding states in \a graph1 and \a graph2
// are final. If they are, make the new state final with the sum of final weights.
if (was_new_state && (graph1.is_final_state(target1) && graph2.is_final_state(target2)))
@@ -4184,13 +3998,13 @@
//
// @pre \a graph1 and \a graph2 must be arc-sorted (via sort_arcs()) to make transition matching faster.
// @pre \a graph1 and \a graph2 must be deterministic. (todo: handle equivalent transitions, maybe even epsilons?)
- HFSTDLL static void find_matches
- (HfstTransitionGraph & graph1, HfstState state1, HfstTransitionGraph & graph2, HfstState state2,
- HfstTransitionGraph & intersection, HfstState state, StateMap & state_map, std::set<HfstState> & agenda)
+ void HfstBasicTransducer::find_matches
+ (HfstBasicTransducer & graph1, HfstState state1, HfstBasicTransducer & graph2, HfstState state2,
+ HfstBasicTransducer & intersection, HfstState state, StateMap & state_map, std::set<HfstState> & agenda)
{
agenda.insert(state); // do not handle \a state twice
- HfstTransitions & tr1 = graph1.state_vector[state1]; // transitions of graph1
- HfstTransitions & tr2 = graph2.state_vector[state2]; // transitions of graph2
+ HfstBasicTransitions & tr1 = graph1.state_vector[state1]; // transitions of graph1
+ HfstBasicTransitions & tr2 = graph2.state_vector[state2]; // transitions of graph2
if (tr1.size() == 0 || tr2.size() == 0)
{
@@ -4205,16 +4019,16 @@
// than tr1[i], a match for tr1[i+1] can be searched starting from tr2[j]. ***
for (unsigned int i=0; i < tr1.size(); i++)
{
- HfstTransition <C> & transition1 = tr1[i];
+ HfstBasicTransition & transition1 = tr1[i];
// Transition data (input and output symbols) to be compared.
- const C & transition_data1 = transition1.get_transition_data();
+ const HfstTropicalTransducerTransitionData & transition_data1 = transition1.get_transition_data();
// --- Go through tr2 starting from start_search_from. ---
for(unsigned int j=start_search_from; j < tr2.size(); j++)
{
- HfstTransition <C> & transition2 = tr2[j];
+ HfstBasicTransition & transition2 = tr2[j];
// Transition data (input and output symbols) to be compared.
- const C & transition_data2 = transition2.get_transition_data();
+ const HfstTropicalTransducerTransitionData & transition_data2 = transition2.get_transition_data();
// todo: input:output duplicates with different weights? (lower weight is chosen always?)
if (transition_data2.less_than_ignore_weight(transition_data1))
{
@@ -4247,10 +4061,10 @@
return;
}
- HFSTDLL static HfstTransitionGraph intersect
- (HfstTransitionGraph & graph1, HfstTransitionGraph & graph2)
+ HfstBasicTransducer HfstBasicTransducer::intersect
+ (HfstBasicTransducer & graph1, HfstBasicTransducer & graph2)
{
- HfstTransitionGraph retval;
+ HfstBasicTransducer retval;
StateMap state_map;
std::set<HfstState> agenda;
graph1.sort_arcs();
@@ -4271,9 +4085,9 @@
// A function used by find_matches_for_merge
// Copy matching transition graph_tr/merger_tr to state \a result_state in \a result and return
// the target state of that transition. Also make that state final, if needed.
- HFSTDLL static HfstState handle_non_list_match(const HfstTransitionGraph & graph, const HfstTransition <C> & graph_transition,
- const HfstTransitionGraph & merger, HfstState merger_target,
- HfstTransitionGraph & result, HfstState result_state, StateMap & state_map)
+ HfstState HfstBasicTransducer::handle_non_list_match(const HfstBasicTransducer & graph, const HfstBasicTransition & graph_transition,
+ const HfstBasicTransducer & merger, HfstState merger_target,
+ HfstBasicTransducer & result, HfstState result_state, StateMap & state_map)
{
HfstState graph_target = graph_transition.get_target_state();
@@ -4281,7 +4095,7 @@
HfstState retval = find_target_state
(graph_target, merger_target, state_map, result, was_new_state);
result.add_transition
- (result_state, HfstTransition <C>
+ (result_state, HfstBasicTransition
(retval, graph_transition.get_input_symbol(), graph_transition.get_output_symbol(), graph_transition.get_weight()));
// For each new state added, check if the corresponding states in \a graph and \a merger
// are final. If they are, make the new state final with the sum of final weights.
@@ -4297,9 +4111,9 @@
// A function used by find_matches_for_merge
// Copy matching transition graph_tr/merger_tr to state \a result_state in \a result and return
// the target state of that transition. Also make that state final, if needed.
- HFSTDLL static HfstState handle_list_match(const HfstTransitionGraph & graph, const HfstTransition <C> & graph_transition,
- const HfstTransitionGraph & merger, const HfstTransition <C> & merger_transition,
- HfstTransitionGraph & result, HfstState result_state, StateMap & state_map, std::set<std::string> & markers_added)
+ HfstState HfstBasicTransducer::handle_list_match(const HfstBasicTransducer & graph, const HfstBasicTransition & graph_transition,
+ const HfstBasicTransducer & merger, const HfstBasicTransition & merger_transition,
+ HfstBasicTransducer & result, HfstState result_state, StateMap & state_map, std::set<std::string> & markers_added)
{
HfstState graph_target = graph_transition.get_target_state();
HfstState merger_target = merger_transition.get_target_state();
@@ -4312,13 +4126,13 @@
// testing: add a marker
HfstState extra_state = result.add_state();
result.add_transition
- (result_state, HfstTransition <C>
+ (result_state, HfstBasicTransition
(extra_state, "@" + graph_transition.get_input_symbol() + "@", "@" + graph_transition.get_output_symbol() + "@", 0));
markers_added.insert("@" + graph_transition.get_input_symbol() + "@");
result.add_transition
- (extra_state /*result_state*/, HfstTransition <C>
- (retval, merger_transition.get_input_symbol(), merger_transition.get_output_symbol(), transition_weight));
+ (extra_state /*result_state*/, HfstBasicTransition
+ (retval, merger_transition.get_input_symbol(), merger_transition.get_output_symbol(), transition_weight));
// For each new state added, check if the corresponding states in \a graph1 and \a graph2
// are final. If they are, make the new state final with the sum of final weights.
if (was_new_state && (graph.is_final_state(graph_target) && merger.is_final_state(merger_target)))
@@ -4331,7 +4145,7 @@
- HFSTDLL static bool is_list_symbol(const C & transition_data, const std::map<std::string, std::set<std::string> > & list_symbols)
+ bool HfstBasicTransducer::is_list_symbol(const HfstTropicalTransducerTransitionData & transition_data, const std::map<std::string, std::set<std::string> > & list_symbols)
{
std::string isymbol = transition_data.get_input_symbol();
std::string osymbol = transition_data.get_output_symbol();
@@ -4346,7 +4160,7 @@
/*
// @pre \a transition_data is a list symbol
// @pre list symbols cannot contain '_' or '@'
- static std::set<std::string> get_list_symbols(const std::string & list_symbol)
+ std::set<std::string> get_list_symbols(const std::string & list_symbol)
{
std::set<std::string> result;
unsigned int i = 6;
@@ -4391,14 +4205,14 @@
//
// @pre \a graph and \a merger must be arc-sorted (via sort_arcs()) to make transition matching faster.
// @pre \a graph and \a merger must be deterministic. (todo: handle equivalent transitions, maybe even epsilons?)
- HFSTDLL static void find_matches_for_merge
- (HfstTransitionGraph & graph, HfstState graph_state, HfstTransitionGraph & merger, HfstState merger_state,
- HfstTransitionGraph & result, HfstState result_state, StateMap & state_map, std::set<HfstState> & agenda,
+ void HfstBasicTransducer::find_matches_for_merge
+ (HfstBasicTransducer & graph, HfstState graph_state, HfstBasicTransducer & merger, HfstState merger_state,
+ HfstBasicTransducer & result, HfstState result_state, StateMap & state_map, std::set<HfstState> & agenda,
const std::map<std::string, std::set<std::string> > & list_symbols, std::set<std::string> & markers_added)
{
agenda.insert(result_state); // do not handle \a result_state twice
- HfstTransitions & graph_transitions = graph.state_vector[graph_state]; // transitions of graph
- HfstTransitions & merger_transitions = merger.state_vector[merger_state]; // transitions of merger
+ HfstBasicTransitions & graph_transitions = graph.state_vector[graph_state]; // transitions of graph
+ HfstBasicTransitions & merger_transitions = merger.state_vector[merger_state]; // transitions of merger
if (graph_transitions.size() == 0)
{
@@ -4408,8 +4222,8 @@
// Go through all transitions in state \a graph_state of \a graph.
for (unsigned int i=0; i < graph_transitions.size(); i++)
{
- HfstTransition <C> & graph_transition = graph_transitions[i];
- const C & graph_transition_data = graph_transition.get_transition_data();
+ HfstBasicTransition & graph_transition = graph_transitions[i];
+ const HfstTropicalTransducerTransitionData & graph_transition_data = graph_transition.get_transition_data();
// List symbols must be checked separately
if (is_list_symbol(graph_transition_data, list_symbols))
@@ -4419,8 +4233,8 @@
// Find all matches
for(unsigned int j=0; j < merger_transitions.size(); j++)
{
- HfstTransition <C> & merger_transition = merger_transitions[j];
- const C & merger_transition_data = merger_transition.get_transition_data();
+ HfstBasicTransition & merger_transition = merger_transitions[j];
+ const HfstTropicalTransducerTransitionData & merger_transition_data = merger_transition.get_transition_data();
const std::string & isymbol = merger_transition_data.get_input_symbol();
const std::string & osymbol = merger_transition_data.get_output_symbol();
@@ -4451,16 +4265,16 @@
{
find_matches_for_merge(graph, graph_transition.get_target_state(), merger, /*merger_transition.get_target_state()*/ merger_state, result, target, state_map, agenda, list_symbols, markers_added);
}
- // --- A transition in graph compared for all corresponding transitions in merger, compare next transition. ---
+ // --- A transition in graph compared for all corresponding transitions in merger, compare next transition. ---
}
// *** All transitions in state gone through. ***
return;
}
- HFSTDLL static HfstTransitionGraph merge
- (HfstTransitionGraph & graph, HfstTransitionGraph & merger, const std::map<std::string, std::set<std::string> > & list_symbols, std::set<std::string> & markers_added)
+ HfstBasicTransducer HfstBasicTransducer::merge
+ (HfstBasicTransducer & graph, HfstBasicTransducer & merger, const std::map<std::string, std::set<std::string> > & list_symbols, std::set<std::string> & markers_added)
{
- HfstTransitionGraph result;
+ HfstBasicTransducer result;
StateMap state_map;
std::set<HfstState> agenda;
graph.sort_arcs();
@@ -4473,7 +4287,7 @@
result.set_final_weight(0, final_weight);
}
- try
+ try
{
find_matches_for_merge(graph, 0, merger, 0, result, 0, state_map, agenda, list_symbols, markers_added);
}
@@ -4485,53 +4299,54 @@
return result;
}
+ };
+ };
+#else // MAIN_TEST was defined
+#include <iostream>
+int main(int argc, char * argv[])
+{
+ using namespace hfst::implementations;
+ std::cout << "Unit tests for " __FILE__ ":" << std::endl;
+ return EXIT_SUCCESS;
+ HfstBasicTransducer g1;
+ g1.add_state(1);
+ g1.add_transition(0, HfstBasicTransition(1, "a", "a", 0.3));
+ g1.add_transition(0, HfstBasicTransition(1, "b", "b", 0.1));
+ //g1.add_transition(0, HfstBasicTransition(1, "@_IDENTITY_SYMBOL_@", "@_IDENTITY_SYMBOL_@", 0.1));
+ g1.add_transition(0, HfstBasicTransition(1, "c", "c", 0.1));
+ g1.add_transition(0, HfstBasicTransition(1, "d", "d", 0.1));
+ g1.set_final_weight(1, 0.5);
+ //g1.write_in_att_format(stderr);
+ //std::cerr << "--" << std::endl;
+ HfstBasicTransducer subst1;
+ subst1.add_state(1);
+ subst1.add_transition(0, HfstBasicTransition(1, "A", "A", 0.2));
+ subst1.set_final_weight(1, 0.5);
-/* /\** @brief Determine whether this graph has input-epsilon cycles. */
-/* *\/ */
-/* bool has_input_epsilon_cycles(void) */
-/* { */
-/* typedef std::map<HfstState, */
-/* std::set<HfstTransition<C> > > */
-/* HfstStates; */
-/* HfstStates state_map; */
-
-/* std::set<HfstState> total_seen; */
-/* for (state_vector::iterator it = state_vector.begin(); */
-/* it != state_vector.end(); ++it) { */
-/* if (total_seen.count(*it) != 0) { */
-/* continue; */
-/* } */
-
-/* } */
-/* } */
-
+ HfstBasicTransducer subst2;
+ subst2.add_state(1);
+ subst2.add_transition(0, HfstBasicTransition(1, "B", "B", 0.2));
+ subst2.set_final_weight(1, 0.5);
- // --- Friends ---
+ std::map<std::string, HfstBasicTransducer> subst_map;
+ subst_map["a"] = subst1;
+ subst_map["b"] = subst2;
+ g1.substitute(subst_map, false);
+ //std::pair<std::string, std::string> p = std::pair<std::string, std::string>("a", "a");
+ //g1.substitute(p, subst);
- friend class ConversionFunctions;
- friend class hfst::HarmonizeUnknownAndIdentitySymbols;
- };
-
- /** @brief An HfstTransitionGraph with transitions of type
- HfstTropicalTransducerTransitionData and weight type float.
-
- This is probably the most useful kind of HfstTransitionGraph. */
- typedef HfstTransitionGraph <HfstTropicalTransducerTransitionData>
- HfstBasicTransducer;
-
- /** @brief A specialization for faster conversion. */
- //typedef HfstTransitionGraph <HfstFastTransitionData>
- // HfstFastTransducer;
+ g1.remove_transition(0, HfstBasicTransition(1, "d", "d", 0.1));
- }
-
- }
+ g1.write_in_att_format(stdout);
+ std::cout << std::endl << "ok" << std::endl;
+ return EXIT_SUCCESS;
+}
-#endif // #ifndef _HFST_TRANSITION_GRAPH_H_
+#endif // MAIN_TEST
diff --git a/libhfst/src/implementations/HfstBasicTransducer.h b/libhfst/src/implementations/HfstBasicTransducer.h
new file mode 100644
index 0000000..2fd90e7
--- /dev/null
+++ b/libhfst/src/implementations/HfstBasicTransducer.h
@@ -0,0 +1,1120 @@
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 3 of the License, or (at your option) any later version.
+// See the file COPYING included with this distribution for more
+// information.
+
+ #ifndef _HFST_BASIC_TRANSDUCER_H_
+ #define _HFST_BASIC_TRANSDUCER_H_
+
+ /** @file HfstBasicTransducer.h
+ @brief Class HfstBasicTransducer */
+
+ #include <cstdio>
+ #include <string>
+ #include <set>
+ #include <cassert>
+ #include <iosfwd>
+ #include <algorithm>
+ #include <stack>
+
+ #include "../HfstSymbolDefs.h"
+ #include "../HfstExceptionDefs.h"
+ #include "../HfstDataTypes.h"
+ #include "../HarmonizeUnknownAndIdentitySymbols.h"
+ #include "../HfstFlagDiacritics.h"
+ #include "../HfstLookupFlagDiacritics.h"
+ #include "../HfstEpsilonHandler.h"
+ #include "ConvertTransducerFormat.h"
+ #include "HfstBasicTransition.h"
+ #include "HfstTropicalTransducerTransitionData.h"
+ #include "../string-utils.h"
+
+ #include "../hfstdll.h"
+
+ namespace hfst {
+
+ /** @brief A namespace for all code that forms a bridge between
+ backend libraries and HFST.
+
+ Most code in this namespace is not intended to be accessed
+ by the user of the HFST interface. An exception is HFST's
+ own transducer class hfst::HfstTransitionGraph and classes that
+ are needed to use it.
+ */
+ namespace implementations {
+
+ /** @brief The number of a state in an HfstTransitionGraph. */
+ typedef unsigned int HfstState;
+
+ typedef std::pair<HfstState, std::vector<std::pair<std::string, std::string> > > HfstReplacement;
+ typedef std::vector<HfstReplacement> HfstReplacements;
+ typedef std::map<HfstState, HfstReplacements > HfstReplacementsMap;
+
+ /** @brief Datatype for the states of a transition in a graph. */
+ typedef std::vector<hfst::implementations::HfstBasicTransition> HfstBasicTransitions;
+ /* Datatype for the states of a graph and their transitions.
+ Each index of the vector is a state and the transitions
+ on that index are the transitions of that state. */
+ typedef std::vector<hfst::implementations::HfstBasicTransitions> HfstBasicStates;
+
+ /** @brief A simple transition graph format that consists of
+ states and transitions between those states.
+
+ Probably the easiest way to use this template is to choose
+ the implementations #HfstBasicTransducer
+ (HfstTransitionGraph<HfstTropicalTransducerTransitionData>)
+ and #HfstBasicTransition
+ (HfstTransition<HfstTropicalTransducerTransitionData>).
+ The class HfstTropicalTransducerTransitionData contains an input string,
+ an output string and a float weight. HfstBasicTransducer is the
+ implementation that is used as an example in this documentation.
+
+ An example of creating a HfstBasicTransducer [foo:bar baz:baz]
+ with weight 0.4 from scratch:
+
+ \verbatim
+ // Create an empty transducer
+ // The transducer has initially one start state (number zero)
+ // that is not final
+ HfstBasicTransducer fsm;
+ // Add two states to the transducer
+ fsm.add_state(1);
+ fsm.add_state(2);
+ // Create a transition [foo:bar] leading to state 1 with weight 0.1 ...
+ HfstBasicTransition tr(1, "foo", "bar", 0.1);
+ // ... and add it to state zero
+ fsm.add_transition(0, tr);
+ // Add a transition [baz:baz] with weight 0 from state 1 to state 2
+ fsm.add_transition(1, HfstBasicTransition(2, "baz", "baz", 0.0));
+ // Set state 2 as final with weight 0.3
+ fsm.set_final_weight(2, 0.3);
+ \endverbatim
+
+ An example of iterating through a HfstBasicTransducer's states
+ and transitions when printing it in AT&T format to stderr:
+
+ \verbatim
+ // The first state is always number zero.
+ unsigned int source_state=0;
+
+ // Go through all states
+ for (HfstBasicTransducer::const_iterator it = fsm.begin();
+ it != fsm.end(); it++ )
+ {
+ // Go through all transitions
+ for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
+ = it->begin(); tr_it != it->end(); tr_it++)
+ {
+ std::cerr << source_state << "\t"
+ << tr_it->get_target_state() << "\t"
+ << tr_it->get_input_symbol() << "\t"
+ << tr_it->get_output_symbol() << "\t"
+ << tr_it->get_weight() << std::endl;
+ }
+
+ if (fsm.is_final_state(source_state))
+ {
+ std::cerr << source_state << "\t"
+ << fsm.get_final_weight(source_state) << std::endl;
+ }
+
+ // the next state is numbered source_state + 1
+ source_state++;
+ }
+ \endverbatim
+
+ @see #HfstBasicTransducer HfstBasicTransition */
+ class HfstBasicTransducer
+ {
+
+ // --- Datatypes and variables ---
+
+ public:
+ /** @brief Datatype for a symbol in a transition. */
+ typedef HfstTropicalTransducerTransitionData::SymbolType HfstSymbol;
+ /** @brief Datatype for a symbol pair in a transition. */
+ typedef std::pair<HfstSymbol, HfstSymbol>
+ HfstSymbolPair;
+ /** @brief A set of symbol pairs. */
+ typedef std::set<HfstSymbolPair> HfstSymbolPairSet;
+ /** @brief A set of symbol pairs. */
+ typedef std::set<HfstSymbol> HfstSymbolSet;
+ /** @brief A vector of symbol pairs. */
+ typedef std::vector<HfstSymbolPair> HfstSymbolPairVector;
+ /** @brief Datatype for the alphabet of a graph. */
+ typedef std::set<HfstSymbol> HfstAlphabet;
+
+ /* States of the graph and their transitions. */
+ HfstBasicStates state_vector;
+
+ protected:
+ /* The initial state number. */
+ static const HfstState INITIAL_STATE = 0;
+
+ /* Datatype for the final states and their weights in a graph. */
+ typedef std::map<HfstState,HfstTropicalTransducerTransitionData::WeightType> FinalWeightMap;
+ /* The final states and their weights in the graph. */
+ FinalWeightMap final_weight_map;
+
+ /* The alphabet of the graph. */
+ HfstAlphabet alphabet;
+
+ /* Used by substitute function. */
+ typedef unsigned int HfstNumber;
+ typedef std::vector<HfstNumber> HfstNumberVector;
+ typedef std::pair<HfstNumber, HfstNumber> HfstNumberPair;
+ typedef std::map<HfstNumberPair, HfstNumberPair> HfstNumberPairSubstitutions;
+
+ protected:
+ /* @brief An iterator type that points to a state in a graph.
+
+ The value pointed by the iterator is of type HfstBasicTransitions. */
+ typedef HfstBasicStates::iterator iterator;
+
+ public:
+ /** @brief A const iterator type that points a state in a graph.
+
+ The value pointed by the iterator is of type HfstBasicTransitions. */
+ typedef HfstBasicStates::const_iterator const_iterator;
+
+ /** @brief The name of the graph. */
+ std::string name;
+
+ /** @brief The states of the graph. */
+ std::vector<HfstState> states() const;
+
+ /** @brief The states of the graph and their transitions. */
+ const HfstBasicStates & states_and_transitions() const;
+
+ /** @brief The states of the graph and their transitions, mutable. */
+ HfstBasicStates & states_and_transitions();
+
+ // --------------------------------------------------------
+ // --- Construction, assignment, copying and conversion ---
+ // --------------------------------------------------------
+
+ /** @brief Create a graph with one initial state that has state number
+ zero and is not a final state, i.e. create an empty graph. */
+ HFSTDLL HfstBasicTransducer(void);
+
+ HFSTDLL HfstBasicTransducer(FILE *file);
+
+
+ /** @brief The assignment operator. */
+ HFSTDLL HfstBasicTransducer &operator=(const HfstBasicTransducer &graph);
+
+ HFSTDLL HfstBasicTransducer &assign(const HfstBasicTransducer &graph);
+
+ /** @brief Create a deep copy of HfstBasicTransducer \a graph. */
+ HFSTDLL HfstBasicTransducer(const HfstBasicTransducer &graph);
+
+ /** @brief Create an HfstBasicTransducer equivalent to HfstTransducer
+ \a transducer. FIXME: move to a separate file */
+ HFSTDLL HfstBasicTransducer(const hfst::HfstTransducer &transducer);
+
+ // --------------------------------------------------
+ // --- Initialization, optimization and debugging ---
+ // --------------------------------------------------
+
+ protected:
+ /* Add epsilon, unknown and identity symbols to the alphabet
+ \a alpha. */
+ void initialize_alphabet(HfstAlphabet &alpha);
+
+ /* Check that all symbols that occur in the transitions of the graph
+ are also in the alphabet. */
+ bool check_alphabet();
+
+ public:
+ /* Print the alphabet of the graph to standard error stream. */
+ HFSTDLL void print_alphabet() const;
+
+ protected:
+ /* Get the number of the \a symbol. */
+ unsigned int get_symbol_number
+ (const HfstSymbol &symbol) const;
+
+ /* For internal optimization: Reserve space for
+ \a number_of_states states. */
+ void initialize_state_vector
+ (unsigned int number_of_states);
+
+ /* For internal optimization: Reserve space for
+ \a number_of_transitions transitions for state number
+ \a state_number. */
+ void initialize_transition_vector
+ (unsigned int state_number, unsigned int number_of_transitions);
+
+ // -----------------------------------
+ // ---------- The alphabet -----------
+ // -----------------------------------
+
+ public:
+ /** @brief Explicitly add \a symbol to the alphabet of the graph.
+
+ @note Usually the user does not have to take care of the alphabet
+ of a graph. This function can be useful in some special cases. */
+ HFSTDLL void add_symbol_to_alphabet(const HfstSymbol &symbol);
+
+ /** @brief Remove symbol \a symbol from the alphabet of the graph.
+
+ @note Use with care, removing symbols that occur in the transitions
+ of the graph can have unexpected results. */
+ HFSTDLL void remove_symbol_from_alphabet(const HfstSymbol &symbol);
+
+ HFSTDLL void remove_symbols_from_alphabet(const HfstSymbolSet &symbols);
+
+ /** @brief Same as #add_symbol_to_alphabet for each symbol in
+ \a symbols. */
+ HFSTDLL void add_symbols_to_alphabet(const HfstSymbolSet &symbols);
+
+ HFSTDLL void add_symbols_to_alphabet(const HfstSymbolPairSet &symbols);
+
+ /* Remove all symbols that are given in \a symbols but do not occur
+ in transitions of the graph from its alphabet. */
+ HFSTDLL void prune_alphabet_after_substitution(const std::set<unsigned int> &symbols);
+
+ HFSTDLL HfstAlphabet symbols_used();
+
+ /** @brief Remove all symbols that do not occur in transitions of
+ the graph from its alphabet.
+
+ @param force Whether unused symbols are removed even if
+ unknown or identity symbols occur in transitions.
+
+ Epsilon, unknown and identity \link hfst::String symbols\endlink
+ are always included in the alphabet. */
+ HFSTDLL void prune_alphabet(bool force=true);
+
+ /** @brief Get the set of HfstSymbols in the alphabet
+ of the graph.
+
+ The HfstSymbols do not necessarily occur in any transitions
+ of the graph. Epsilon, unknown and identity \link
+ hfst::String symbols\endlink are always included in the alphabet. */
+ HFSTDLL const HfstAlphabet &get_alphabet() const;
+
+ HFSTDLL StringPairSet get_transition_pairs() const;
+
+ // ----------------------------------------------------------------
+ // --- Adding states and transitions and iterating through them ---
+ // ----------------------------------------------------------------
+
+ /** @brief Add a new state to this graph and return its number.
+
+ @return The next (smallest) free state number. */
+ HFSTDLL HfstState add_state(void);
+
+ /** @brief Add a state \a s to this graph.
+
+ If the state already exists, it is not added again.
+ All states with state number smaller than \a s are also
+ added to the graph if they did not exist before.
+ @return \a s*/
+ HFSTDLL HfstState add_state(HfstState s);
+
+ /** @brief Get the biggest state number in use. */
+ HFSTDLL HfstState get_max_state() const;
+
+ /** @brief Add a transition \a transition to state \a s.
+
+ If state \a s does not exist, it is created. */
+ HFSTDLL void add_transition(HfstState s, const HfstBasicTransition & transition,
+ bool add_symbols_to_alphabet=true);
+
+ /** @brief Remove transition \a transition from state \a s.
+ \a remove_symbols_from_alphabet defines whether
+ symbols in \a transition are removed from the alphabet
+ if they are no longer used in the graph.
+
+ If \a state or \a transition does not exist, nothing is done. */
+ HFSTDLL void remove_transition(HfstState s, const HfstBasicTransition & transition,
+ bool remove_symbols_from_alphabet=false);
+
+ /** @brief Whether state \a s is final.
+ FIXME: return positive infinity instead if not final. */
+ HFSTDLL bool is_final_state(HfstState s) const;
+
+ /** Get the final weight of state \a s in this graph. */
+ HFSTDLL HfstTropicalTransducerTransitionData::WeightType get_final_weight(HfstState s) const;
+
+ /** @brief Set the final weight of state \a s in this graph
+ to \a weight.
+
+ If the state does not exist, it is created. */
+ HFSTDLL void set_final_weight(HfstState s,
+ const HfstTropicalTransducerTransitionData::WeightType & weight);
+
+ /** @brief Sort the arcs of this transducer according to input and
+ output symbols. */
+ HFSTDLL HfstBasicTransducer &sort_arcs(void);
+
+ /** @brief Get an iterator to the beginning of the states in
+ the graph.
+
+ For an example, see #HfstTransitionGraph */
+ HFSTDLL iterator begin();
+
+ /** @brief Get a const iterator to the beginning of
+ states in the graph. */
+ HFSTDLL const_iterator begin() const;
+
+ /** @brief Get an iterator to the end of states (last state + 1)
+ in the graph. */
+ HFSTDLL iterator end();
+
+ /** @brief Get a const iterator to the end of states (last state + 1)
+ in the graph. */
+ HFSTDLL const_iterator end() const;
+
+
+ /** @brief Get the set of transitions of state \a s in this graph.
+
+ If the state does not exist, a @a StateIndexOutOfBoundsException
+ is thrown.
+ */
+ HFSTDLL const HfstBasicTransitions & operator[](HfstState s) const;
+
+ /** @brief Alternative name for operator[].
+
+ Python interface uses this function as '[]' is not a legal name.
+
+ @see operator[]
+ */
+ HFSTDLL const HfstBasicTransitions & transitions(HfstState s) const;
+
+ /** @brief Get mutable transitions.
+ */
+ HFSTDLL HfstBasicTransitions & transitions(HfstState s);
+
+ // --------------------------------------------------
+ // ----- Reading and writing in AT&T format -----
+ // --------------------------------------------------
+
+ protected:
+ /* Change state numbers s1 to s2 and vice versa. */
+ void swap_state_numbers(HfstState s1, HfstState s2);
+
+ static void write_weight(FILE * file, float weight);
+
+ static void write_weight(std::ostream & os, float weight);
+
+ static void xfstize(std::string & symbol);
+
+ static void xfstize_symbol(std::string & symbol);
+
+ void print_xfst_state(std::ostream & os, HfstState state);
+
+ void print_xfst_state(FILE * file, HfstState state);
+
+ void print_xfst_arc(std::ostream & os, HfstTropicalTransducerTransitionData data);
+
+ void print_xfst_arc(FILE * file, HfstTropicalTransducerTransitionData data);
+
+ public:
+
+ /** @brief Write the graph in xfst text format to ostream \a os.
+ \a write_weights defines whether weights are printed (todo). */
+ HFSTDLL void write_in_xfst_format(std::ostream &os, bool write_weights=true);
+
+ // note: unknown and identity are both '?'
+ HFSTDLL static std::string prologize_symbol(const std::string & symbol);
+
+ // caveat: '?' is always unknown
+ HFSTDLL static std::string deprologize_symbol(const std::string & symbol);
+
+ HFSTDLL static void print_prolog_arc_symbols(FILE * file, HfstTropicalTransducerTransitionData data);
+
+ HFSTDLL static void print_prolog_arc_symbols(std::ostream & os, HfstTropicalTransducerTransitionData data);
+
+ /** @brief Write the graph in prolog format to FILE \a file.
+ \a write_weights defines whether weights are printed (todo). */
+ HFSTDLL void write_in_prolog_format(FILE * file, const std::string & name,
+ bool write_weights=true);
+
+ /** @brief Write the graph in prolog format to ostream \a os.
+ \a write_weights defines whether weights are printed (todo). */
+ HFSTDLL void write_in_prolog_format(std::ostream & os, const std::string & name,
+ bool write_weights=true);
+
+ // If \a str is of format ".+", change it to .+ and return true.
+ // Else, return false.
+ HFSTDLL static bool strip_quotes_from_both_sides(std::string & str);
+
+ // If \a str is of format .+)\.", change it to .+ and return true.
+ // Else, return false.
+ HFSTDLL static bool strip_ending_parenthesis_and_comma(std::string & str);
+
+ HFSTDLL static bool parse_prolog_network_line(const std::string & line, HfstBasicTransducer & graph);
+
+ // Get positions of \a c in \a str. If \a esc is precedes
+ // \a c, \a c is not included.
+ HFSTDLL static std::vector<unsigned int> get_positions_of_unescaped_char
+ (const std::string & str, char c, char esc);
+
+ // Extract input and output symbols, if possible, from prolog arc
+ // \a str and store them to \a isymbol and \a osymbol.
+ // Return whether symbols were successfully extracted.
+ // \a str must be of format "foo":"bar" or "foo"
+ HFSTDLL static bool get_prolog_arc_symbols
+ (const std::string & str, std::string & isymbol, std::string & osymbol);
+
+ HFSTDLL static bool extract_weight(std::string & symbol, float & weight);
+
+ HFSTDLL static bool parse_prolog_arc_line(const std::string & line, HfstBasicTransducer & graph);
+
+ HFSTDLL static bool parse_prolog_final_line(const std::string & line, HfstBasicTransducer & graph);
+
+ HFSTDLL static bool parse_prolog_symbol_line(const std::string & line, HfstBasicTransducer & graph);
+
+ // Erase newlines from the end of \a str and return \a str.
+ HFSTDLL static std::string strip_newlines(std::string & str);
+
+ // Try to get a line from \a is (if \a file == NULL)
+ // or from \a file. If successfull, strip the line from newlines,
+ // increment \a linecount by one and return the line.
+ // Else, throw an EndOfStreamException.
+ HFSTDLL static std::string get_stripped_line
+ (std::istream & is, FILE * file, unsigned int & linecount);
+
+ /* Create an HfstTransitionGraph as defined in prolog format
+ in istream \a is or FILE \a file.
+
+ The functions is called by functions
+ read_in_prolog_format(istream&) and
+ read_in_prolog_format(FILE*).
+ If \a file is NULL, it is ignored and \a is is used.
+ If \a file is not NULL, it is used and \a is is ignored. */
+ HFSTDLL static HfstBasicTransducer read_in_prolog_format
+ (std::istream &is, FILE *file, unsigned int & linecount);
+
+ HFSTDLL static HfstBasicTransducer read_in_prolog_format
+ (std::istream &is,
+ unsigned int & linecount);
+
+ HFSTDLL static HfstBasicTransducer read_in_prolog_format
+ (FILE *file,
+ unsigned int & linecount);
+
+ /** @brief Write the graph in xfst text format to FILE \a file.
+ \a write_weights defines whether weights are printed (todo). */
+ HFSTDLL void write_in_xfst_format(FILE * file, bool write_weights=true);
+
+ /** @brief Write the graph in AT&T format to ostream \a os.
+ \a write_weights defines whether weights are printed. */
+ HFSTDLL void write_in_att_format(std::ostream &os, bool write_weights=true);
+
+ /** @brief Write the graph in AT&T format to FILE \a file.
+ \a write_weights defines whether weights are printed. */
+ HFSTDLL void write_in_att_format(FILE *file, bool write_weights=true);
+
+ HFSTDLL void write_in_att_format(char * ptr, bool write_weights=true);
+
+ /** @brief Write the graph in AT&T format to FILE \a file using numbers
+ instead of symbol names.
+ \a write_weights defines whether weights are printed. */
+ HFSTDLL void write_in_att_format_number(FILE *file, bool write_weights=true);
+
+ bool add_att_line(char * line, const std::string & epsilon_symbol);
+
+ /* Create an HfstTransitionGraph as defined in AT&T format
+ in istream \a is or FILE \a file. \a epsilon_symbol defines
+ how epsilon is represented.
+
+ The functions is called by functions
+ read_in_att_format(istream&, std::string) and
+ read_in_att_format(FILE*, std::string).
+ If \a file is NULL, it is ignored and \a is is used.
+ If \a file is not NULL, it is used and \a is is ignored. */
+ HFSTDLL static HfstBasicTransducer read_in_att_format
+ (std::istream &is,
+ FILE *file,
+ std::string epsilon_symbol,
+ unsigned int & linecount);
+
+ /** @brief Create an HfstTransitionGraph as defined in AT&T
+ transducer format in istream \a is. \a epsilon_symbol
+ defines how epsilon is represented.
+ @pre \a is not at end, otherwise an exception is thrown.
+ @note Multiple AT&T transducer definitions are separated with
+ the line "--". */
+ HFSTDLL static HfstBasicTransducer read_in_att_format
+ (std::istream &is,
+ std::string epsilon_symbol,
+ unsigned int & linecount);
+
+ /** @brief Create an HfstTransitionGraph as defined
+ in AT&T transducer format in FILE \a file.
+ \a epsilon_symbol defines how epsilon is represented.
+ @pre \a is not at end, otherwise an exception is thrown.
+ @note Multiple AT&T transducer definitions are separated with
+ the line "--". */
+ HFSTDLL static HfstBasicTransducer read_in_att_format
+ (FILE *file,
+ std::string epsilon_symbol,
+ unsigned int & linecount);
+
+ // ----------------------------------------------
+ // ----- Substitution functions -----
+ // ----------------------------------------------
+
+ protected:
+
+ /* A function that performs in-place-substitution in the graph. */
+
+ void substitute_(HfstSymbol old_symbol,
+ HfstSymbol new_symbol,
+ bool input_side=true,
+ bool output_side=true);
+
+ /* A function that performs in-place substitutions in the graph
+ as defined in \a substitutions.
+
+ substitutions[from_number] = to_number,
+ if substitutions[from_number] = no_substitution, no substitution is made */
+ void substitute_(const HfstNumberVector &substitutions,
+ unsigned int no_substitution);
+
+ /* A function that performs in-place substitutions in the graph
+ as defined in \a substitutions. */
+ void substitute_(const HfstNumberPairSubstitutions &substitutions);
+
+ public:
+
+ /* A function that performs in-place removal of all transitions
+ equivalent to \a sp in the graph. */
+ HFSTDLL void remove_transitions(const HfstSymbolPair &sp);
+
+ protected:
+
+ /* A function that performs in-place-substitution in the graph. */
+ void substitute_(const HfstSymbolPair &old_sp,
+ const HfstSymbolPairSet &new_sps);
+
+ /* A function that performs in-place-substitution in the graph. */
+ void substitute_(bool (*func)
+ (const HfstSymbolPair &sp, HfstSymbolPairSet &sps));
+
+ public:
+
+ /* ----------------------------------------
+ The public substitution functions.
+ ---------------------------------------- */
+
+ /** @brief Substitute \a old_symbol with \a new_symbol in
+ all transitions. \a input_side and \a output_side define
+ whether the substitution is made on input and output sides. */
+ HFSTDLL HfstBasicTransducer &
+ substitute(const HfstSymbol &old_symbol,
+ const HfstSymbol &new_symbol,
+ bool input_side=true,
+ bool output_side=true);
+
+ HFSTDLL HfstBasicTransducer &substitute_symbols
+ (const HfstSymbolSubstitutions &substitutions);
+
+ /** @brief Substitute all transitions as defined in \a substitutions */
+ HfstBasicTransducer &substitute
+ (const HfstSymbolSubstitutions &substitutions);
+
+ HFSTDLL HfstBasicTransducer &substitute_symbol_pairs
+ (const HfstSymbolPairSubstitutions &substitutions);
+
+ /** @brief Substitute all transitions as defined in \a substitutions.
+
+ For each transition x:y, \a substitutions is searched and if
+ a mapping x:y -> X:Y is found, the transition x:y is replaced
+ with X:Y. If no mapping is found, the transition remains the same.
+ */
+ HFSTDLL HfstBasicTransducer &substitute
+ (const HfstSymbolPairSubstitutions &substitutions);
+
+ /** @brief Substitute all transitions \a sp with a set of transitions
+ \a sps. */
+ HFSTDLL HfstBasicTransducer &substitute
+ (const HfstSymbolPair &sp, const HfstSymbolPairSet &sps);
+
+ /** @brief Substitute all transitions \a old_pair with
+ \a new_pair. */
+ HFSTDLL HfstBasicTransducer &substitute
+ (const HfstSymbolPair &old_pair,
+ const HfstSymbolPair &new_pair);
+
+ /** @brief Substitute all transitions with a set of transitions as
+ defined by function \a func.
+
+ \a func takes as its argument a transition \a sp and inserts
+ into the set of transitions \a sps the transitions with which
+ the original transition \a sp must be replaced. \a func returns
+ a value indicating whether any substitution must be made, i.e.
+ whether any transition was inserted into \a sps. */
+ HFSTDLL HfstBasicTransducer &
+ substitute(bool (*func)
+ (const HfstSymbolPair &sp, HfstSymbolPairSet &sps) );
+
+ /* ----------------------------------------------------
+ Substitute string pair with a transition graph
+ ---------------------------------------------------- */
+
+ /* Used in function
+ substitute(const StringPair&, HfstTransitionGraph&) */
+ struct substitution_data
+ {
+ HfstState origin_state;
+ HfstState target_state;
+ HfstTropicalTransducerTransitionData::WeightType weight;
+ HfstBasicTransducer * substituting_graph;
+
+ substitution_data(HfstState origin,
+ HfstState target,
+ HfstTropicalTransducerTransitionData::WeightType weight,
+ HfstBasicTransducer * substituting)
+ {
+ origin_state=origin;
+ target_state=target;
+ this->weight=weight;
+ substituting_graph=substituting;
+ }
+ };
+
+ /* Used in function substitute(const StringPair&,
+ HfstTransitionGraph&)
+ Add a copy of substituting graph with epsilon transitions between
+ states and with weight as defined in \a sub. */
+ void add_substitution(const substitution_data &sub);
+
+ public:
+
+ /** @brief Substitute all transitions \a old_symbol : \a new_symbol
+ with a copy of \a graph.
+
+ Copies of \a graph are attached to this graph with
+ epsilon transitions.
+
+ The weights of the transitions to be substituted are copied
+ to epsilon transitions leaving from the source state of
+ the transitions to be substituted to the initial state
+ of a copy of \a graph.
+
+ The final weights in \a
+ graph are copied to epsilon transitions leading from
+ the final states (after substitution non-final states)
+ of \a graph to target states of transitions
+ \a old_symbol : \a new_symbol (that are substituted)
+ in this graph.
+ */
+ HFSTDLL HfstBasicTransducer &
+ substitute(const HfstSymbolPair &sp,
+ const HfstBasicTransducer &graph);
+
+ HFSTDLL std::string weight2marker(float weight);
+
+ HFSTDLL HfstBasicTransducer & substitute_weights_with_markers();
+
+ // ####
+ // another version of substitute for internal use..
+ // ####
+ typedef std::map<HfstSymbol, HfstBasicTransducer> SubstMap;
+
+ HFSTDLL HfstBasicTransducer &
+ substitute(SubstMap & substitution_map,
+ bool harmonize);
+
+ HFSTDLL bool marker2weight(const std::string & str, float & weight);
+
+ HFSTDLL HfstBasicTransducer & substitute_markers_with_weights();
+
+ // aliases
+ HFSTDLL HfstBasicTransducer & substitute_symbol(const std::string &old_symbol, const std::string &new_symbol, bool input_side=true, bool output_side=true);
+
+ HFSTDLL HfstBasicTransducer & substitute_symbol_pair(const StringPair &old_symbol_pair, const StringPair &new_symbol_pair);
+
+ HFSTDLL HfstBasicTransducer & substitute_symbol_pair_with_set(const StringPair &old_symbol_pair, const hfst::StringPairSet &new_symbol_pair_set);
+
+ HFSTDLL HfstBasicTransducer & substitute_symbol_pair_with_transducer(const StringPair &symbol_pair, HfstBasicTransducer &transducer);
+
+
+ /* ----------------------------
+ Insert freely functions
+ ---------------------------- */
+
+
+ /** @brief Insert freely any number of \a symbol_pair in
+ the graph with weight \a weight. */
+ HFSTDLL HfstBasicTransducer &insert_freely
+ (const HfstSymbolPair &symbol_pair, HfstTropicalTransducerTransitionData::WeightType weight);
+
+ /** @brief Insert freely any number of any symbol in \a symbol_pairs in
+ the graph with weight \a weight. */
+ HFSTDLL HfstBasicTransducer &insert_freely
+ (const HfstSymbolPairSet &symbol_pairs,
+ HfstTropicalTransducerTransitionData::WeightType weight);
+
+ /** @brief Insert freely any number of \a graph in this
+ graph. */
+ HFSTDLL HfstBasicTransducer &insert_freely
+ (const HfstBasicTransducer &graph);
+
+ /* -------------------------------
+ Harmonization function
+ ------------------------------- */
+
+ /** @brief Harmonize this HfstTransitionGraph and \a another.
+
+ In harmonization the unknown and identity symbols in
+ transitions of both graphs are expanded according to
+ the symbols that are previously unknown to the graph.
+
+ For example the graphs
+ \verbatim
+ [a:b ?:?]
+ [c:d ? ?:c]
+ \endverbatim
+ are expanded to
+ \verbatim
+ [ a:b [?:? | ?:c | ?:d | c:d | d:c | c:? | d:?] ]
+ [ c:d [? | a | b] [?:c| a:c | b:?] ]
+ \endverbatim
+ when harmonized.
+ The symbol "?" means \@_UNKNOWN_SYMBOL_\@ in either or
+ both sides of a transition
+ (transitions of type [?:x], [x:?] and [?:?]).
+ The transition [?] means [\@_IDENTITY_SYMBOL_\@].
+
+ @note This function is always called for arguments of functions
+ that take two or more graphs as their arguments, unless otherwise
+ said.
+ */
+ HFSTDLL HfstBasicTransducer &harmonize(HfstBasicTransducer &another);
+
+ /* -------------------------------
+ Disjunction functions
+ ------------------------------- */
+
+ protected:
+ /* Disjunct the transition of path \a spv pointed by \a it
+ to state \a s. If the transition does not exist in the graph,
+ it is created as well as its target state.
+
+ @return The final state of path \a spv, when \a it is at end. */
+ HfstState disjunct(const StringPairVector &spv,
+ StringPairVector::const_iterator &it,
+ HfstState s);
+
+ public:
+
+ /** @brief Disjunct this graph with a one-path graph
+ defined by string pair vector \a spv with weight \a weight.
+
+ @pre This graph must be a trie where all weights are in
+ final states, i.e. all transitions have a zero weight.
+
+ There is no way to test whether a graph is a trie, so the use
+ of this function is probably limited to fast construction
+ of a lexicon. Here is an example:
+
+ \verbatim
+ HfstBasicTransducer lexicon;
+ HfstTokenizer TOK;
+ lexicon.disjunct(TOK.tokenize("dog"), 0.3);
+ lexicon.disjunct(TOK.tokenize("cat"), 0.5);
+ lexicon.disjunct(TOK.tokenize("elephant"), 1.6);
+ \endverbatim
+
+ */
+ HFSTDLL HfstBasicTransducer &disjunct
+ (const StringPairVector &spv, HfstTropicalTransducerTransitionData::WeightType weight);
+
+ HFSTDLL bool is_special_symbol(const std::string & symbol);
+
+ HFSTDLL HfstBasicTransducer &complete();
+
+ HFSTDLL StringSet get_flags() const;
+
+ // Whether symbol \a symbol must be purged from transitions and alphabet
+ // of a transducer after \a flag has been eliminated from the transducer.
+ // If \a flag is the empty string, all flags have been eliminated.
+ HFSTDLL bool purge_symbol(const std::string & symbol, const std::string & flag);
+
+ // Replace arcs in \a transducer that use flag \a flag with epsilon arcs
+ // and remove \a flag from alphabet of \a transducer. If \a flag is the empty
+ // string, replace/remove all flags.
+ HFSTDLL void flag_purge(const std::string & flag);
+
+ /* A topological sort. */
+ struct TopologicalSort
+ {
+ std::vector<int> distance_of_state;
+ std::vector<std::set<HfstState> > states_at_distance;
+
+ /* Initialize the TopologicalSort by reserving space for a transducer
+ with biggest state number \a biggest_state_number, */
+ HFSTDLL void set_biggest_state_number(unsigned int biggest_state_number)
+ {
+ distance_of_state = std::vector<int>(biggest_state_number+1, -1);
+ }
+
+ /* Set the maximum distance of \a state to \a distance, */
+ HFSTDLL void set_state_at_distance(HfstState state, unsigned int distance,
+ bool overwrite)
+ {
+ // see that 'state' does not exceed the maximum state number given in initialization
+ if (state > (distance_of_state.size() - 1))
+ {
+ std::cerr << "ERROR in TopologicalSort::set_state_at_distance: first argument ("
+ << state << ") is out of range (should be < " << distance_of_state.size()
+ << ")" << std::endl;
+ }
+ // if there is nothing on index 'state',
+ // push back empty sets of states up to index 'state', including
+ while (distance + 1 > (unsigned int)states_at_distance.size())
+ {
+ std::set<HfstState> empty_set;
+ states_at_distance.push_back(empty_set);
+ }
+ // if there was previous distance defined for 'state', erase it, if needed
+ int previous_distance = distance_of_state.at(state);
+ if (previous_distance != -1 && previous_distance != (int)distance && overwrite)
+ {
+ states_at_distance.at(previous_distance).erase(state);
+ }
+ // set state and distance
+ states_at_distance.at(distance).insert(state);
+ distance_of_state.at(state) = distance;
+ }
+
+ /* The states that have a maximum distance of \a distance. */
+ HFSTDLL const std::set<HfstState> & get_states_at_distance(unsigned int distance)
+ {
+ // if there is nothing on index 'state',
+ // push back empty sets of states up to index 'state', including
+ while (distance > (states_at_distance.size() - 1))
+ {
+ std::set<HfstState> empty_set;
+ states_at_distance.push_back(empty_set);
+ }
+ return states_at_distance.at(distance);
+ }
+ };
+
+ enum SortDistance { MaximumDistance, MinimumDistance };
+
+ /*
+ Get a topological (maximum distance) sort of this graph.
+ @return A vector of sets of states. At each vector index ind, the
+ result contains the set of all states whose (maximum) distance from
+ the start state is ind.
+ */
+ HFSTDLL std::vector<std::set<HfstState> > topsort(SortDistance dist) const;
+
+ /** The length of longest string accepted by this graph.
+ If no string is accepted, return -1. */
+ HFSTDLL int longest_path_size();
+
+ /** The lengths of strings accepted by this graph, in descending order.
+ If not string is accepted, return an empty vector. */
+ HFSTDLL std::vector<unsigned int> path_sizes();
+
+ bool has_negative_epsilon_cycles
+ (HfstState state,
+ float total_weight,
+ std::map<HfstState, float> & state_weights);
+
+ bool has_negative_epsilon_cycles();
+
+ HFSTDLL bool is_infinitely_ambiguous
+ (HfstState state,
+ std::set<HfstState> &epsilon_path_states,
+ std::vector<unsigned int> &states_handled);
+
+ HFSTDLL bool is_infinitely_ambiguous();
+
+ bool is_possible_flag(std::string symbol, StringVector & fds, bool obey_flags);
+
+ HFSTDLL bool is_lookup_infinitely_ambiguous
+ (const HfstOneLevelPath& s,
+ unsigned int& index, HfstState state,
+ std::set<HfstState> &epsilon_path_states,
+ StringVector & fds, bool obey_flags);
+
+ HFSTDLL bool is_lookup_infinitely_ambiguous(const HfstOneLevelPath & s, bool obey_flags=false);
+
+ HFSTDLL bool is_lookup_infinitely_ambiguous(const StringVector & s, bool obey_flags=false);
+
+ HFSTDLL static void push_back_to_two_level_path
+ (HfstTwoLevelPath &path,
+ const StringPair &sp,
+ const float &weight,
+ StringVector * fds_so_far = NULL);
+
+ HFSTDLL static void pop_back_from_two_level_path
+ (HfstTwoLevelPath &path,
+ const float &weight,
+ StringVector * fds_so_far = NULL);
+
+ HFSTDLL static void add_to_results
+ (HfstTwoLevelPaths &results,
+ HfstTwoLevelPath &path_so_far,
+ const float &final_weight,
+ float * max_weight);
+
+ HFSTDLL static bool is_possible_transition
+ (const HfstBasicTransition &transition,
+ const StringVector &lookup_path,
+ const unsigned int &lookup_index,
+ const StringSet &alphabet,
+ bool &input_symbol_consumed,
+ StringVector * fds_so_far = NULL);
+
+ HFSTDLL void lookup
+ (const StringVector &lookup_path,
+ HfstTwoLevelPaths &results,
+ HfstState state,
+ unsigned int lookup_index, // an iterator instead?
+ HfstTwoLevelPath &path_so_far,
+ StringSet &alphabet,
+ HfstEpsilonHandler Eh,
+ size_t max_epsilon_cycles,
+ float * max_weight = NULL,
+ StringVector * flag_diacritic_path = NULL);
+
+ HFSTDLL void lookup
+ (const StringVector &lookup_path,
+ HfstTwoLevelPaths &results,
+ size_t * max_epsilon_cycles = NULL,
+ float * max_weight = NULL,
+ bool obey_flags = false);
+
+ HFSTDLL void check_regexp_state_for_cycle(HfstState s, const std::set<HfstState> & states_visited);
+
+ // Returns whether tr is "^]":"^]". If tr is not allowed, throws an error message.
+ HFSTDLL bool check_regexp_transition_end(const HfstBasicTransition & tr, bool input_side);
+
+ // If there is a "^[":"^[" transition leading to state \a s from some state
+ // S and state S is included in \a states_visited and \a path and \a full_paths
+ // are empty, this function can be used to find all (sub-)paths of form
+ // [x:y]* "^]" (x and y cannot be "^]" or "^[") starting from state \a s. The resulting
+ // paths are stored in \a full_paths. \a path is used to keep track of each path so
+ // far. Weights are currently ignored.
+ HFSTDLL void find_regexp_paths
+ (HfstState s,
+ std::set<HfstState> & states_visited,
+ std::vector<std::pair<std::string, std::string> > & path,
+ HfstReplacements & full_paths, bool input_side);
+
+ // For each "^[":"^[" transition in state \a s, find continuing paths of form [x:y]* "^]":"^]"
+ // (where x and y can freely be any symbols except "^]" or "^[") and store those paths in \a full_paths
+ // vector where the first member of each element is the state where the ending "^]":"^]" transition
+ // leads to and the second element is a vector of transitions (i.e. string pairs) without the ending
+ // "^]":"^]" transition.
+ // An error is thrown if mismatched "^[" or "^]" symbols, special symbols (epsilon, unknown, identity),
+ // or loops are encountered on a regexp path. Final states are allowed on regexp paths as they are also
+ // allowed by Xerox tools.
+ // Weights are currently ignored.
+ HFSTDLL void find_regexp_paths
+ (HfstState s,
+ std::vector<std::pair<HfstState, std::vector<std::pair<std::string, std::string> > > > & full_paths,
+ bool input_side);
+
+ // Find all subpaths of form "^[" [x:y]* "^]" (x and y cannot be "^[" or "^]") and return them.
+ // retval[start_state] == vector(pair(end_state, vector(pair(isymbol,osymbol) ) ) )
+ // Weights are currently ignored.
+ HFSTDLL HfstReplacementsMap find_replacements(bool input_side);
+
+ // Attach a copy of \a graph between states \a state1 and \a state2 with epsilon transitions.
+ // There will be an epsilon transition with weight zero from state \a state1 to the
+ // initial state of \a graph and one epsilon transition from each final state of
+ // \a graph to state \a state2 with a weight of that state's final weight. Final states of
+ // \a graph as well as its initial state are made normal non-final states when copying \a graph.
+ // Todo: copy alphabet? harmonize graphs?
+ HFSTDLL void insert_transducer(HfstState state1, HfstState state2, const HfstBasicTransducer & graph);
+
+ typedef std::pair<HfstState, HfstState> StatePair;
+ typedef std::map<StatePair, HfstState> StateMap;
+
+ // Find target state corresponding to state pair \a target1, \a target2 in \a state_map and return that state.
+ // If not found, add a new state to \a intersection, add it to \a state_map and return it.
+ // \a was_new_state specifies whether a new state was added.
+ static HfstState find_target_state
+ (HfstState target1, HfstState target2, StateMap & state_map,
+ HfstBasicTransducer & intersection, bool & was_new_state);
+
+ // A function used by find_matches.
+ // Copy matching transition tr1/tr2 to state \a state in \a intersection and return
+ // the target state of that transition. Also make that state final, if needed.
+ HFSTDLL static HfstState handle_match(const HfstBasicTransducer & graph1, const HfstBasicTransition & tr1,
+ const HfstBasicTransducer & graph2, const HfstBasicTransition & tr2,
+ HfstBasicTransducer & intersection, HfstState state, StateMap & state_map);
+
+ // A recursive function used by function intersect.
+ //
+ // @param graph1 The first transducer argument of intersection.
+ // @param state1 The current state of \a graph1.
+ // @param graph2 The second transducer argument of intersection.
+ // @param state2 The current state of \a graph2.
+ // @param intersection The intersection of \a graph1 and \a graph2.
+ // @param state The current state of \a intersection.
+ // @param state_map State pairs from \a graph1 and \a graph2 mapped to states in \a intersection.
+ // @param agenda States in \a intersection already handled or scheduled to be handled.
+ //
+ // @pre \a graph1 and \a graph2 must be arc-sorted (via sort_arcs()) to make transition matching faster.
+ // @pre \a graph1 and \a graph2 must be deterministic. (todo: handle equivalent transitions, maybe even epsilons?)
+ HFSTDLL static void find_matches
+ (HfstBasicTransducer & graph1, HfstState state1, HfstBasicTransducer & graph2, HfstState state2,
+ HfstBasicTransducer & intersection, HfstState state, StateMap & state_map, std::set<HfstState> & agenda);
+
+ HFSTDLL static HfstBasicTransducer intersect
+ (HfstBasicTransducer & graph1, HfstBasicTransducer & graph2);
+
+ // A function used by find_matches_for_merge
+ // Copy matching transition graph_tr/merger_tr to state \a result_state in \a result and return
+ // the target state of that transition. Also make that state final, if needed.
+ HFSTDLL static HfstState handle_non_list_match(const HfstBasicTransducer & graph, const HfstBasicTransition & graph_transition,
+ const HfstBasicTransducer & merger, HfstState merger_target,
+ HfstBasicTransducer & result, HfstState result_state, StateMap & state_map);
+
+ // A function used by find_matches_for_merge
+ // Copy matching transition graph_tr/merger_tr to state \a result_state in \a result and return
+ // the target state of that transition. Also make that state final, if needed.
+ HFSTDLL static HfstState handle_list_match(const HfstBasicTransducer & graph, const HfstBasicTransition & graph_transition,
+ const HfstBasicTransducer & merger, const HfstBasicTransition & merger_transition,
+ HfstBasicTransducer & result, HfstState result_state, StateMap & state_map, std::set<std::string> & markers_added);
+
+ HFSTDLL static bool is_list_symbol(const HfstTropicalTransducerTransitionData & transition_data, const std::map<std::string, std::set<std::string> > & list_symbols);
+
+ // A recursive function used by function intersect.
+ //
+ // @param graph The first transducer argument of intersection.
+ // @param graph_state The current state of \a graph1.
+ // @param merger The second transducer argument of intersection.
+ // @param merger_state The current state of \a graph2.
+ // @param result The intersection of \a graph1 and \a graph2.
+ // @param result_state The current state of \a intersection.
+ // @param state_map State pairs from \a graph and \a merger mapped to states in \a result.
+ // @param agenda States in \a result already handled or scheduled to be handled.
+ //
+ // @pre \a graph and \a merger must be arc-sorted (via sort_arcs()) to make transition matching faster.
+ // @pre \a graph and \a merger must be deterministic. (todo: handle equivalent transitions, maybe even epsilons?)
+ HFSTDLL static void find_matches_for_merge
+ (HfstBasicTransducer & graph, HfstState graph_state, HfstBasicTransducer & merger, HfstState merger_state,
+ HfstBasicTransducer & result, HfstState result_state, StateMap & state_map, std::set<HfstState> & agenda,
+ const std::map<std::string, std::set<std::string> > & list_symbols, std::set<std::string> & markers_added);
+
+ HFSTDLL static HfstBasicTransducer merge
+ (HfstBasicTransducer & graph, HfstBasicTransducer & merger, const std::map<std::string, std::set<std::string> > & list_symbols, std::set<std::string> & markers_added);
+
+ // --- Friends ---
+
+ friend class ConversionFunctions;
+ friend class hfst::HarmonizeUnknownAndIdentitySymbols;
+ };
+
+ }
+
+ }
+
+#endif // #ifndef _HFST_BASIC_TRANSDUCER_H_
diff --git a/libhfst/src/implementations/HfstBasicTransition.cc b/libhfst/src/implementations/HfstBasicTransition.cc
new file mode 100644
index 0000000..1cb831d
--- /dev/null
+++ b/libhfst/src/implementations/HfstBasicTransition.cc
@@ -0,0 +1,99 @@
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 3 of the License, or (at your option) any later version.
+// See the file COPYING included with this distribution for more
+// information.
+
+#include "HfstBasicTransition.h"
+
+namespace hfst {
+
+ namespace implementations {
+
+ /*unsigned int HfstBasicTransition::get_symbol_number
+ (const HfstTropicalTransducerTransitionData::SymbolType &symbol)
+ {
+ return HfstTropicalTransducerTransitionData::get_symbol_number(symbol);
+ }*/
+
+ HfstBasicTransition::HfstBasicTransition(): target_state(0) {};
+
+ HfstBasicTransition::HfstBasicTransition(HfstState s,
+ HfstTropicalTransducerTransitionData::SymbolType isymbol,
+ HfstTropicalTransducerTransitionData::SymbolType osymbol,
+ HfstTropicalTransducerTransitionData::WeightType weight):
+ target_state(s), transition_data(isymbol, osymbol, weight)
+ {};
+
+ HfstBasicTransition::HfstBasicTransition(HfstState s,
+ unsigned int inumber,
+ unsigned int onumber,
+ HfstTropicalTransducerTransitionData::WeightType weight,
+ bool foo):
+ target_state(s), transition_data(inumber, onumber, weight)
+ { (void)foo; };
+
+ HfstBasicTransition::HfstBasicTransition(const HfstBasicTransition &another):
+ target_state(another.target_state),
+ transition_data(another.transition_data)
+ {};
+
+ HfstBasicTransition::~HfstBasicTransition() {};
+
+ bool HfstBasicTransition::operator<(const HfstBasicTransition &another) const {
+ if (target_state == another.target_state)
+ return (transition_data < another.transition_data);
+ return (target_state < another.target_state);
+ }
+
+ void HfstBasicTransition::operator=(const HfstBasicTransition &another) {
+ target_state = another.target_state;
+ transition_data = another.transition_data;
+ }
+
+ HfstState HfstBasicTransition::get_target_state() const {
+ return target_state;
+ }
+
+ const HfstTropicalTransducerTransitionData & HfstBasicTransition::get_transition_data() const {
+ return transition_data;
+ }
+
+ HfstTropicalTransducerTransitionData::SymbolType HfstBasicTransition::get_input_symbol() const {
+ return transition_data.get_input_symbol();
+ }
+
+ void HfstBasicTransition::set_input_symbol(const HfstTropicalTransducerTransitionData::SymbolType & symbol) {
+ transition_data.set_input_symbol(symbol);
+ }
+
+ HfstTropicalTransducerTransitionData::SymbolType HfstBasicTransition::get_output_symbol() const {
+ return transition_data.get_output_symbol();
+ }
+
+ void HfstBasicTransition::set_output_symbol(const HfstTropicalTransducerTransitionData::SymbolType & symbol) {
+ transition_data.set_output_symbol(symbol);
+ }
+
+ unsigned int HfstBasicTransition::get_input_number() const {
+ return transition_data.get_input_number();
+ }
+
+ unsigned int HfstBasicTransition::get_output_number() const {
+ return transition_data.get_output_number();
+ }
+
+ HfstTropicalTransducerTransitionData::WeightType HfstBasicTransition::get_weight() const {
+ return transition_data.get_weight();
+ }
+
+ void HfstBasicTransition::set_weight(HfstTropicalTransducerTransitionData::WeightType w) {
+ transition_data.set_weight(w);
+ }
+ }
+
+}
+
diff --git a/libhfst/src/implementations/HfstBasicTransition.h b/libhfst/src/implementations/HfstBasicTransition.h
new file mode 100644
index 0000000..a9bd390
--- /dev/null
+++ b/libhfst/src/implementations/HfstBasicTransition.h
@@ -0,0 +1,116 @@
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 3 of the License, or (at your option) any later version.
+// See the file COPYING included with this distribution for more
+// information.
+
+#ifndef _HFST_BASIC_TRANSITION_H_
+#define _HFST_BASIC_TRANSITION_H_
+
+/** @file HfstBasicTransition.h
+ @brief Class HfstBasicTransition */
+
+#include "../HfstDataTypes.h"
+#include "HfstTropicalTransducerTransitionData.h"
+
+#include "../hfstdll.h"
+
+namespace hfst {
+
+ namespace implementations {
+
+ /** @brief A transition that consists of a target state and
+ transition data represented by class C.
+
+ The easiest way to use this template is to choose the
+ the implementation #HfstBasicTransition which is compatible with
+ #HfstBasicTransducer.
+
+ @see HfstBasicTransition
+ */
+ class HfstBasicTransition
+ {
+ protected:
+ HfstState target_state; // the state where the transition leads
+ HfstTropicalTransducerTransitionData transition_data; // the actual transition data
+
+ /* Get the number that represents the symbol in the transition data. */
+ /*static unsigned int get_symbol_number
+ (const HfstTropicalTransducerTransitionData::SymbolType &symbol);*/
+
+ public:
+
+ /** @brief Create a transition leading to state zero with input and
+ output symbols and weight as given by default constructors
+ of C::SymbolType and C::WeightType. */
+ HFSTDLL HfstBasicTransition();
+
+ /** @brief Create a transition leading to state \a s with input symbol
+ \a isymbol, output_symbol \a osymbol and weight \a weight. */
+ HFSTDLL HfstBasicTransition(HfstState s,
+ HfstTropicalTransducerTransitionData::SymbolType isymbol,
+ HfstTropicalTransducerTransitionData::SymbolType osymbol,
+ HfstTropicalTransducerTransitionData::WeightType weight);
+
+ HFSTDLL HfstBasicTransition(HfstState s,
+ unsigned int inumber,
+ unsigned int onumber,
+ HfstTropicalTransducerTransitionData::WeightType weight,
+ bool foo);
+
+ /** @brief Create a deep copy of transition \a another. */
+ HFSTDLL HfstBasicTransition(const HfstBasicTransition &another);
+
+ HFSTDLL ~HfstBasicTransition();
+
+ /** @brief Whether this transition is less than transition \a
+ another. Needed for storing transitions in a set. */
+ HFSTDLL bool operator<(const HfstBasicTransition &another) const;
+
+ /** @brief Assign this transition the same value as transition
+ \a another. */
+ HFSTDLL void operator=(const HfstBasicTransition &another);
+
+ /** @brief Get the target state of the transition. */
+ HFSTDLL HfstState get_target_state() const;
+
+ /** @brief Get the transition data of the transition. */
+ HFSTDLL const HfstTropicalTransducerTransitionData & get_transition_data() const;
+
+ /** @brief Get the input symbol of the transition. */
+ HFSTDLL HfstTropicalTransducerTransitionData::SymbolType get_input_symbol() const;
+
+ /** @brief Set the input symbol of the transition. */
+ HFSTDLL void set_input_symbol(const HfstTropicalTransducerTransitionData::SymbolType & symbol);
+
+ /** @brief Get the output symbol of the transition. */
+ HFSTDLL HfstTropicalTransducerTransitionData::SymbolType get_output_symbol() const;
+
+ /** @brief Set the output symbol of the transition. */
+ HFSTDLL void set_output_symbol(const HfstTropicalTransducerTransitionData::SymbolType & symbol);
+
+ /* Get the internal input number of the transition. */
+ HFSTDLL unsigned int get_input_number() const;
+
+ /* Get the internal output number of the transition. */
+ HFSTDLL unsigned int get_output_number() const;
+
+ /** @brief Get the weight of the transition. */
+ HFSTDLL HfstTropicalTransducerTransitionData::WeightType get_weight() const;
+
+ /** @brief Set the weight of the transition. */
+ HFSTDLL void set_weight(HfstTropicalTransducerTransitionData::WeightType w);
+
+ friend class ComposeIntersectFst;
+ friend class ComposeIntersectLexicon;
+ friend class ComposeIntersectRule;
+ friend class ComposeIntersectRulePair;
+ };
+ }
+
+}
+
+#endif // _HFST_BASIC_TRANSITION_H_
diff --git a/libhfst/src/implementations/HfstOlTransducer.cc b/libhfst/src/implementations/HfstOlTransducer.cc
index 71be21f..4c3cda2 100644
--- a/libhfst/src/implementations/HfstOlTransducer.cc
+++ b/libhfst/src/implementations/HfstOlTransducer.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include <cstring>
@@ -40,7 +40,7 @@ namespace hfst { namespace implementations
{ i_stream.close(); }
}
bool HfstOlInputStream::is_open(void) const
- {
+ {
if (filename != string())
{ return i_stream.is_open(); }
return true;
@@ -54,7 +54,7 @@ namespace hfst { namespace implementations
if (filename == string())
{ return std::cin.bad(); }
else
- { return input_stream.bad(); }
+ { return input_stream.bad(); }
}
bool HfstOlInputStream::is_good(void) const
{
@@ -90,8 +90,11 @@ namespace hfst { namespace implementations
else
{ res = 0; }
- for(int i=num_read-1;i>=0;i--)
- { ungetc(buffer[i], f); }
+ if (num_read > 0)
+ {
+ for(int i=hfst::size_t_to_int(num_read-1);i>=0;i--)
+ { ungetc(buffer[i], f); }
+ }
if(num_read != 24)
{ clearerr(f); }
@@ -117,8 +120,11 @@ namespace hfst { namespace implementations
else
{ res = 0; }
- for(int i=num_read-1;i>=0;i--)
- { s.putback(buffer[i]); }
+ if (num_read > 0)
+ {
+ for(int i=hfst::size_t_to_int(num_read-1);i>=0;i--)
+ { s.putback(buffer[i]); }
+ }
if(num_read != 24)
{ s.clear(); }
@@ -148,9 +154,9 @@ void HfstOlInputStream::ignore(unsigned int n)
hfst_ol::Transducer * HfstOlInputStream::read_transducer(bool has_header)
{
if (is_eof())
- {
+ {
HFST_THROW(StreamIsClosedException); }
- try
+ try
{
if (has_header)
skip_hfst_header();
@@ -177,7 +183,7 @@ void HfstOlInputStream::ignore(unsigned int n)
fprintf(stderr, "HfstOlOutputStream: ERROR: failbit set (3).\n");
}
- void HfstOlOutputStream::write_transducer(hfst_ol::Transducer * transducer)
+ void HfstOlOutputStream::write_transducer(hfst_ol::Transducer * transducer)
{
if (!output_stream)
fprintf(stderr, "HfstOlOutputStream: ERROR: failbit set (1).\n");
@@ -185,7 +191,7 @@ void HfstOlInputStream::ignore(unsigned int n)
}
void HfstOlOutputStream::open(void) {}
- void HfstOlOutputStream::close(void)
+ void HfstOlOutputStream::close(void)
{
if (filename != string())
{ o_stream.close(); }
@@ -205,13 +211,13 @@ void HfstOlInputStream::ignore(unsigned int n)
static bool extract_paths
(hfst_ol::Transducer* t, hfst_ol::TransitionTableIndex s,
- std::map<hfst_ol::TransitionTableIndex,unsigned short> all_visitations,
+ std::map<hfst_ol::TransitionTableIndex,unsigned short> all_visitations,
std::map<hfst_ol::TransitionTableIndex, unsigned short> path_visitations,
- /*std::vector<char>& lbuffer, int lpos,
+ /*std::vector<char>& lbuffer, int lpos,
std::vector<char>& ubuffer, int upos,*/
float weight_sum,
ExtractStringsCb& callback, int cycles,
- std::vector<hfst::FdState<hfst_ol::SymbolNumber> >* fd_state_stack,
+ std::vector<hfst::FdState<hfst_ol::SymbolNumber> >* fd_state_stack,
bool filter_fd, StringPairVector &spv)
{
if(cycles >= 0 && path_visitations[s] > cycles)
@@ -219,7 +225,7 @@ void HfstOlInputStream::ignore(unsigned int n)
all_visitations[s]++;
path_visitations[s]++;
- /*
+ /*
if(lpos > 0 && upos > 0)
{
lbuffer[lpos]=0;
@@ -300,7 +306,7 @@ void HfstOlInputStream::ignore(unsigned int n)
// sort arcs by number of visitations
- hfst_ol::TransitionTableIndexSet transitions
+ hfst_ol::TransitionTableIndexSet transitions
= t->get_transitions_from_state(s);
std::vector<hfst_ol::TransitionTableIndex> sorted_transitions;
for(hfst_ol::TransitionTableIndexSet::const_iterator it
@@ -309,7 +315,7 @@ void HfstOlInputStream::ignore(unsigned int n)
const hfst_ol::Transition& transition = t->get_transition(*it);
size_t i;
for( i=0; i<sorted_transitions.size(); i++ )
- if(all_visitations[transition.get_target()] <
+ if(all_visitations[transition.get_target()] <
all_visitations[t->get_transition
(sorted_transitions[i]).get_target()])
break;
@@ -322,7 +328,7 @@ void HfstOlInputStream::ignore(unsigned int n)
bool res = true;
for(size_t i=0; i<sorted_transitions.size() && res == true; i++)
{
- const hfst_ol::Transition& transition
+ const hfst_ol::Transition& transition
= t->get_transition(sorted_transitions[i]);
hfst_ol::SymbolNumber input = transition.get_input_symbol();
hfst_ol::SymbolNumber output = transition.get_output_symbol();
@@ -345,7 +351,7 @@ void HfstOlInputStream::ignore(unsigned int n)
int lp=lpos;
int up=upos;
- if(input != 0 && (!filter_fd ||
+ if(input != 0 && (!filter_fd ||
fd_state_stack->back().get_table().
get_operation(input)==NULL))
{
@@ -355,7 +361,7 @@ void HfstOlInputStream::ignore(unsigned int n)
strcpy(&lbuffer[lpos], str.c_str());
lp += str.length();
}
- if(output != 0 && (!filter_fd ||
+ if(output != 0 && (!filter_fd ||
fd_state_stack->back().get_table()
.get_operation(output)==NULL))
{
@@ -367,18 +373,18 @@ void HfstOlInputStream::ignore(unsigned int n)
}
*/
- /* Handle spv here. Special symbols (flags, epsilons)
+ /* Handle spv here. Special symbols (flags, epsilons)
are always inserted. */
std::string istring("");
std::string ostring("");
assert((fd_state_stack != NULL) || !filter_fd);
- if (!filter_fd ||
+ if (!filter_fd ||
fd_state_stack->back().get_table().
get_operation(input)==NULL)
istring = t->get_alphabet().get_symbol_table()[input];
- if (!filter_fd ||
+ if (!filter_fd ||
fd_state_stack->back().get_table().
get_operation(output)==NULL)
ostring = t->get_alphabet().get_symbol_table()[output];
@@ -387,8 +393,8 @@ void HfstOlInputStream::ignore(unsigned int n)
res = extract_paths
(t, transition.get_target(), all_visitations, path_visitations,
- /*lbuffer,lp, ubuffer,up,*/
- weight_sum + (t->get_header().probe_flag(hfst_ol::Weighted) ?
+ /*lbuffer,lp, ubuffer,up,*/
+ weight_sum + (t->get_header().probe_flag(hfst_ol::Weighted) ?
dynamic_cast<const hfst_ol::TransitionW&>(transition)
.get_weight() : 0.0f),
callback, cycles, fd_state_stack, filter_fd, spv);
@@ -413,8 +419,8 @@ void HfstOlInputStream::ignore(unsigned int n)
//std::vector<char> ubuffer(BUFFER_START_SIZE, 0);
std::map<hfst_ol::TransitionTableIndex, unsigned short> all_visitations;
std::map<hfst_ol::TransitionTableIndex, unsigned short> path_visitations;
- std::vector<hfst::FdState<hfst_ol::SymbolNumber> >* fd_state_stack
- = (fd==NULL) ? NULL :
+ std::vector<hfst::FdState<hfst_ol::SymbolNumber> >* fd_state_stack
+ = (fd==NULL) ? NULL :
new std::vector<hfst::FdState<hfst_ol::SymbolNumber> >
(1, hfst::FdState<hfst_ol::SymbolNumber>(*fd));
@@ -446,7 +452,7 @@ StringSet HfstOlTransducer::get_alphabet(hfst_ol::Transducer * t)
#include <iostream>
using namespace hfst::implementations;
-int main(int argc, char * argv[])
+int main(int argc, char * argv[])
{
std::cout << "Unit tests for " __FILE__ ":";
std::cout << std::endl << "ok" << std::endl;
diff --git a/libhfst/src/implementations/HfstOlTransducer.h b/libhfst/src/implementations/HfstOlTransducer.h
index a1c06a7..9c66441 100644
--- a/libhfst/src/implementations/HfstOlTransducer.h
+++ b/libhfst/src/implementations/HfstOlTransducer.h
@@ -1,16 +1,16 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _HFST_OL_TRANSDUCER_H_
#define _HFST_OL_TRANSDUCER_H_
-#include <iostream>
+#include <iosfwd>
#include <fstream>
#include "../HfstExceptionDefs.h"
#include "../HfstFlagDiacritics.h"
@@ -18,7 +18,7 @@
#include "optimized-lookup/transducer.h"
/** @file HfstOlTransducer.h
- \brief Declaration of classes for HFST's optimized lookup
+ \brief Declaration of classes for HFST's optimized lookup
transducer format. */
namespace hfst { namespace implementations
@@ -63,7 +63,7 @@ namespace hfst { namespace implementations
static int is_fst(istream &s);
};
- class HfstOlOutputStream
+ class HfstOlOutputStream
{
private:
std::string filename;
@@ -88,9 +88,9 @@ namespace hfst { namespace implementations
static void extract_paths
(hfst_ol::Transducer * t, hfst::ExtractStringsCb& callback,
- int cycles=-1, const FdTable<hfst_ol::SymbolNumber>* fd=NULL,
+ int cycles=-1, const FdTable<hfst_ol::SymbolNumber>* fd=NULL,
bool filter_fd=false);
- static const FdTable<hfst_ol::SymbolNumber>*
+ static const FdTable<hfst_ol::SymbolNumber>*
get_flag_diacritics(hfst_ol::Transducer* t);
static StringSet get_alphabet(hfst_ol::Transducer * t);
diff --git a/libhfst/src/implementations/HfstTransition.h b/libhfst/src/implementations/HfstTransition.h
index b17b7c5..0dc578d 100644
--- a/libhfst/src/implementations/HfstTransition.h
+++ b/libhfst/src/implementations/HfstTransition.h
@@ -1,16 +1,16 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _HFST_TRANSITION_H_
#define _HFST_TRANSITION_H_
-/** @file HfstTransition.h
+/** @file HfstTransition.h
@brief Class HfstTransition */
#include "../HfstDataTypes.h"
@@ -21,16 +21,16 @@ namespace hfst {
namespace implementations {
- /** @brief A transition that consists of a target state and
- transition data represented by class C.
+ /** @brief A transition that consists of a target state and
+ transition data represented by class C.
- The easiest way to use this template is to choose the
+ The easiest way to use this template is to choose the
the implementation #HfstBasicTransition which is compatible with
#HfstBasicTransducer.
@see HfstBasicTransition
*/
- template <class C> class HfstTransition
+ template <class C> class HfstTransition
{
protected:
HfstState target_state; // the state where the transition leads
@@ -52,25 +52,25 @@ namespace hfst {
/** @brief Create a transition leading to state \a s with input symbol
\a isymbol, output_symbol \a osymbol and weight \a weight. */
- HFSTDLL HfstTransition(HfstState s,
- typename C::SymbolType isymbol,
- typename C::SymbolType osymbol,
+ HFSTDLL HfstTransition(HfstState s,
+ typename C::SymbolType isymbol,
+ typename C::SymbolType osymbol,
typename C::WeightType weight):
target_state(s), transition_data(isymbol, osymbol, weight)
{}
- HFSTDLL HfstTransition(HfstState s,
- unsigned int inumber,
- unsigned int onumber,
+ HFSTDLL HfstTransition(HfstState s,
+ unsigned int inumber,
+ unsigned int onumber,
typename C::WeightType weight,
bool foo):
target_state(s), transition_data(inumber, onumber, weight)
{ (void)foo; }
/** @brief Create a deep copy of transition \a another. */
- HFSTDLL HfstTransition(const HfstTransition<C> &another):
- target_state(another.target_state),
- transition_data(another.transition_data)
+ HFSTDLL HfstTransition(const HfstTransition<C> &another):
+ target_state(another.target_state),
+ transition_data(another.transition_data)
{}
HFSTDLL ~HfstTransition() {}
@@ -83,7 +83,7 @@ namespace hfst {
return (target_state < another.target_state);
}
- /** @brief Assign this transition the same value as transition
+ /** @brief Assign this transition the same value as transition
\a another. */
HFSTDLL void operator=(const HfstTransition<C> &another) {
target_state = another.target_state;
@@ -137,12 +137,12 @@ namespace hfst {
};
/** @brief An HfstTransition with transition data of type
- HfstTropicalTransducerTransitionData.
+ HfstTropicalTransducerTransitionData.
This implementation is compatible with #HfstBasicTransducer.
@see HfstTropicalTransducerTransitionData HfstBasicTransducer */
- typedef HfstTransition<HfstTropicalTransducerTransitionData>
+ typedef HfstTransition<HfstTropicalTransducerTransitionData>
HfstBasicTransition;
// TODO: remove?
diff --git a/libhfst/src/implementations/HfstTransitionGraph.cc b/libhfst/src/implementations/HfstTransitionGraph.cc
index de24cfb..42f5549 100644
--- a/libhfst/src/implementations/HfstTransitionGraph.cc
+++ b/libhfst/src/implementations/HfstTransitionGraph.cc
@@ -1,16 +1,16 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
-#include "HfstTransitionGraph.h"
+#include "HfstBasicTransducer.h"
#ifndef MAIN_TEST
-namespace hfst {
+namespace hfst {
namespace implementations {
@@ -20,7 +20,7 @@ namespace hfst {
#else // MAIN_TEST was defined
#include <iostream>
-int main(int argc, char * argv[])
+int main(int argc, char * argv[])
{
using namespace hfst::implementations;
std::cout << "Unit tests for " __FILE__ ":" << std::endl;
diff --git a/libhfst/src/implementations/HfstTransitionGraph.h b/libhfst/src/implementations/HfstTransitionGraph.h
index 1b157db..4835aa8 100644
--- a/libhfst/src/implementations/HfstTransitionGraph.h
+++ b/libhfst/src/implementations/HfstTransitionGraph.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _HFST_TRANSITION_GRAPH_H_
@@ -17,7 +17,7 @@
#include <string>
#include <set>
#include <cassert>
- #include <iostream>
+ #include <iosfwd>
#include <algorithm>
#include <stack>
@@ -26,6 +26,7 @@
#include "../HfstDataTypes.h"
#include "../HarmonizeUnknownAndIdentitySymbols.h"
#include "../HfstFlagDiacritics.h"
+ #include "../HfstLookupFlagDiacritics.h"
#include "../HfstEpsilonHandler.h"
#include "ConvertTransducerFormat.h"
#include "HfstTransition.h"
@@ -65,15 +66,15 @@
and #HfstBasicTransition
(HfstTransition<HfstTropicalTransducerTransitionData>).
The class HfstTropicalTransducerTransitionData contains an input string,
- an output string and a float weight. HfstBasicTransducer is the
+ an output string and a float weight. HfstBasicTransducer is the
implementation that is used as an example in this documentation.
- An example of creating a HfstBasicTransducer [foo:bar baz:baz]
+ An example of creating a HfstBasicTransducer [foo:bar baz:baz]
with weight 0.4 from scratch:
\verbatim
// Create an empty transducer
- // The transducer has initially one start state (number zero)
+ // The transducer has initially one start state (number zero)
// that is not final
HfstBasicTransducer fsm;
// Add two states to the transducer
@@ -83,7 +84,7 @@
HfstBasicTransition tr(1, "foo", "bar", 0.1);
// ... and add it to state zero
fsm.add_transition(0, tr);
- // Add a transition [baz:baz] with weight 0 from state 1 to state 2
+ // Add a transition [baz:baz] with weight 0 from state 1 to state 2
fsm.add_transition(1, HfstBasicTransition(2, "baz", "baz", 0.0));
// Set state 2 as final with weight 0.3
fsm.set_final_weight(2, 0.3);
@@ -101,7 +102,7 @@
it != fsm.end(); it++ )
{
// Go through all transitions
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
+ for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
= it->begin(); tr_it != it->end(); tr_it++)
{
std::cerr << source_state << "\t"
@@ -111,19 +112,19 @@
<< tr_it->get_weight() << std::endl;
}
- if (fsm.is_final_state(source_state))
+ if (fsm.is_final_state(source_state))
{
std::cerr << source_state << "\t"
<< fsm.get_final_weight(source_state) << std::endl;
}
- // the next state is numbered source_state + 1
+ // the next state is numbered source_state + 1
source_state++;
}
\endverbatim
@see #HfstBasicTransducer HfstBasicTransition */
- template <class C> class HfstTransitionGraph
+ template <class C> class HfstTransitionGraph
{
// --- Datatypes and variables ---
@@ -132,8 +133,8 @@
/** @brief Datatype for a symbol in a transition. */
typedef typename C::SymbolType HfstSymbol;
/** @brief Datatype for a symbol pair in a transition. */
- typedef std::pair<HfstSymbol, HfstSymbol>
- HfstSymbolPair;
+ typedef std::pair<HfstSymbol, HfstSymbol>
+ HfstSymbolPair;
/** @brief A set of symbol pairs. */
typedef std::set<HfstSymbolPair> HfstSymbolPairSet;
/** @brief A set of symbol pairs. */
@@ -147,7 +148,7 @@
typedef std::vector<HfstTransition<C> > HfstTransitions;
/* Datatype for the states of a graph and their transitions.
- Each index of the vector is a state and the transitions
+ Each index of the vector is a state and the transitions
on that index are the transitions of that state. */
typedef std::vector<HfstTransitions> HfstStates;
@@ -173,7 +174,7 @@
typedef std::map<HfstNumberPair, HfstNumberPair> HfstNumberPairSubstitutions;
protected:
- /* @brief An iterator type that points to a state in a graph.
+ /* @brief An iterator type that points to a state in a graph.
The value pointed by the iterator is of type HfstTransitions. */
typedef typename HfstStates::iterator iterator;
@@ -210,6 +211,7 @@
initialize_alphabet(alphabet);
HfstTransitions tr;
state_vector.push_back(tr);
+ name = std::string("");
}
HFSTDLL HfstTransitionGraph(FILE *file) {
@@ -218,6 +220,7 @@
state_vector.push_back(tr);
unsigned int linecount=0;
this->assign(read_in_att_format(file, "@0@", linecount));
+ name = std::string("");
}
@@ -230,6 +233,7 @@
final_weight_map = graph.final_weight_map;
alphabet = graph.alphabet;
assert(alphabet.count(HfstSymbol()) == 0);
+ name = graph.name;
return *this;
}
@@ -243,10 +247,11 @@
state_vector = graph.state_vector;
final_weight_map = graph.final_weight_map;
alphabet = graph.alphabet;
+ name = graph.name;
assert(alphabet.count(HfstSymbol()) == 0);
}
- /** @brief Create an HfstTransitionGraph equivalent to HfstTransducer
+ /** @brief Create an HfstTransitionGraph equivalent to HfstTransducer
\a transducer. FIXME: move to a separate file */
HFSTDLL HfstTransitionGraph(const hfst::HfstTransducer &transducer) {
HfstTransitionGraph<HfstTropicalTransducerTransitionData>
@@ -264,7 +269,7 @@
// --------------------------------------------------
protected:
- /* Add epsilon, unknown and identity symbols to the alphabet
+ /* Add epsilon, unknown and identity symbols to the alphabet
\a alpha. */
void initialize_alphabet(HfstTransitionGraphAlphabet &alpha) {
alpha.insert(C::get_epsilon());
@@ -284,11 +289,11 @@
{
C data = tr_it->get_transition_data();
- if(alphabet.find(data.get_input_symbol())
+ if(alphabet.find(data.get_input_symbol())
== alphabet.end()) {
return false;
}
- if(alphabet.find(data.get_output_symbol())
+ if(alphabet.find(data.get_output_symbol())
== alphabet.end()) {
return false;
}
@@ -299,9 +304,9 @@
public:
/* Print the alphabet of the graph to standard error stream. */
- HFSTDLL void print_alphabet() const
+ HFSTDLL void print_alphabet() const
{
- for (typename HfstTransitionGraphAlphabet::const_iterator it
+ for (typename HfstTransitionGraphAlphabet::const_iterator it
= alphabet.begin(); it != alphabet.end(); it++)
{
if (it != alphabet.begin())
@@ -318,7 +323,7 @@
return C::get_number(symbol);
}
- /* For internal optimization: Reserve space for
+ /* For internal optimization: Reserve space for
\a number_of_states states. */
void initialize_state_vector
(unsigned int number_of_states)
@@ -327,7 +332,7 @@
}
/* For internal optimization: Reserve space for
- \a number_of_transitions transitions for state number
+ \a number_of_transitions transitions for state number
\a state_number. */
void initialize_transition_vector
(unsigned int state_number, unsigned int number_of_transitions)
@@ -350,7 +355,7 @@
alphabet.insert(symbol);
}
- /** @brief Remove symbol \a symbol from the alphabet of the graph.
+ /** @brief Remove symbol \a symbol from the alphabet of the graph.
@note Use with care, removing symbols that occur in the transitions
of the graph can have unexpected results. */
@@ -387,7 +392,7 @@
}
}
- /* Remove all symbols that are given in \a symbols but do not occur
+ /* Remove all symbols that are given in \a symbols but do not occur
in transitions of the graph from its alphabet. */
HFSTDLL void prune_alphabet_after_substitution(const std::set<unsigned int> &symbols)
{
@@ -441,7 +446,7 @@
}
/** @brief Remove all symbols that do not occur in transitions of
- the graph from its alphabet.
+ the graph from its alphabet.
@param force Whether unused symbols are removed even if
unknown or identity symbols occur in transitions.
@@ -454,7 +459,7 @@
HfstTransitionGraphAlphabet symbols_found = symbols_used();
// Whether unknown or identity symbols are used
- bool unknowns_or_identities_used =
+ bool unknowns_or_identities_used =
( (symbols_found.find("@_UNKNOWN_SYMBOL_@") != symbols_found.end()) ||
(symbols_found.find("@_IDENTITY_SYMBOL_@") != symbols_found.end()) );
@@ -464,17 +469,17 @@
return;
// Special symbols are always known
- symbols_found.insert("@_EPSILON_SYMBOL_@");
- symbols_found.insert("@_UNKNOWN_SYMBOL_@");
- symbols_found.insert("@_IDENTITY_SYMBOL_@");
+ symbols_found.insert("@_EPSILON_SYMBOL_@");
+ symbols_found.insert("@_UNKNOWN_SYMBOL_@");
+ symbols_found.insert("@_IDENTITY_SYMBOL_@");
- // Which symbols in the graph's alphabet did not occur in
+ // Which symbols in the graph's alphabet did not occur in
// the graph
HfstTransitionGraphAlphabet symbols_not_found;
- for (typename HfstTransitionGraphAlphabet::iterator it
+ for (typename HfstTransitionGraphAlphabet::iterator it
= alphabet.begin();
- it != alphabet.end(); it++)
+ it != alphabet.end(); it++)
{
if (symbols_found.find(*it) == symbols_found.end())
symbols_not_found.insert(*it);
@@ -482,7 +487,7 @@
// Remove the symbols that did not occur in the graph
// from its alphabet
- for (typename HfstTransitionGraphAlphabet::iterator it
+ for (typename HfstTransitionGraphAlphabet::iterator it
= symbols_not_found.begin();
it != symbols_not_found.end(); it++)
{
@@ -490,11 +495,11 @@
}
}
- /** @brief Get the set of HfstSymbols in the alphabet
- of the graph.
+ /** @brief Get the set of HfstSymbols in the alphabet
+ of the graph.
The HfstSymbols do not necessarily occur in any transitions
- of the graph. Epsilon, unknown and identity \link
+ of the graph. Epsilon, unknown and identity \link
hfst::String symbols\endlink are always included in the alphabet. */
HFSTDLL const HfstTransitionGraphAlphabet &get_alphabet() const {
return alphabet;
@@ -551,7 +556,7 @@
return state_vector.size()-1;
}
- /** @brief Add a transition \a transition to state \a s.
+ /** @brief Add a transition \a transition to state \a s.
If state \a s does not exist, it is created. */
HFSTDLL void add_transition(HfstState s, const HfstTransition<C> & transition,
@@ -598,7 +603,7 @@
it->get_target_state() == transition.get_target_state())
{
// schedule transition to be removed
- elements_to_remove.push(it);
+ elements_to_remove.push(it);
}
}
// remove the transitions in reverse order
@@ -606,7 +611,7 @@
{
state_vector[s].erase(elements_to_remove.top());
elements_to_remove.pop();
- }
+ }
if (remove_symbols_from_alphabet)
{
@@ -618,7 +623,7 @@
}
}
- /** @brief Whether state \a s is final.
+ /** @brief Whether state \a s is final.
FIXME: return positive infinity instead if not final. */
HFSTDLL bool is_final_state(HfstState s) const {
return (final_weight_map.find(s) != final_weight_map.end());
@@ -633,11 +638,11 @@
HFST_THROW(StateIsNotFinalException);
}
- /** @brief Set the final weight of state \a s in this graph
- to \a weight.
+ /** @brief Set the final weight of state \a s in this graph
+ to \a weight.
If the state does not exist, it is created. */
- HFSTDLL void set_final_weight(HfstState s,
+ HFSTDLL void set_final_weight(HfstState s,
const typename C::WeightType & weight) {
add_state(s);
final_weight_map[s] = weight;
@@ -658,17 +663,17 @@
return *this;
}
- /** @brief Get an iterator to the beginning of the states in
- the graph.
+ /** @brief Get an iterator to the beginning of the states in
+ the graph.
For an example, see #HfstTransitionGraph */
HFSTDLL iterator begin() { return state_vector.begin(); }
- /** @brief Get a const iterator to the beginning of
+ /** @brief Get a const iterator to the beginning of
states in the graph. */
HFSTDLL const_iterator begin() const { return state_vector.begin(); }
- /** @brief Get an iterator to the end of states (last state + 1)
+ /** @brief Get an iterator to the end of states (last state + 1)
in the graph. */
HFSTDLL iterator end() { return state_vector.end(); }
@@ -677,17 +682,17 @@
HFSTDLL const_iterator end() const { return state_vector.end(); }
- /** @brief Get the set of transitions of state \a s in this graph.
+ /** @brief Get the set of transitions of state \a s in this graph.
If the state does not exist, a @a StateIndexOutOfBoundsException
is thrown.
*/
HFSTDLL const HfstTransitions & operator[](HfstState s) const
{
- if (s >= state_vector.size()) {
+ if (s >= state_vector.size()) {
HFST_THROW(StateIndexOutOfBoundsException); }
return state_vector[s];
- }
+ }
/** @brief Alternative name for operator[].
@@ -702,9 +707,9 @@
/** @brief Get mutable transitions.
*/
- HFSTDLL HfstTransitions & transitions(HfstState s)
+ HFSTDLL HfstTransitions & transitions(HfstState s)
{
- if (s >= state_vector.size()) {
+ if (s >= state_vector.size()) {
HFST_THROW(StateIndexOutOfBoundsException); }
return state_vector[s];
}
@@ -778,7 +783,7 @@
static void write_weight(FILE * file, float weight)
{
//if (weight == 0) // avoid unnecessary 0.000000's
- // fprintf(file, "%i", 0);
+ // fprintf(file, "%i", 0);
//else
fprintf(file, "%f", weight);
}
@@ -786,7 +791,7 @@
static void write_weight(std::ostream & os, float weight)
{
//if (weight == 0) // avoid unnecessary 0.000000's
- // os << 0;
+ // os << 0;
//else
os << weight;
}
@@ -835,10 +840,10 @@
{
// replace all spaces, epsilons and tabs
if (data.get_input_symbol() !=
- data.get_output_symbol())
+ data.get_output_symbol())
{
os << "<";
- }
+ }
std::string s = data.get_input_symbol();
xfstize_symbol(s);
os << s;
@@ -848,7 +853,7 @@
{
s = data.get_output_symbol();
xfstize_symbol(s);
- os << ":" << s;
+ os << ":" << s;
}
if (data.get_input_symbol() !=
data.get_output_symbol())
@@ -888,7 +893,7 @@
/** @brief Write the graph in xfst text format to ostream \a os.
\a write_weights defines whether weights are printed (todo). */
- HFSTDLL void write_in_xfst_format(std::ostream &os, bool write_weights=true)
+ HFSTDLL void write_in_xfst_format(std::ostream &os, bool write_weights=true)
{
(void)write_weights; // todo
unsigned int source_state=0;
@@ -920,7 +925,7 @@
}
os << "." << std::endl;
source_state++;
- }
+ }
}
// note: unknown and identity are both '?'
@@ -968,7 +973,7 @@
fprintf(file, "\"%s\"", symbol.c_str());
if (data.get_input_symbol() !=
- data.get_output_symbol() ||
+ data.get_output_symbol() ||
data.get_input_symbol() == "@_UNKNOWN_SYMBOL_@")
{
symbol = prologize_symbol(data.get_output_symbol());
@@ -982,7 +987,7 @@
os << "\"" << symbol << "\"";
if (data.get_input_symbol() !=
- data.get_output_symbol() ||
+ data.get_output_symbol() ||
data.get_input_symbol() == "@_UNKNOWN_SYMBOL_@")
{
symbol = prologize_symbol(data.get_output_symbol());
@@ -992,8 +997,8 @@
/** @brief Write the graph in prolog format to FILE \a file.
\a write_weights defines whether weights are printed (todo). */
- HFSTDLL void write_in_prolog_format(FILE * file, const std::string & name,
- bool write_weights=true)
+ HFSTDLL void write_in_prolog_format(FILE * file, const std::string & name,
+ bool write_weights=true)
{
unsigned int source_state=0;
const char * identifier = name.c_str();
@@ -1008,7 +1013,7 @@
// Print symbols that are in the alphabet but not used in arcs.
HfstTransitionGraphAlphabet symbols_used_ = symbols_used();
initialize_alphabet(symbols_used_); // exclude special symbols
- for (typename HfstTransitionGraphAlphabet::const_iterator it
+ for (typename HfstTransitionGraphAlphabet::const_iterator it
= alphabet.begin(); it != alphabet.end(); it++)
{
if (symbols_used_.find(*it) == symbols_used_.end())
@@ -1038,12 +1043,12 @@
}
// Print final states.
- for (typename FinalWeightMap::const_iterator it
+ for (typename FinalWeightMap::const_iterator it
= this->final_weight_map.begin();
it != this->final_weight_map.end(); it++)
{
fprintf(file, "final(%s, %i", identifier, it->first);
- if (write_weights)
+ if (write_weights)
{
fprintf(file, ", ");
write_weight(file, it->second);
@@ -1054,8 +1059,8 @@
/** @brief Write the graph in prolog format to ostream \a os.
\a write_weights defines whether weights are printed (todo). */
- HFSTDLL void write_in_prolog_format(std::ostream & os, const std::string & name,
- bool write_weights=true)
+ HFSTDLL void write_in_prolog_format(std::ostream & os, const std::string & name,
+ bool write_weights=true)
{
unsigned int source_state=0;
@@ -1070,7 +1075,7 @@
// Print symbols that are in the alphabet but not used in arcs.
HfstTransitionGraphAlphabet symbols_used_ = symbols_used();
initialize_alphabet(symbols_used_); // exclude special symbols
- for (typename HfstTransitionGraphAlphabet::const_iterator it
+ for (typename HfstTransitionGraphAlphabet::const_iterator it
= alphabet.begin(); it != alphabet.end(); it++)
{
if (symbols_used_.find(*it) == symbols_used_.end())
@@ -1091,7 +1096,7 @@
print_prolog_arc_symbols(os, data);
if (write_weights) {
os << ", ";
- write_weight(os, data.get_weight());
+ write_weight(os, data.get_weight());
}
os << ")." << std::endl;
}
@@ -1099,7 +1104,7 @@
}
// Print final states.
- for (typename FinalWeightMap::const_iterator it
+ for (typename FinalWeightMap::const_iterator it
= this->final_weight_map.begin();
it != this->final_weight_map.end(); it++)
{
@@ -1137,7 +1142,7 @@
return true;
}
- HFSTDLL static bool parse_prolog_network_line(const std::string & line, std::string & name)
+ HFSTDLL static bool parse_prolog_network_line(const std::string & line, HfstTransitionGraph & graph)
{
// 'network(NAME).'
char namearr[100];
@@ -1150,7 +1155,7 @@
if (!strip_ending_parenthesis_and_comma(namestr))
return false;
- name = namestr;
+ graph.name = namestr;
return true;
}
@@ -1175,8 +1180,8 @@
return retval;
}
- // Extract input and output symbols, if possible, from prolog arc
- // \a str and store them to \a isymbol and \a osymbol.
+ // Extract input and output symbols, if possible, from prolog arc
+ // \a str and store them to \a isymbol and \a osymbol.
// Return whether symbols were successfully extracted.
// \a str must be of format "foo":"bar" or "foo"
HFSTDLL static bool get_prolog_arc_symbols
@@ -1197,7 +1202,7 @@
else if (quote_positions.size() == 4)
{
if (quote_positions[0] != 0 ||
- quote_positions[3] != str.length()-1)
+ quote_positions[3] != str.length()-1)
{
return false; // extra characters outside quotes
}
@@ -1249,7 +1254,7 @@
{ return false; }
if (last_space == std::string::npos) {
- ; // no weight
+ ; // no weight
}
else if (last_double_quote > last_space) {
; // no weight, last space is part of a symbol
@@ -1274,13 +1279,13 @@
char namestr[100]; char sourcestr[100];
char targetstr[100]; char symbolstr[100];
- int n = sscanf(line.c_str(), "arc(%[^,], %[^,], %[^,], %[^\t\n]",
+ int n = sscanf(line.c_str(), "arc(%[^,], %[^,], %[^,], %[^\t\n]",
namestr, sourcestr, targetstr, symbolstr);
std::string symbol(symbolstr);
// strip the ending ")." from symbolstr
- if (!strip_ending_parenthesis_and_comma(symbol))
+ if (!strip_ending_parenthesis_and_comma(symbol))
{ return false; }
if (n != 4)
@@ -1402,26 +1407,26 @@
if (! is.getline(line,255).eof())
HFST_THROW(EndOfStreamException);
}
- else { /* we use FILEs */
+ else { /* we use FILEs */
if (NULL == fgets(line, 255, file))
HFST_THROW(EndOfStreamException);
}
linecount++;
std::string linestr(line);
- return strip_newlines(linestr);
+ return strip_newlines(linestr);
}
- /* Create an HfstTransitionGraph as defined in prolog format
+ /* Create an HfstTransitionGraph as defined in prolog format
in istream \a is or FILE \a file.
- The functions is called by functions
+ The functions is called by functions
read_in_prolog_format(istream&) and
- read_in_prolog_format(FILE*).
+ read_in_prolog_format(FILE*).
If \a file is NULL, it is ignored and \a is is used.
If \a file is not NULL, it is used and \a is is ignored. */
HFSTDLL static HfstTransitionGraph read_in_prolog_format
- (std::istream &is, FILE *file, unsigned int & linecount)
+ (std::istream &is, FILE *file, unsigned int & linecount)
{
HfstTransitionGraph retval;
@@ -1429,13 +1434,13 @@
while(true)
{
- try
+ try
{
linestr = get_stripped_line(is, file, linecount);
- }
- catch (const EndOfStreamException & e)
+ }
+ catch (const EndOfStreamException & e)
{
- HFST_THROW(NotValidPrologFormatException);
+ HFST_THROW(NotValidPrologFormatException);
}
if (linestr.length() != 0 && linestr[0] == '#')
@@ -1449,7 +1454,7 @@
}
- if (! parse_prolog_network_line(linestr, retval.name))
+ if (! parse_prolog_network_line(linestr, retval))
{
std::string message("first line not valid prolog: ");
message.append(linestr);
@@ -1458,15 +1463,15 @@
while(true)
{
- try
+ try
{
linestr = get_stripped_line(is, file, linecount);
- if (linestr == "") // prolog separator
+ if (linestr == "") // prolog separator
{
return retval;
}
- }
- catch (const EndOfStreamException & e)
+ }
+ catch (const EndOfStreamException & e)
{
return retval;
}
@@ -1485,7 +1490,7 @@
HFSTDLL static HfstTransitionGraph read_in_prolog_format
(std::istream &is,
- unsigned int & linecount)
+ unsigned int & linecount)
{
return read_in_prolog_format
(is, NULL /* a dummy variable */,
@@ -1493,18 +1498,18 @@
}
HFSTDLL static HfstTransitionGraph read_in_prolog_format
- (FILE *file,
- unsigned int & linecount)
+ (FILE *file,
+ unsigned int & linecount)
{
return read_in_prolog_format
(std::cin /* a dummy variable */,
file, linecount);
- }
+ }
/** @brief Write the graph in xfst text format to FILE \a file.
\a write_weights defines whether weights are printed (todo). */
- HFSTDLL void write_in_xfst_format(FILE * file, bool write_weights=true)
+ HFSTDLL void write_in_xfst_format(FILE * file, bool write_weights=true)
{
(void)write_weights;
unsigned int source_state=0;
@@ -1537,7 +1542,7 @@
}
fprintf(file, ".\n");
source_state++;
- }
+ }
}
@@ -1545,7 +1550,7 @@
/** @brief Write the graph in AT&T format to ostream \a os.
\a write_weights defines whether weights are printed. */
- HFSTDLL void write_in_att_format(std::ostream &os, bool write_weights=true)
+ HFSTDLL void write_in_att_format(std::ostream &os, bool write_weights=true)
{
unsigned int source_state=0;
for (iterator it = begin(); it != end(); it++)
@@ -1566,7 +1571,7 @@
replace_all(osymbol, "@_EPSILON_SYMBOL_@", "@0@");
replace_all(osymbol, "\t", "@_TAB_@");
- os << source_state << "\t"
+ os << source_state << "\t"
<< tr_it->get_target_state() << "\t"
<< isymbol << "\t"
<< osymbol;
@@ -1587,12 +1592,12 @@
os << "\n";
}
source_state++;
- }
+ }
}
/** @brief Write the graph in AT&T format to FILE \a file.
\a write_weights defines whether weights are printed. */
- HFSTDLL void write_in_att_format(FILE *file, bool write_weights=true)
+ HFSTDLL void write_in_att_format(FILE *file, bool write_weights=true)
{
unsigned int source_state=0;
for (iterator it = begin(); it != end(); it++)
@@ -1622,23 +1627,23 @@
if (write_weights) {
fprintf(file, "\t");
write_weight(file, data.get_weight());
- }
+ }
fprintf(file, "\n");
}
if (is_final_state(source_state))
{
fprintf(file, "%i", source_state);
if (write_weights) {
- fprintf(file, "\t");
+ fprintf(file, "\t");
write_weight(file, get_final_weight(source_state));
}
fprintf(file, "\n");
}
source_state++;
- }
+ }
}
- HFSTDLL void write_in_att_format(char * ptr, bool write_weights=true)
+ HFSTDLL void write_in_att_format(char * ptr, bool write_weights=true)
{
unsigned int source_state=0;
size_t cwt = 0; // characters written in total
@@ -1681,21 +1686,21 @@
cw = sprintf(ptr + cwt, "%i", source_state);
cwt = cwt + cw;
if (write_weights)
- cw = sprintf(ptr + cwt, "\t%f",
+ cw = sprintf(ptr + cwt, "\t%f",
get_final_weight(source_state));
cwt = cwt + cw;
cw = sprintf(ptr + cwt, "\n");
cwt = cwt + cw;
}
source_state++;
- }
+ }
}
/** @brief Write the graph in AT&T format to FILE \a file using numbers
instead of symbol names.
\a write_weights defines whether weights are printed. */
- HFSTDLL void write_in_att_format_number(FILE *file, bool write_weights=true)
+ HFSTDLL void write_in_att_format_number(FILE *file, bool write_weights=true)
{
unsigned int source_state=0;
for (iterator it = begin(); it != end(); it++)
@@ -1714,14 +1719,14 @@
if (write_weights)
fprintf(file, "\t%f",
- data.get_weight());
+ data.get_weight());
fprintf(file, "\n");
if (is_final_state(source_state))
{
fprintf(file, "%i", source_state);
if (write_weights)
- fprintf(file, "\t%f",
+ fprintf(file, "\t%f",
get_final_weight(source_state));
fprintf(file, "\n");
}
@@ -1731,13 +1736,63 @@
}
- /* Create an HfstTransitionGraph as defined in AT&T format
+ bool add_att_line(char * line, const std::string & epsilon_symbol)
+ {
+ // scan one line that can have a maximum of five fields
+ char a1 [100]; char a2 [100]; char a3 [100];
+ char a4 [100]; char a5 [100];
+ // how many fields could be parsed
+ int n = sscanf(line, "%s%s%s%s%s", a1, a2, a3, a4, a5);
+
+ // set value of weight
+ float weight = 0;
+ if (n == 2) // a final state line with weight
+ weight = atof(a2);
+ if (n == 5) // a transition line with weight
+ weight = atof(a5);
+
+ if (n == 1 || n == 2) // a final state line
+ set_final_weight( atoi(a1), weight );
+
+ else if (n == 4 || n == 5) { // a transition line
+ std::string input_symbol=std::string(a3);
+ std::string output_symbol=std::string(a4);
+
+ // replace "@_SPACE_@"s with " " and "@0@"s with
+ // "@_EPSILON_SYMBOL_@"
+ replace_all(input_symbol, "@_SPACE_@", " ");
+ replace_all(input_symbol, "@0@", "@_EPSILON_SYMBOL_@");
+ replace_all(input_symbol, "@_TAB_@", "\t");
+ replace_all(input_symbol, "@_COLON_@", ":");
+
+ replace_all(output_symbol, "@_SPACE_@", " ");
+ replace_all(output_symbol, "@0@", "@_EPSILON_SYMBOL_@");
+ replace_all(output_symbol, "@_TAB_@", "\t");
+ replace_all(output_symbol, "@_COLON_@", ":");
+
+ if (epsilon_symbol.compare(input_symbol) == 0)
+ input_symbol="@_EPSILON_SYMBOL_@";
+ if (epsilon_symbol.compare(output_symbol) == 0)
+ output_symbol="@_EPSILON_SYMBOL_@";
+
+ HfstTransition <C> tr( atoi(a2), input_symbol,
+ output_symbol, weight );
+ add_transition( atoi(a1), tr );
+ }
+
+ else { // line could not be parsed
+ return false;
+ }
+ return true;
+ }
+
+ /* Create an HfstTransitionGraph as defined in AT&T format
in istream \a is or FILE \a file. \a epsilon_symbol defines
- how epsilon is represented.
+ how epsilon is represented.
- The functions is called by functions
+ The functions is called by functions
read_in_att_format(istream&, std::string) and
- read_in_att_format(FILE*, std::string).
+ read_in_att_format(FILE*, std::string).
If \a file is NULL, it is ignored and \a is is used.
If \a file is not NULL, it is used and \a is is ignored. */
HFSTDLL static HfstTransitionGraph read_in_att_format
@@ -1765,7 +1820,7 @@
if (! is.getline(line,255).eof())
break;
}
- else { /* we use FILEs */
+ else { /* we use FILEs */
if (NULL == fgets(line, 255, file))
break;
}
@@ -1775,7 +1830,7 @@
// an empty line signifying an empty transducer,
// a special case that is accepted if it is the only
// transducer in the stream
- if ( // empty line with or without a newline
+ if ( // empty line with or without a newline
(line[0] == '\0') ||
(line[0] == '\n' && line[1] == '\0') ||
// windows newline
@@ -1793,90 +1848,49 @@
if (*line == '-') // transducer separator line is "--"
return retval;
- // scan one line that can have a maximum of five fields
- char a1 [100]; char a2 [100]; char a3 [100];
- char a4 [100]; char a5 [100];
- // how many fields could be parsed
- int n = sscanf(line, "%s%s%s%s%s", a1, a2, a3, a4, a5);
-
- // set value of weight
- float weight = 0;
- if (n == 2) // a final state line with weight
- weight = atof(a2);
- if (n == 5) // a transition line with weight
- weight = atof(a5);
-
- if (n == 1 || n == 2) // a final state line
- retval.set_final_weight( atoi(a1), weight );
-
- else if (n == 4 || n == 5) { // a transition line
- std::string input_symbol=std::string(a3);
- std::string output_symbol=std::string(a4);
-
- // replace "@_SPACE_@"s with " " and "@0@"s with
- // "@_EPSILON_SYMBOL_@"
- replace_all(input_symbol, "@_SPACE_@", " ");
- replace_all(input_symbol, "@0@", "@_EPSILON_SYMBOL_@");
- replace_all(input_symbol, "@_TAB_@", "\t");
- replace_all(input_symbol, "@_COLON_@", ":");
-
- replace_all(output_symbol, "@_SPACE_@", " ");
- replace_all(output_symbol, "@0@", "@_EPSILON_SYMBOL_@");
- replace_all(output_symbol, "@_TAB_@", "\t");
- replace_all(output_symbol, "@_COLON_@", ":");
-
- if (epsilon_symbol.compare(input_symbol) == 0)
- input_symbol="@_EPSILON_SYMBOL_@";
- if (epsilon_symbol.compare(output_symbol) == 0)
- output_symbol="@_EPSILON_SYMBOL_@";
-
- HfstTransition <C> tr( atoi(a2), input_symbol,
- output_symbol, weight );
- retval.add_transition( atoi(a1), tr );
- }
-
- else { // line could not be parsed
- std::string message(line);
- HFST_THROW_MESSAGE
- (NotValidAttFormatException,
- message);
- }
+ if (! retval.add_att_line(line, epsilon_symbol))
+ {
+ std::string message(line);
+ HFST_THROW_MESSAGE
+ (NotValidAttFormatException,
+ message);
+ }
}
return retval;
}
- /** @brief Create an HfstTransitionGraph as defined in AT&T
- transducer format in istream \a is. \a epsilon_symbol
- defines how epsilon is represented.
- @pre \a is not at end, otherwise an exception is thrown.
- @note Multiple AT&T transducer definitions are separated with
+ /** @brief Create an HfstTransitionGraph as defined in AT&T
+ transducer format in istream \a is. \a epsilon_symbol
+ defines how epsilon is represented.
+ @pre \a is not at end, otherwise an exception is thrown.
+ @note Multiple AT&T transducer definitions are separated with
the line "--". */
HFSTDLL static HfstTransitionGraph read_in_att_format
(std::istream &is,
std::string epsilon_symbol,
- unsigned int & linecount)
+ unsigned int & linecount)
{
return read_in_att_format
(is, NULL /* a dummy variable */,
epsilon_symbol, linecount);
}
- /** @brief Create an HfstTransitionGraph as defined
- in AT&T transducer format in FILE \a file.
- \a epsilon_symbol defines how epsilon is represented.
- @pre \a is not at end, otherwise an exception is thrown.
- @note Multiple AT&T transducer definitions are separated with
+ /** @brief Create an HfstTransitionGraph as defined
+ in AT&T transducer format in FILE \a file.
+ \a epsilon_symbol defines how epsilon is represented.
+ @pre \a is not at end, otherwise an exception is thrown.
+ @note Multiple AT&T transducer definitions are separated with
the line "--". */
HFSTDLL static HfstTransitionGraph read_in_att_format
- (FILE *file,
+ (FILE *file,
std::string epsilon_symbol,
- unsigned int & linecount)
+ unsigned int & linecount)
{
return read_in_att_format
(std::cin /* a dummy variable */,
file, epsilon_symbol, linecount);
- }
+ }
// ----------------------------------------------
@@ -1889,7 +1903,7 @@
void substitute_(HfstSymbol old_symbol,
HfstSymbol new_symbol,
- bool input_side=true,
+ bool input_side=true,
bool output_side=true)
{
// ----- Go through all states -----
@@ -1900,7 +1914,7 @@
{
HfstTransition<C> &tr_it = it->operator[](i);
- // The substituting input and output symbols for the
+ // The substituting input and output symbols for the
// current transition.
HfstSymbol substituting_input_symbol
= tr_it.get_input_symbol();
@@ -1946,7 +1960,7 @@
/* A function that performs in-place substitutions in the graph
as defined in \a substitutions.
- substitutions[from_number] = to_number,
+ substitutions[from_number] = to_number,
if substitutions[from_number] = no_substitution, no substitution is made */
void substitute_(const HfstNumberVector &substitutions,
unsigned int no_substitution)
@@ -1967,7 +1981,7 @@
// If a substitution is to be performed,
if (new_inumber != no_substitution ||
- new_onumber != no_substitution)
+ new_onumber != no_substitution)
{
if (new_inumber != no_substitution)
add_symbol_to_alphabet(C::get_symbol(new_inumber));
@@ -2068,7 +2082,7 @@
unsigned int out_tr = tr_it.get_output_number();
if (in_tr == in_match && out_tr == out_match) {
it->erase(it->begin()+i); }
- else
+ else
{
if (in_tr == in_match || out_tr == in_match) {
in_match_used=true; }
@@ -2088,7 +2102,7 @@
protected:
/* A function that performs in-place-substitution in the graph. */
- void substitute_(const HfstSymbolPair &old_sp,
+ void substitute_(const HfstSymbolPair &old_sp,
const HfstSymbolPairSet &new_sps)
{
if (new_sps.empty())
@@ -2121,7 +2135,7 @@
// change the current transition so that it is equivalent
// to the first substituting transition in new_sps
- typename HfstSymbolPairSet::const_iterator IT
+ typename HfstSymbolPairSet::const_iterator IT
= new_sps.begin();
HfstTransition<C> tr
@@ -2148,7 +2162,7 @@
IT++;
}
- } // (substitution and scheduling done)
+ } // (substitution and scheduling done)
} // (all transitions of a state gone through)
@@ -2208,7 +2222,7 @@
// If a substitution is to be performed,
bool perform_substitution=false;
try {
- perform_substitution =
+ perform_substitution =
(*func)(transition_symbol_pair, substituting_transitions);
}
catch (const HfstException & e)
@@ -2216,10 +2230,10 @@
throw e;
}
if (perform_substitution)
- {
+ {
// change the transition to the first element
// in new_sps
- typename HfstSymbolPairSet::const_iterator IT
+ typename HfstSymbolPairSet::const_iterator IT
= substituting_transitions.begin();
if (! C::is_valid_symbol(IT->first) ||
@@ -2264,7 +2278,7 @@
IT++;
}
- } // Substitution and scheduling performed.
+ } // Substitution and scheduling performed.
} // All transitions gone through.
@@ -2287,16 +2301,16 @@
The public substitution functions.
---------------------------------------- */
- /** @brief Substitute \a old_symbol with \a new_symbol in
+ /** @brief Substitute \a old_symbol with \a new_symbol in
all transitions. \a input_side and \a output_side define
whether the substitution is made on input and output sides. */
HFSTDLL HfstTransitionGraph &
- substitute(const HfstSymbol &old_symbol,
+ substitute(const HfstSymbol &old_symbol,
const HfstSymbol &new_symbol,
- bool input_side=true,
+ bool input_side=true,
bool output_side=true) {
- if (! C::is_valid_symbol(old_symbol) ||
+ if (! C::is_valid_symbol(old_symbol) ||
! C::is_valid_symbol(new_symbol) ) {
HFST_THROW_MESSAGE
(EmptyStringException,
@@ -2313,7 +2327,7 @@
// if the substitution is made on both sides.
if (input_side && output_side) {
/* Special symbols are always included in the alphabet */
- if (! is_epsilon(old_symbol) &&
+ if (! is_epsilon(old_symbol) &&
! is_unknown(old_symbol) &&
! is_identity(old_symbol)) {
alphabet.erase(old_symbol); }
@@ -2371,7 +2385,7 @@
/** @brief Substitute all transitions as defined in \a substitutions.
- For each transition x:y, \a substitutions is searched and if
+ For each transition x:y, \a substitutions is searched and if
a mapping x:y -> X:Y is found, the transition x:y is replaced
with X:Y. If no mapping is found, the transition remains the same.
*/
@@ -2380,15 +2394,15 @@
{
// Convert from symbols to numbers
HfstNumberPairSubstitutions substitutions_;
- for (HfstSymbolPairSubstitutions::const_iterator it
+ for (HfstSymbolPairSubstitutions::const_iterator it
= substitutions.begin();
it != substitutions.end(); it++)
{
HfstNumberPair from_transition
- (get_symbol_number(it->first.first),
+ (get_symbol_number(it->first.first),
get_symbol_number(it->first.second));
HfstNumberPair to_transition
- (get_symbol_number(it->second.first),
+ (get_symbol_number(it->second.first),
get_symbol_number(it->second.second));
substitutions_[from_transition] = to_transition;
}
@@ -2401,9 +2415,9 @@
/** @brief Substitute all transitions \a sp with a set of transitions
\a sps. */
HFSTDLL HfstTransitionGraph &substitute
- (const HfstSymbolPair &sp, const HfstSymbolPairSet &sps)
+ (const HfstSymbolPair &sp, const HfstSymbolPairSet &sps)
{
- if (! C::is_valid_symbol(sp.first) ||
+ if (! C::is_valid_symbol(sp.first) ||
! C::is_valid_symbol(sp.second) ) {
HFST_THROW_MESSAGE
(EmptyStringException,
@@ -2412,7 +2426,7 @@
for (typename HfstSymbolPairSet::const_iterator it = sps.begin();
it != sps.end(); it++)
{
- if (! C::is_valid_symbol(it->first) ||
+ if (! C::is_valid_symbol(it->first) ||
! C::is_valid_symbol(it->second) ) {
HFST_THROW_MESSAGE
(EmptyStringException,
@@ -2424,15 +2438,15 @@
return *this;
}
- /** @brief Substitute all transitions \a old_pair with
+ /** @brief Substitute all transitions \a old_pair with
\a new_pair. */
HFSTDLL HfstTransitionGraph &substitute
- (const HfstSymbolPair &old_pair,
- const HfstSymbolPair &new_pair)
+ (const HfstSymbolPair &old_pair,
+ const HfstSymbolPair &new_pair)
{
- if (! C::is_valid_symbol(old_pair.first) ||
+ if (! C::is_valid_symbol(old_pair.first) ||
! C::is_valid_symbol(new_pair.first) ||
- ! C::is_valid_symbol(old_pair.second) ||
+ ! C::is_valid_symbol(old_pair.second) ||
! C::is_valid_symbol(new_pair.second) ) {
HFST_THROW_MESSAGE
(EmptyStringException,
@@ -2443,10 +2457,10 @@
substitute_(old_pair, new_pair_set);
return *this;
- }
+ }
/** @brief Substitute all transitions with a set of transitions as
- defined by function \a func.
+ defined by function \a func.
\a func takes as its argument a transition \a sp and inserts
into the set of transitions \a sps the transitions with which
@@ -2455,31 +2469,31 @@
whether any transition was inserted into \a sps. */
HFSTDLL HfstTransitionGraph &
substitute(bool (*func)
- (const HfstSymbolPair &sp, HfstSymbolPairSet &sps) )
- {
+ (const HfstSymbolPair &sp, HfstSymbolPairSet &sps) )
+ {
substitute_(func);
return *this;
}
- /* ----------------------------------------------------
+ /* ----------------------------------------------------
Substitute string pair with a transition graph
---------------------------------------------------- */
protected:
- /* Used in function
+ /* Used in function
substitute(const StringPair&, HfstTransitionGraph&) */
- struct substitution_data
+ struct substitution_data
{
HfstState origin_state;
HfstState target_state;
typename C::WeightType weight;
HfstTransitionGraph * substituting_graph;
- substitution_data(HfstState origin,
+ substitution_data(HfstState origin,
HfstState target,
- typename C::WeightType weight,
+ typename C::WeightType weight,
HfstTransitionGraph * substituting)
{
origin_state=origin;
@@ -2489,15 +2503,15 @@
}
};
- /* Used in function substitute(const StringPair&,
+ /* Used in function substitute(const StringPair&,
HfstTransitionGraph&)
- Add a copy of substituting graph with epsilon transitions between
+ Add a copy of substituting graph with epsilon transitions between
states and with weight as defined in \a sub. */
void add_substitution(const substitution_data &sub) {
// Epsilon transition to initial state of \a graph
HfstState s = add_state();
HfstTransition <C> epsilon_transition
- (s, C::get_epsilon(), C::get_epsilon(),
+ (s, C::get_epsilon(), C::get_epsilon(),
sub.weight);
add_transition(sub.origin_state, epsilon_transition);
@@ -2507,7 +2521,7 @@
// Copy \a graph
const HfstTransitionGraph * graph = sub.substituting_graph;
HfstState source_state=0;
- for (const_iterator it = graph->begin();
+ for (const_iterator it = graph->begin();
it != graph->end(); it++)
{
for (typename HfstTransitions::const_iterator tr_it
@@ -2517,7 +2531,7 @@
C data = tr_it->get_transition_data();
HfstTransition <C> transition
- (tr_it->get_target_state() + offset,
+ (tr_it->get_target_state() + offset,
data.get_input_symbol(),
data.get_output_symbol(),
data.get_weight());
@@ -2528,7 +2542,7 @@
}
// Epsilon transitions from final states of \a graph
- for (typename FinalWeightMap::const_iterator it
+ for (typename FinalWeightMap::const_iterator it
= graph->final_weight_map.begin();
it != graph->final_weight_map.end(); it++)
{
@@ -2546,28 +2560,28 @@
with a copy of \a graph.
Copies of \a graph are attached to this graph with
- epsilon transitions.
+ epsilon transitions.
The weights of the transitions to be substituted are copied
to epsilon transitions leaving from the source state of
the transitions to be substituted to the initial state
of a copy of \a graph.
- The final weights in \a
+ The final weights in \a
graph are copied to epsilon transitions leading from
the final states (after substitution non-final states)
of \a graph to target states of transitions
\a old_symbol : \a new_symbol (that are substituted)
- in this graph.
+ in this graph.
*/
HFSTDLL HfstTransitionGraph &
- substitute(const HfstSymbolPair &sp,
+ substitute(const HfstSymbolPair &sp,
const HfstTransitionGraph &graph) {
- if ( ! ( C::is_valid_symbol(sp.first) &&
+ if ( ! ( C::is_valid_symbol(sp.first) &&
C::is_valid_symbol(sp.second) ) ) {
HFST_THROW_MESSAGE
- (EmptyStringException,
+ (EmptyStringException,
"HfstTransitionGraph::substitute(const HfstSymbolPair&, "
"const HfstTransitionGraph&)");
}
@@ -2575,7 +2589,7 @@
// If neither symbol to be substituted is known to the graph,
// do nothing.
- if (alphabet.find(sp.first) == alphabet.end() &&
+ if (alphabet.find(sp.first) == alphabet.end() &&
alphabet.find(sp.second) == alphabet.end())
return *this;
@@ -2589,7 +2603,7 @@
{
// The transitions that are substituted, i.e. removed
- std::vector<typename HfstTransitions::iterator>
+ std::vector<typename HfstTransitions::iterator>
old_transitions;
// Go through all transitions
@@ -2599,28 +2613,28 @@
{
C data = tr_it->get_transition_data();
- // Whether there is anything to substitute
+ // Whether there is anything to substitute
// in this transition
if (data.get_input_symbol() == sp.first &&
- data.get_output_symbol() == sp.second)
+ data.get_output_symbol() == sp.second)
{
// schedule a substitution
substitutions.push_back(substitution_data
- (source_state,
- tr_it->get_target_state(),
+ (source_state,
+ tr_it->get_target_state(),
data.get_weight(),
const_cast<HfstTransitionGraph *>(&graph)));
// schedule the old transition to be deleted
old_transitions.push_back(tr_it);
}
// (one transition gone through)
- }
+ }
// (all transitions in a state gone through)
// Remove the substituted transitions
- for (typename std::vector<typename
+ for (typename std::vector<typename
HfstTransitions::iterator>::iterator IT =
- old_transitions.begin();
+ old_transitions.begin();
IT != old_transitions.end(); IT++) {
it->erase(*IT);
}
@@ -2630,7 +2644,7 @@
// (all states gone trough)
// Add the substitutions
- for (typename std::vector<substitution_data>::iterator IT
+ for (typename std::vector<substitution_data>::iterator IT
= substitutions.begin();
IT != substitutions.end(); IT++)
{
@@ -2666,7 +2680,7 @@
for (HfstState state = 0; state < limit; state++)
{
// The transitions that are substituted
- std::stack<typename HfstTransitions::iterator>
+ std::stack<typename HfstTransitions::iterator>
old_transitions;
// The transitions that will substitute
std::vector<HfstTransition <C> > new_transitions;
@@ -2678,21 +2692,21 @@
{
C data = tr_it->get_transition_data();
- // Whether there is anything to substitute
+ // Whether there is anything to substitute
// in this transition
if (data.get_weight() != 0 )
{
// schedule a substitution
new_transitions.push_back
- (HfstTransition <C> (tr_it->get_target_state(),
- data.get_input_symbol(),
- data.get_output_symbol(),
+ (HfstTransition <C> (tr_it->get_target_state(),
+ data.get_input_symbol(),
+ data.get_output_symbol(),
data.get_weight()));
// schedule the old transition to be deleted
old_transitions.push(tr_it);
}
// (one transition gone through)
- }
+ }
// (all transitions in a state gone through)
// Remove the substituted transitions
@@ -2702,7 +2716,7 @@
}
// Add the substituting transitions
- for (typename std::vector<HfstTransition <C> >::iterator IT
+ for (typename std::vector<HfstTransition <C> >::iterator IT
= new_transitions.begin();
IT != new_transitions.end(); IT++)
{
@@ -2767,11 +2781,11 @@
{
if ( ! ( C::is_valid_symbol(it->first) ))
{
- HFST_THROW_MESSAGE(EmptyStringException,
+ HFST_THROW_MESSAGE(EmptyStringException,
"HfstTransitionGraph::substitute "
"(const std::map<HfstSymbol, HfstTransitionGraph> &)");
}
- if (!symbol_found && alphabet.find(it->first) != alphabet.end())
+ if (!symbol_found && alphabet.find(it->first) != alphabet.end())
{
symbol_found = true;
}
@@ -2796,7 +2810,7 @@
{
// The transitions that are substituted, i.e. removed
- std::stack<typename HfstTransitions::iterator>
+ std::stack<typename HfstTransitions::iterator>
old_transitions;
// Go through all transitions
@@ -2806,7 +2820,7 @@
{
C data = tr_it->get_transition_data();
- // Whether there is anything to substitute
+ // Whether there is anything to substitute
// in this transition
String istr = data.get_input_symbol();
String ostr = data.get_output_symbol();
@@ -2828,7 +2842,7 @@
// schedule a substitution
substitution_data sd
(source_state,
- tr_it->get_target_state(),
+ tr_it->get_target_state(),
data.get_weight(),
&(map_it_input->second));
substitutions.push_back(sd);
@@ -2838,11 +2852,11 @@
substitutions_performed_for_symbols.insert(istr);
}
// (one transition gone through)
- }
+ }
// (all transitions in a state gone through)
// Remove the substituted transitions
- while (!old_transitions.empty())
+ while (!old_transitions.empty())
{
it->erase(old_transitions.top());
old_transitions.pop();
@@ -2871,7 +2885,7 @@
}
// Add the substitutions
- for (typename std::vector<substitution_data>::iterator IT
+ for (typename std::vector<substitution_data>::iterator IT
= substitutions.begin();
IT != substitutions.end(); IT++)
{
@@ -2883,7 +2897,7 @@
- HFSTDLL bool marker2weight(const std::string & str, float & weight)
+ HFSTDLL bool marker2weight(const std::string & str, float & weight)
{
if (str.size() < 3)
return false;
@@ -2895,9 +2909,9 @@
sstream >> weight;
if (sstream.fail()) {
return false;
- }
+ }
return true;
- }
+ }
HFSTDLL HfstTransitionGraph & substitute_markers_with_weights() {
@@ -2906,7 +2920,7 @@
for (HfstState state = 0; state < limit; state++)
{
// The transitions that are substituted
- std::stack<typename HfstTransitions::iterator>
+ std::stack<typename HfstTransitions::iterator>
old_transitions;
// The transitions that will substitute
std::vector<HfstTransition <C> > new_transitions;
@@ -2919,17 +2933,17 @@
C data = tr_it->get_transition_data();
float weight = 0;
- // Whether there is anything to substitute
+ // Whether there is anything to substitute
// in this transition
- if ( (!marker2weight(data.get_input_symbol(), weight)) &&
+ if ( (!marker2weight(data.get_input_symbol(), weight)) &&
marker2weight(data.get_output_symbol(), weight) )
{
//std::cerr << "got weight '" << weight << "'" << std::endl;
// schedule a substitution
new_transitions.push_back
- (HfstTransition <C> (tr_it->get_target_state(),
- data.get_input_symbol(),
- hfst::internal_epsilon,
+ (HfstTransition <C> (tr_it->get_target_state(),
+ data.get_input_symbol(),
+ hfst::internal_epsilon,
weight));
// schedule the old transition to be deleted
old_transitions.push(tr_it);
@@ -2944,7 +2958,7 @@
}
else {}
// (one transition gone through)
- }
+ }
// (all transitions in a state gone through)
// Remove the substituted transitions
@@ -2954,7 +2968,7 @@
}
// Add the substituting transitions
- for (typename std::vector<HfstTransition <C> >::iterator IT
+ for (typename std::vector<HfstTransition <C> >::iterator IT
= new_transitions.begin();
IT != new_transitions.end(); IT++)
{
@@ -2980,7 +2994,7 @@
weight_markers.pop();
}
- return *this;
+ return *this;
}
@@ -3003,15 +3017,15 @@
---------------------------- */
- /** @brief Insert freely any number of \a symbol_pair in
+ /** @brief Insert freely any number of \a symbol_pair in
the graph with weight \a weight. */
HFSTDLL HfstTransitionGraph &insert_freely
- (const HfstSymbolPair &symbol_pair, typename C::WeightType weight)
- {
- if ( ! ( C::is_valid_symbol(symbol_pair.first) &&
+ (const HfstSymbolPair &symbol_pair, typename C::WeightType weight)
+ {
+ if ( ! ( C::is_valid_symbol(symbol_pair.first) &&
C::is_valid_symbol(symbol_pair.second) ) ) {
HFST_THROW_MESSAGE
- (EmptyStringException,
+ (EmptyStringException,
"HfstTransitionGraph::insert_freely"
"(const HfstSymbolPair&, W)");
}
@@ -3021,8 +3035,8 @@
HfstState source_state=0;
for (iterator it = begin(); it != end(); it++) {
- HfstTransition <C> tr( source_state, symbol_pair.first,
- symbol_pair.second, weight );
+ HfstTransition <C> tr( source_state, symbol_pair.first,
+ symbol_pair.second, weight );
it->push_back(tr);
source_state++;
}
@@ -3030,20 +3044,20 @@
return *this;
}
- /** @brief Insert freely any number of any symbol in \a symbol_pairs in
+ /** @brief Insert freely any number of any symbol in \a symbol_pairs in
the graph with weight \a weight. */
HFSTDLL HfstTransitionGraph &insert_freely
- (const HfstSymbolPairSet &symbol_pairs,
- typename C::WeightType weight)
+ (const HfstSymbolPairSet &symbol_pairs,
+ typename C::WeightType weight)
{
- for (typename HfstSymbolPairSet::const_iterator symbols_it
+ for (typename HfstSymbolPairSet::const_iterator symbols_it
= symbol_pairs.begin();
symbols_it != symbol_pairs.end(); symbols_it++)
{
- if ( ! ( C::is_valid_symbol(symbols_it->first) &&
+ if ( ! ( C::is_valid_symbol(symbols_it->first) &&
C::is_valid_symbol(symbols_it->second) ) ) {
HFST_THROW_MESSAGE
- (EmptyStringException,
+ (EmptyStringException,
"HfstTransitionGraph::insert_freely"
"(const HfstSymbolPairSet&, W)");
}
@@ -3053,13 +3067,13 @@
}
HfstState source_state=0;
- for (iterator it = begin(); it != end(); it++)
+ for (iterator it = begin(); it != end(); it++)
{
- for (typename HfstSymbolPairSet::const_iterator symbols_it
+ for (typename HfstSymbolPairSet::const_iterator symbols_it
= symbol_pairs.begin();
symbols_it != symbol_pairs.end(); symbols_it++)
{
- HfstTransition <C> tr( source_state, symbols_it->first,
+ HfstTransition <C> tr( source_state, symbols_it->first,
symbols_it->second, weight );
it->push_back(tr);
}
@@ -3095,23 +3109,23 @@
/** @brief Harmonize this HfstTransitionGraph and \a another.
- In harmonization the unknown and identity symbols in
+ In harmonization the unknown and identity symbols in
transitions of both graphs are expanded according to
- the symbols that are previously unknown to the graph.
+ the symbols that are previously unknown to the graph.
For example the graphs
- \verbatim
+ \verbatim
[a:b ?:?]
[c:d ? ?:c]
\endverbatim
are expanded to
\verbatim
- [ a:b [?:? | ?:c | ?:d | c:d | d:c | c:? | d:?] ]
+ [ a:b [?:? | ?:c | ?:d | c:d | d:c | c:? | d:?] ]
[ c:d [? | a | b] [?:c| a:c | b:?] ]
\endverbatim
when harmonized.
- The symbol "?" means \@_UNKNOWN_SYMBOL_\@ in either or
- both sides of a transition
+ The symbol "?" means \@_UNKNOWN_SYMBOL_\@ in either or
+ both sides of a transition
(transitions of type [?:x], [x:?] and [?:?]).
The transition [?] means [\@_IDENTITY_SYMBOL_\@].
@@ -3119,7 +3133,7 @@
that take two or more graphs as their arguments, unless otherwise
said.
*/
- HFSTDLL HfstTransitionGraph &harmonize(HfstTransitionGraph &another)
+ HFSTDLL HfstTransitionGraph &harmonize(HfstTransitionGraph &another)
{
HarmonizeUnknownAndIdentitySymbols foo(*this, another);
return *this;
@@ -3136,7 +3150,7 @@
it is created as well as its target state.
@return The final state of path \a spv, when \a it is at end. */
- HfstState disjunct(const StringPairVector &spv,
+ HfstState disjunct(const StringPairVector &spv,
StringPairVector::const_iterator &it,
HfstState s)
{
@@ -3148,7 +3162,7 @@
HfstTransitions tr = state_vector[s];
bool transition_found=false;
/* The target state of the transition followed or added */
- HfstState next_state;
+ HfstState next_state;
// Find the transition
// (Searching is slow?)
@@ -3181,15 +3195,15 @@
public:
- /** @brief Disjunct this graph with a one-path graph
- defined by string pair vector \a spv with weight \a weight.
+ /** @brief Disjunct this graph with a one-path graph
+ defined by string pair vector \a spv with weight \a weight.
@pre This graph must be a trie where all weights are in
- final states, i.e. all transitions have a zero weight.
+ final states, i.e. all transitions have a zero weight.
There is no way to test whether a graph is a trie, so the use
- of this function is probably limited to fast construction
- of a lexicon. Here is an example:
+ of this function is probably limited to fast construction
+ of a lexicon. Here is an example:
\verbatim
HfstBasicTransducer lexicon;
@@ -3201,16 +3215,16 @@
*/
HFSTDLL HfstTransitionGraph &disjunct
- (const StringPairVector &spv, typename C::WeightType weight)
+ (const StringPairVector &spv, typename C::WeightType weight)
{
StringPairVector::const_iterator it = spv.begin();
HfstState final_state = disjunct(spv, it, INITIAL_STATE);
// Set the weight of final state
- if (is_final_state(final_state))
+ if (is_final_state(final_state))
{
float old_weight = get_final_weight(final_state);
- if (old_weight < weight)
+ if (old_weight < weight)
return *this; /* The same path with smaller weight remains */
}
set_final_weight(final_state, weight);
@@ -3250,8 +3264,8 @@
for (std::set<std::string>::const_iterator alpha_it = alphabet.begin();
alpha_it != alphabet.end(); alpha_it++)
{
- if (symbols_present.find(*alpha_it) ==
- symbols_present.end() &&
+ if (symbols_present.find(*alpha_it) ==
+ symbols_present.end() &&
! is_special_symbol(*alpha_it))
{
add_transition
@@ -3268,7 +3282,7 @@
{
StringSet flags;
for (StringSet::const_iterator it = alphabet.begin();
- it != alphabet.end(); it++)
+ it != alphabet.end(); it++)
{
if (FdOperation::is_diacritic(*it)) {
flags.insert(*it);
@@ -3281,7 +3295,7 @@
// of a transducer after \a flag has been eliminated from the transducer.
// If \a flag is the empty string, all flags have been eliminated.
HFSTDLL bool purge_symbol(const std::string & symbol, const std::string & flag)
- {
+ {
if (! FdOperation::is_diacritic(symbol))
return false;
if (flag == "")
@@ -3292,7 +3306,7 @@
}
// Replace arcs in \a transducer that use flag \a flag with epsilon arcs
- // and remove \a flag from alphabet of \a transducer. If \a flag is the empty
+ // and remove \a flag from alphabet of \a transducer. If \a flag is the empty
// string, replace/remove all flags.
HFSTDLL void flag_purge(const std::string & flag)
{
@@ -3317,7 +3331,7 @@
// (2) Go through the alphabet
StringSet extra_symbols;
for (StringSet::const_iterator it = alphabet.begin();
- it != alphabet.end(); it++)
+ it != alphabet.end(); it++)
{
if (purge_symbol(*it, flag))
extra_symbols.insert(*it);
@@ -3332,7 +3346,7 @@
std::vector<int> distance_of_state;
std::vector<std::set<HfstState> > states_at_distance;
- /* Initialize the TopologicalSort by reserving space for a transducer
+ /* Initialize the TopologicalSort by reserving space for a transducer
with biggest state number \a biggest_state_number, */
HFSTDLL void set_biggest_state_number(unsigned int biggest_state_number)
{
@@ -3347,12 +3361,12 @@
if (state > (distance_of_state.size() - 1))
{
std::cerr << "ERROR in TopologicalSort::set_state_at_distance: first argument ("
- << state << ") is out of range (should be < " << distance_of_state.size()
+ << state << ") is out of range (should be < " << distance_of_state.size()
<< ")" << std::endl;
}
// if there is nothing on index 'state',
// push back empty sets of states up to index 'state', including
- while (distance + 1 > (unsigned int)states_at_distance.size())
+ while (distance + 1 > (unsigned int)states_at_distance.size())
{
std::set<HfstState> empty_set;
states_at_distance.push_back(empty_set);
@@ -3373,7 +3387,7 @@
{
// if there is nothing on index 'state',
// push back empty sets of states up to index 'state', including
- while (distance > (states_at_distance.size() - 1))
+ while (distance > (states_at_distance.size() - 1))
{
std::set<HfstState> empty_set;
states_at_distance.push_back(empty_set);
@@ -3384,7 +3398,7 @@
enum SortDistance { MaximumDistance, MinimumDistance };
- /*
+ /*
Get a topological (maximum distance) sort of this graph.
@return A vector of sets of states. At each vector index ind, the
result contains the set of all states whose (maximum) distance from
@@ -3406,31 +3420,31 @@
std::set<HfstState> new_states;
// go through all states at current distance
- const std::set<HfstState> & states =
+ const std::set<HfstState> & states =
TopSort.get_states_at_distance(current_distance);
for (StateIt state_it = states.begin();
state_it != states.end(); state_it++)
{
// go through all transitions of each state
- const HfstTransitions & transitions
+ const HfstTransitions & transitions
= this->state_vector.at(*state_it);
- for (typename HfstTransitions::const_iterator transition_it
+ for (typename HfstTransitions::const_iterator transition_it
= transitions.begin();
transition_it != transitions.end(); transition_it++)
{
new_states_found = true;
new_states.insert(transition_it->get_target_state());
- }
+ }
// all transitions gone through
}
// all states gone through
// set each accessible state at distance one higher than the
// current distance
- for (StateIt it = new_states.begin();
+ for (StateIt it = new_states.begin();
it != new_states.end(); it++)
{
- TopSort.set_state_at_distance(*it, current_distance + 1, (dist == MaximumDistance));
+ TopSort.set_state_at_distance(*it, current_distance + 1, (dist == MaximumDistance));
}
current_distance++;
}
@@ -3439,7 +3453,7 @@
return TopSort.states_at_distance;
}
- /** The length of longest string accepted by this graph.
+ /** The length of longest string accepted by this graph.
If no string is accepted, return -1. */
HFSTDLL int longest_path_size()
{
@@ -3448,7 +3462,7 @@
// go through all sets of states in descending order
for (int distance = states_sorted.size() - 1; distance >= 0; distance--)
{
- const std::set<HfstState> & states
+ const std::set<HfstState> & states
= states_sorted.at((unsigned int)distance);
// go through all states in a set
for (std::set<HfstState>::const_iterator it = states.begin();
@@ -3466,7 +3480,7 @@
return -1;
}
- /** The lengths of strings accepted by this graph, in descending order.
+ /** The lengths of strings accepted by this graph, in descending order.
If not string is accepted, return an empty vector. */
HFSTDLL std::vector<unsigned int> path_sizes()
{
@@ -3476,7 +3490,7 @@
// go through all sets of states in descending order
for (int distance = states_sorted.size() - 1; distance >= 0; distance--)
{
- const std::set<HfstState> & states
+ const std::set<HfstState> & states
= states_sorted.at((unsigned int)distance);
// go through all states in a set
for (std::set<HfstState>::const_iterator it = states.begin();
@@ -3510,8 +3524,8 @@
}
state_weights[state] = total_weight;
- // Go through all transitions in this state
- const HfstBasicTransducer::HfstTransitions &transitions
+ // Go through all transitions in this state
+ const HfstBasicTransducer::HfstTransitions &transitions
= this->operator[](state);
for (HfstBasicTransducer::HfstTransitions::const_iterator it
= transitions.begin();
@@ -3555,27 +3569,27 @@
if (has_negative_epsilon_cycles(state, 0, state_weights))
return true;
}
- return false;
+ return false;
}
HFSTDLL bool is_infinitely_ambiguous
- (HfstState state,
+ (HfstState state,
std::set<HfstState> &epsilon_path_states,
std::vector<unsigned int> &states_handled)
{
if (states_handled[state] != 0)
return false;
- // Go through all transitions in this state
- const HfstBasicTransducer::HfstTransitions &transitions
+ // Go through all transitions in this state
+ const HfstBasicTransducer::HfstTransitions &transitions
= this->operator[](state);
for (HfstBasicTransducer::HfstTransitions::const_iterator it
= transitions.begin();
it != transitions.end(); it++)
{
- // (Diacritics are also treated as epsilons, although it might cause false
- // positive results, because loops with diacritics can be invalidated by
- // other diacritics.)
+ // (Diacritics are also treated as epsilons, although it might cause false
+ // positive results, because loops with diacritics can be invalidated by
+ // other diacritics.)
if ( is_epsilon(it->get_input_symbol()) ||
FdOperation::is_diacritic(it->get_input_symbol()) )
{
@@ -3591,7 +3605,7 @@
return true;
}
epsilon_path_states.erase(state);
- }
+ }
}
// mark state as handled
states_handled[state] = 1;
@@ -3612,33 +3626,51 @@
return false;
}
+ bool is_possible_flag(std::string symbol, StringVector & fds, bool obey_flags)
+ {
+ if (FdOperation::is_diacritic(symbol))
+ {
+ FlagDiacriticTable FdT;
+ fds.push_back(symbol);
+ if ((!obey_flags) || FdT.is_valid_string(fds))
+ { return true; }
+ else
+ {
+ fds.pop_back();
+ return false;
+ }
+ }
+ return false;
+ }
+
HFSTDLL bool is_lookup_infinitely_ambiguous
(const HfstOneLevelPath& s,
unsigned int& index, HfstState state,
- std::set<HfstState> &epsilon_path_states
- )
+ std::set<HfstState> &epsilon_path_states,
+ StringVector & fds, bool obey_flags)
{
- // Whether the end of the lookup path s has been reached
+ // Whether the end of the lookup path s has been reached
bool only_epsilons=false;
if ((unsigned int)s.second.size() == index)
{
only_epsilons=true;
}
- // Go through all transitions in this state
- const HfstBasicTransducer::HfstTransitions &transitions
+ // Go through all transitions in this state
+ const HfstBasicTransducer::HfstTransitions &transitions
= this->operator[](state);
for (HfstBasicTransducer::HfstTransitions::const_iterator it
= transitions.begin();
it != transitions.end(); it++)
{
- // CASE 1: Input epsilons do not consume a symbol in the lookup path s,
- // so they can be added freely.
- // (Diacritics are also treated as epsilons, although it might cause false
- // positive results, because loops with diacritics can be invalidated by
- // other diacritics.)
+ // CASE 1: Input epsilons do not consume a symbol in the lookup path s,
+ // so they can be added freely.
+ // (Diacritics are also treated as epsilons, although it might cause false
+ // positive results, because loops with diacritics can be invalidated by
+ // other diacritics.)
+ bool possible_flag = is_possible_flag(it->get_input_symbol(), fds, obey_flags);
if ( is_epsilon(it->get_input_symbol()) ||
- FdOperation::is_diacritic(it->get_input_symbol()) )
+ possible_flag )
{
epsilon_path_states.insert(state);
if (epsilon_path_states.find(it->get_target_state())
@@ -3647,23 +3679,25 @@
return true;
}
if (is_lookup_infinitely_ambiguous
- (s, index, it->get_target_state(), epsilon_path_states))
+ (s, index, it->get_target_state(), epsilon_path_states, fds, obey_flags))
{
return true;
}
epsilon_path_states.erase(state);
+ if (possible_flag)
+ { fds.pop_back(); }
}
- /* CASE 2: Other input symbols consume a symbol in the lookup path s,
- so they can be added only if the end of the lookup path s has not
+ /* CASE 2: Other input symbols consume a symbol in the lookup path s,
+ so they can be added only if the end of the lookup path s has not
been reached. */
else if (! only_epsilons)
{
bool continu = false;
if (it->get_input_symbol().compare(s.second.at(index)) == 0)
continu = true;
- else if ((it->get_input_symbol().compare("@_UNKNOWN_SYMBOL_@") ||
- it->get_input_symbol().compare("@_IDENTITY_SYMBOL_@"))
+ else if (((it->get_input_symbol().compare("@_UNKNOWN_SYMBOL_@") == 0) ||
+ (it->get_input_symbol().compare("@_IDENTITY_SYMBOL_@") == 0))
&&
(alphabet.find(s.second.at(index)) == alphabet.end()))
{
@@ -3672,39 +3706,41 @@
if (continu)
{
- index++; // consume an input symbol in the lookup path s
+ index++; // consume an input symbol in the lookup path s
std::set<HfstState> empty_set;
if (is_lookup_infinitely_ambiguous
- (s, index, it->get_target_state(), empty_set))
+ (s, index, it->get_target_state(), empty_set, fds, obey_flags))
{
return true;
}
- index--; // add the input symbol back to the lookup path s.
+ index--; // add the input symbol back to the lookup path s.
}
}
}
return false;
}
- HFSTDLL bool is_lookup_infinitely_ambiguous(const HfstOneLevelPath & s)
+ HFSTDLL bool is_lookup_infinitely_ambiguous(const HfstOneLevelPath & s, bool obey_flags=false)
{
std::set<HfstState> epsilon_path_states;
epsilon_path_states.insert(0);
unsigned int index=0;
+ StringVector fds;
return is_lookup_infinitely_ambiguous(s, index, INITIAL_STATE,
- epsilon_path_states);
+ epsilon_path_states, fds, obey_flags);
}
- HFSTDLL bool is_lookup_infinitely_ambiguous(const StringVector & s)
+ HFSTDLL bool is_lookup_infinitely_ambiguous(const StringVector & s, bool obey_flags=false)
{
std::set<HfstState> epsilon_path_states;
epsilon_path_states.insert(0);
unsigned int index=0;
HfstOneLevelPath path((float)0, s);
+ StringVector fds;
return is_lookup_infinitely_ambiguous(path, index, INITIAL_STATE,
- epsilon_path_states);
+ epsilon_path_states, fds, obey_flags);
}
@@ -3712,16 +3748,29 @@
HFSTDLL static void push_back_to_two_level_path
(HfstTwoLevelPath &path,
const StringPair &sp,
- const float &weight)
+ const float &weight,
+ StringVector * fds_so_far = NULL)
{
path.second.push_back(sp);
path.first = path.first + weight;
+ if (fds_so_far != NULL)
+ {
+ if (FdOperation::is_diacritic(sp.first))
+ { fds_so_far->push_back(sp.first); }
+ }
}
HFSTDLL static void pop_back_from_two_level_path
(HfstTwoLevelPath &path,
- const float &weight)
+ const float &weight,
+ StringVector * fds_so_far = NULL)
{
+ if (fds_so_far != NULL)
+ {
+ StringPair sp = path.second.back();
+ if (FdOperation::is_diacritic(sp.first))
+ { fds_so_far->pop_back(); }
+ }
path.second.pop_back();
path.first = path.first - weight;
}
@@ -3754,7 +3803,8 @@
const StringVector &lookup_path,
const unsigned int &lookup_index,
const StringSet &alphabet,
- bool &input_symbol_consumed)
+ bool &input_symbol_consumed,
+ StringVector * fds_so_far = NULL)
{
std::string isymbol = transition.get_input_symbol();
@@ -3769,8 +3819,8 @@
// the current symbol is not found in the alphabet
// of the transducer.
( (is_identity(isymbol) || is_unknown(isymbol)) &&
- (alphabet.find(lookup_path.at(lookup_index))
- == alphabet.end()) )
+ (alphabet.find(lookup_path.at(lookup_index))
+ == alphabet.end()) )
)
{
input_symbol_consumed=true;
@@ -3780,18 +3830,37 @@
// Whether there are more symbols in lookup_path or not,
// we can always go further if the input symbol of the transition
// is an epsilon or a flag diacritic.
- if ( is_epsilon(isymbol) ||
- FdOperation::is_diacritic(isymbol) )
+ if ( is_epsilon(isymbol) )
{
input_symbol_consumed=false;
return true;
}
+ if ( FdOperation::is_diacritic(isymbol) )
+ {
+ if (fds_so_far == NULL)
+ {
+ input_symbol_consumed=false;
+ return true;
+ }
+ else
+ {
+ FlagDiacriticTable FdT;
+ fds_so_far->push_back(isymbol);
+ bool valid = FdT.is_valid_string(*fds_so_far);
+ fds_so_far->pop_back();
+ if (valid)
+ {
+ input_symbol_consumed=false;
+ return true;
+ }
+ }
+ }
// No matches.
return false;
}
- HFSTDLL void lookup_fd
+ HFSTDLL void lookup
(const StringVector &lookup_path,
HfstTwoLevelPaths &results,
HfstState state,
@@ -3799,8 +3868,9 @@
HfstTwoLevelPath &path_so_far,
StringSet &alphabet,
HfstEpsilonHandler Eh,
- size_t infinite_cutoff,
- float * max_weight = NULL)
+ size_t max_epsilon_cycles,
+ float * max_weight = NULL,
+ StringVector * flag_diacritic_path = NULL)
{
// Check whether the number of input epsilon cycles is exceeded
if (! Eh.can_continue(state)) {
@@ -3814,8 +3884,8 @@
// If we are at the end of lookup_path,
if (lookup_index == lookup_path.size())
{
- // and if the current state is final,
- if (this->is_final_state(state))
+ // and if the current state is final,
+ if (this->is_final_state(state))
{
// path_so_far is a valid result if max_weight is not exceeded
add_to_results
@@ -3825,25 +3895,25 @@
// Whether there are more symbols in lookup_path or not,
// go through all transitions in the current state.
- const HfstBasicTransducer::HfstTransitions &transitions
+ const HfstBasicTransducer::HfstTransitions &transitions
= this->operator[](state);
- for (HfstBasicTransducer::HfstTransitions::const_iterator it
+ for (HfstBasicTransducer::HfstTransitions::const_iterator it
= transitions.begin();
it != transitions.end(); it++)
{
bool input_symbol_consumed=false;
if ( is_possible_transition
- (*it, lookup_path, lookup_index, alphabet,
- input_symbol_consumed) )
+ (*it, lookup_path, lookup_index, alphabet,
+ input_symbol_consumed, flag_diacritic_path) )
{
// update path_so_far and lookup_index
std::string istr;
std::string ostr;
// identity symbol is replaced with the lookup symbol
- if (is_identity(it->get_input_symbol()))
+ if (is_identity(it->get_input_symbol()))
{
- istr = lookup_path.at(lookup_index);
+ istr = lookup_path.at(lookup_index);
ostr = istr;
}
else
@@ -3857,17 +3927,17 @@
ostr = std::string("?");
else*/
ostr = it->get_output_symbol();
- }
+ }
push_back_to_two_level_path
(path_so_far,
StringPair(istr, ostr),
- it->get_weight());
-
+ it->get_weight(), flag_diacritic_path);
+
HfstEpsilonHandler * Ehp = NULL;
if (input_symbol_consumed) {
lookup_index++;
- Ehp = new HfstEpsilonHandler(infinite_cutoff);
+ Ehp = new HfstEpsilonHandler(max_epsilon_cycles);
}
else {
Eh.push_back(state);
@@ -3875,10 +3945,10 @@
}
// call lookup for the target state of the transition
- lookup_fd(lookup_path, results, it->get_target_state(),
- lookup_index, path_so_far, alphabet, *Ehp, infinite_cutoff, max_weight);
+ lookup(lookup_path, results, it->get_target_state(),
+ lookup_index, path_so_far, alphabet, *Ehp, max_epsilon_cycles, max_weight, flag_diacritic_path);
- // return to the original values of path_so_far
+ // return to the original values of path_so_far
// and lookup_index
if (input_symbol_consumed) {
lookup_index--;
@@ -3889,34 +3959,40 @@
// of Eh is automatically called next
}
- pop_back_from_two_level_path(path_so_far, it->get_weight());
+ pop_back_from_two_level_path(path_so_far, it->get_weight(), flag_diacritic_path);
}
}
}
- HFSTDLL void lookup_fd
+ HFSTDLL void lookup
(const StringVector &lookup_path,
HfstTwoLevelPaths &results,
- size_t * infinite_cutoff = NULL,
- float * max_weight = NULL)
+ size_t * max_epsilon_cycles = NULL,
+ float * max_weight = NULL,
+ bool obey_flags = false)
{
HfstState state = 0;
unsigned int lookup_index = 0;
HfstTwoLevelPath path_so_far;
StringSet alphabet = this->get_alphabet();
- if (infinite_cutoff != NULL)
+ StringVector * flag_diacritic_path = (obey_flags)? new StringVector() : NULL;
+
+ if (max_epsilon_cycles != NULL)
{
- HfstEpsilonHandler Eh(*infinite_cutoff);
- lookup_fd(lookup_path, results, state, lookup_index, path_so_far,
- alphabet, Eh, *infinite_cutoff, max_weight);
+ HfstEpsilonHandler Eh(*max_epsilon_cycles);
+ lookup(lookup_path, results, state, lookup_index, path_so_far,
+ alphabet, Eh, *max_epsilon_cycles, max_weight, flag_diacritic_path);
}
else
{
HfstEpsilonHandler Eh(100000);
- lookup_fd(lookup_path, results, state, lookup_index, path_so_far,
- alphabet, Eh, 100000, max_weight);
+ lookup(lookup_path, results, state, lookup_index, path_so_far,
+ alphabet, Eh, 100000, max_weight, flag_diacritic_path);
}
+
+ if (flag_diacritic_path != NULL)
+ delete flag_diacritic_path;
}
@@ -3941,8 +4017,8 @@
else if ((input_side && is_special_symbol(istr)) || (!input_side && is_special_symbol(ostr)))
{
throw "error: special symbol detected in compile-replace regular expression";
- }
- else
+ }
+ else
{}
if ((input_side && ("^[" == istr)) || (!input_side && ("^[" == ostr)))
@@ -3969,9 +4045,9 @@
// paths are stored in \a full_paths. \a path is used to keep track of each path so
// far. Weights are currently ignored.
HFSTDLL void find_regexp_paths
- (HfstState s,
- std::set<HfstState> & states_visited,
- std::vector<std::pair<std::string, std::string> > & path,
+ (HfstState s,
+ std::set<HfstState> & states_visited,
+ std::vector<std::pair<std::string, std::string> > & path,
HfstReplacements & full_paths, bool input_side)
{
// no cycles allowed inside "^[" and "^]"
@@ -3979,9 +4055,9 @@
states_visited.insert(s);
// go through all transitions
- const HfstBasicTransducer::HfstTransitions &transitions
+ const HfstBasicTransducer::HfstTransitions &transitions
= this->operator[](s);
- for (HfstBasicTransducer::HfstTransitions::const_iterator it
+ for (HfstBasicTransducer::HfstTransitions::const_iterator it
= transitions.begin();
it != transitions.end(); it++)
{
@@ -4006,7 +4082,7 @@
path,
full_paths, input_side);
path.pop_back();
- }
+ }
}
// all transitions gone through
states_visited.erase(s);
@@ -4023,13 +4099,13 @@
// Weights are currently ignored.
HFSTDLL void find_regexp_paths
(HfstState s,
- std::vector<std::pair<HfstState, std::vector<std::pair<std::string, std::string> > > > & full_paths,
+ std::vector<std::pair<HfstState, std::vector<std::pair<std::string, std::string> > > > & full_paths,
bool input_side)
{
// go through all transitions
- const HfstBasicTransducer::HfstTransitions &transitions
+ const HfstBasicTransducer::HfstTransitions &transitions
= this->operator[](s);
- for (HfstBasicTransducer::HfstTransitions::const_iterator it
+ for (HfstBasicTransducer::HfstTransitions::const_iterator it
= transitions.begin();
it != transitions.end(); it++)
{
@@ -4043,7 +4119,7 @@
}*/
std::set<HfstState> states_visited;
states_visited.insert(s);
- std::vector<std::pair<std::string, std::string> > path;
+ std::vector<std::pair<std::string, std::string> > path;
path.push_back(std::pair<std::string, std::string>(istr, ostr));
find_regexp_paths(it->get_target_state(), states_visited, path, full_paths, input_side);
//fprintf(stderr, "%u regexp paths found for state %u\n", (unsigned int)full_paths.size(), s); // debug
@@ -4080,7 +4156,7 @@
// Todo: copy alphabet? harmonize graphs?
HFSTDLL void insert_transducer(HfstState state1, HfstState state2, const HfstTransitionGraph & graph)
{
- HfstState offset = add_state();
+ HfstState offset = add_state();
HfstState source_state=0;
for (const_iterator it = graph.begin();
it != graph.end(); it++)
@@ -4126,7 +4202,7 @@
// If not found, add a new state to \a intersection, add it to \a state_map and return it.
// \a was_new_state specifies whether a new state was added.
static HfstState find_target_state
- (HfstState target1, HfstState target2, StateMap & state_map,
+ (HfstState target1, HfstState target2, StateMap & state_map,
HfstTransitionGraph & intersection, bool & was_new_state)
{
StatePair state_pair(target1, target2);
@@ -4158,8 +4234,8 @@
// The sum of weight is copied to the resulting intersection.
float transition_weight = tr1.get_weight() + tr2.get_weight();
intersection.add_transition
- (state, HfstTransition <C>
- (retval, tr1.get_input_symbol(), tr1.get_output_symbol(), transition_weight));
+ (state, HfstTransition <C>
+ (retval, tr1.get_input_symbol(), tr1.get_output_symbol(), transition_weight));
// For each new state added, check if the corresponding states in \a graph1 and \a graph2
// are final. If they are, make the new state final with the sum of final weights.
if (was_new_state && (graph1.is_final_state(target1) && graph2.is_final_state(target2)))
@@ -4281,7 +4357,7 @@
HfstState retval = find_target_state
(graph_target, merger_target, state_map, result, was_new_state);
result.add_transition
- (result_state, HfstTransition <C>
+ (result_state, HfstTransition <C>
(retval, graph_transition.get_input_symbol(), graph_transition.get_output_symbol(), graph_transition.get_weight()));
// For each new state added, check if the corresponding states in \a graph and \a merger
// are final. If they are, make the new state final with the sum of final weights.
@@ -4312,13 +4388,13 @@
// testing: add a marker
HfstState extra_state = result.add_state();
result.add_transition
- (result_state, HfstTransition <C>
+ (result_state, HfstTransition <C>
(extra_state, "@" + graph_transition.get_input_symbol() + "@", "@" + graph_transition.get_output_symbol() + "@", 0));
markers_added.insert("@" + graph_transition.get_input_symbol() + "@");
result.add_transition
- (extra_state /*result_state*/, HfstTransition <C>
- (retval, merger_transition.get_input_symbol(), merger_transition.get_output_symbol(), transition_weight));
+ (extra_state /*result_state*/, HfstTransition <C>
+ (retval, merger_transition.get_input_symbol(), merger_transition.get_output_symbol(), transition_weight));
// For each new state added, check if the corresponding states in \a graph1 and \a graph2
// are final. If they are, make the new state final with the sum of final weights.
if (was_new_state && (graph.is_final_state(graph_target) && merger.is_final_state(merger_target)))
@@ -4451,7 +4527,7 @@
{
find_matches_for_merge(graph, graph_transition.get_target_state(), merger, /*merger_transition.get_target_state()*/ merger_state, result, target, state_map, agenda, list_symbols, markers_added);
}
- // --- A transition in graph compared for all corresponding transitions in merger, compare next transition. ---
+ // --- A transition in graph compared for all corresponding transitions in merger, compare next transition. ---
}
// *** All transitions in state gone through. ***
return;
@@ -4473,7 +4549,7 @@
result.set_final_weight(0, final_weight);
}
- try
+ try
{
find_matches_for_merge(graph, 0, merger, 0, result, 0, state_map, agenda, list_symbols, markers_added);
}
@@ -4519,15 +4595,15 @@
friend class hfst::HarmonizeUnknownAndIdentitySymbols;
};
- /** @brief An HfstTransitionGraph with transitions of type
+ /** @brief An HfstTransitionGraph with transitions of type
HfstTropicalTransducerTransitionData and weight type float.
This is probably the most useful kind of HfstTransitionGraph. */
- typedef HfstTransitionGraph <HfstTropicalTransducerTransitionData>
+ typedef HfstTransitionGraph <HfstTropicalTransducerTransitionData>
HfstBasicTransducer;
/** @brief A specialization for faster conversion. */
- //typedef HfstTransitionGraph <HfstFastTransitionData>
+ //typedef HfstTransitionGraph <HfstFastTransitionData>
// HfstFastTransducer;
}
diff --git a/libhfst/src/implementations/HfstTropicalTransducerTransitionData.cc b/libhfst/src/implementations/HfstTropicalTransducerTransitionData.cc
index 9b5aa00..0d9ae83 100644
--- a/libhfst/src/implementations/HfstTropicalTransducerTransitionData.cc
+++ b/libhfst/src/implementations/HfstTropicalTransducerTransitionData.cc
@@ -1,30 +1,268 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "HfstTropicalTransducerTransitionData.h"
+#include <string>
+#include <map>
+#include <set>
+#include <cassert>
+#include <cstdio>
+#include <iostream>
+#include <vector>
+#include "../HfstExceptionDefs.h"
+
+#include "../hfstdll.h"
namespace hfst {
namespace implementations {
+ HfstTropicalTransducerTransitionData::SymbolType HfstTropicalTransducerTransitionData::get_epsilon()
+ {
+ return SymbolType("@_EPSILON_SYMBOL_@");
+ }
+
+ HfstTropicalTransducerTransitionData::SymbolType HfstTropicalTransducerTransitionData::get_unknown()
+ {
+ return SymbolType("@_UNKNOWN_SYMBOL_@");
+ }
+
+ HfstTropicalTransducerTransitionData::SymbolType HfstTropicalTransducerTransitionData::get_identity()
+ {
+ return SymbolType("@_IDENTITY_SYMBOL_@");
+ }
+
HfstTropicalTransducerTransitionData::Number2SymbolVector
HfstTropicalTransducerTransitionData::number2symbol_map;
- Number2SymbolVectorInitializer
+ Number2SymbolVectorInitializer
dummy1(HfstTropicalTransducerTransitionData::number2symbol_map);
-
- HfstTropicalTransducerTransitionData::Symbol2NumberMap
+ HfstTropicalTransducerTransitionData::Symbol2NumberMap
HfstTropicalTransducerTransitionData::symbol2number_map;
- Symbol2NumberMapInitializer
+ Symbol2NumberMapInitializer
dummy2(HfstTropicalTransducerTransitionData::symbol2number_map);
-
unsigned int HfstTropicalTransducerTransitionData::max_number=2;
- }
+ unsigned int HfstTropicalTransducerTransitionData::get_max_number() {
+ return max_number;
+ }
+
+ std::vector<unsigned int> HfstTropicalTransducerTransitionData::get_harmonization_vector
+ (const std::vector<HfstTropicalTransducerTransitionData::SymbolType> &symbols)
+ {
+ std::vector<unsigned int> harmv;
+ harmv.reserve(symbols.size());
+ harmv.resize(symbols.size(), 0);
+ for (unsigned int i=0; i<symbols.size(); i++)
+ {
+ if (symbols.at(i) != "")
+ harmv.at(i) = get_number(symbols.at(i));
+ }
+ return harmv;
+ }
+
+ std::vector<unsigned int> HfstTropicalTransducerTransitionData::get_reverse_harmonization_vector
+ (const std::map<HfstTropicalTransducerTransitionData::SymbolType, unsigned int> &symbols)
+ {
+ std::vector<unsigned int> harmv;
+ harmv.reserve(max_number+1);
+ harmv.resize(max_number+1, 0);
+ for (unsigned int i=0; i<harmv.size(); i++)
+ {
+ std::map<SymbolType, unsigned int>::const_iterator it
+ = symbols.find(get_symbol(i));
+ if (it != symbols.end())
+ harmv.at(i) = it->second;
+ }
+ return harmv;
+ }
+
+ const std::string & HfstTropicalTransducerTransitionData::get_symbol(unsigned int number)
+ {
+ if (number >= number2symbol_map.size()) {
+ std::string message("HfstTropicalTransducerTransitionData: "
+ "number ");
+ std::ostringstream oss;
+ oss << number;
+ message.append(oss.str());
+ message.append(" is not mapped to any symbol");
+ HFST_THROW_MESSAGE
+ (HfstFatalException, message);
+ }
+ return number2symbol_map[number];
+ }
+
+ unsigned int HfstTropicalTransducerTransitionData::get_number(const std::string &symbol)
+ {
+ if(symbol == "") { // FAIL
+ Symbol2NumberMap::iterator it = symbol2number_map.find(symbol);
+ if (it == symbol2number_map.end()) {
+ std::cerr << "ERROR: No number for the empty symbol\n"
+ << std::endl;
+ }
+ else {
+ std::cerr << "ERROR: The empty symbol corresdponds to number "
+ << it->second << std::endl;
+ }
+ assert(false);
+ }
+
+ Symbol2NumberMap::iterator it = symbol2number_map.find(symbol);
+ if (it == symbol2number_map.end())
+ {
+ max_number++;
+ symbol2number_map[symbol] = max_number;
+ number2symbol_map.push_back(symbol);
+ return max_number;
+ }
+ return it->second;
+ }
+
+
+ void HfstTropicalTransducerTransitionData::print_transition_data()
+ {
+ fprintf(stderr, "%i:%i %f\n",
+ input_number, output_number, weight);
+ }
+
+ HfstTropicalTransducerTransitionData::HfstTropicalTransducerTransitionData():
+ input_number(0), output_number(0), weight(0) {}
+
+ HfstTropicalTransducerTransitionData::HfstTropicalTransducerTransitionData
+ (const HfstTropicalTransducerTransitionData &data) {
+ input_number = data.input_number;
+ output_number = data.output_number;
+ weight = data.weight;
+ }
+
+ HfstTropicalTransducerTransitionData::HfstTropicalTransducerTransitionData(HfstTropicalTransducerTransitionData::SymbolType isymbol,
+ HfstTropicalTransducerTransitionData::SymbolType osymbol,
+ HfstTropicalTransducerTransitionData::WeightType weight) {
+ if (isymbol == "" || osymbol == "")
+ HFST_THROW_MESSAGE
+ (EmptyStringException,
+ "HfstTropicalTransducerTransitionData"
+ "(SymbolType, SymbolType, WeightType)");
+
+ input_number = get_number(isymbol);
+ output_number = get_number(osymbol);
+ this->weight = weight;
+ }
+
+ HfstTropicalTransducerTransitionData::HfstTropicalTransducerTransitionData
+ (unsigned int inumber,
+ unsigned int onumber,
+ WeightType weight) {
+ input_number = inumber;
+ output_number = onumber;
+ this->weight = weight;
+ }
+
+ const HfstTropicalTransducerTransitionData::SymbolType & HfstTropicalTransducerTransitionData::get_input_symbol() const {
+ return get_symbol(input_number);
+ }
+
+ void HfstTropicalTransducerTransitionData::set_input_symbol(const HfstTropicalTransducerTransitionData::SymbolType & symbol) {
+ input_number = get_number(symbol);
+ }
+
+ const HfstTropicalTransducerTransitionData::SymbolType & HfstTropicalTransducerTransitionData::get_output_symbol() const {
+ return get_symbol(output_number);
+ }
+
+ void HfstTropicalTransducerTransitionData::set_output_symbol(const HfstTropicalTransducerTransitionData::SymbolType & symbol) {
+ output_number = get_number(symbol);
+ }
+
+ unsigned int HfstTropicalTransducerTransitionData::get_input_number() const {
+ return input_number;
+ }
+
+ unsigned int HfstTropicalTransducerTransitionData::get_output_number() const {
+ return output_number;
+ }
+
+ HfstTropicalTransducerTransitionData::WeightType HfstTropicalTransducerTransitionData::get_weight() const {
+ return weight;
+ }
+
+ void HfstTropicalTransducerTransitionData::set_weight(WeightType w) {
+ weight = w;
+ }
+
+
+ bool HfstTropicalTransducerTransitionData::is_epsilon(const SymbolType &symbol) {
+ return (symbol.compare("@_EPSILON_SYMBOL_@") == 0);
+ }
+ bool HfstTropicalTransducerTransitionData::is_unknown(const SymbolType &symbol) {
+ return (symbol.compare("@_UNKNOWN_SYMBOL_@") == 0);
+ }
+ bool HfstTropicalTransducerTransitionData::is_identity(const SymbolType &symbol) {
+ return (symbol.compare("@_IDENTITY_SYMBOL_@") == 0);
+ }
+ bool HfstTropicalTransducerTransitionData::is_valid_symbol(const SymbolType &symbol) {
+ if (symbol == "")
+ return false;
+ return true;
+ }
+
+ HfstTropicalTransducerTransitionData::SymbolType HfstTropicalTransducerTransitionData::get_marker(const SymbolTypeSet &sts) {
+ (void)sts;
+ return SymbolType("@_MARKER_SYMBOL_@");
+ }
+
+ bool HfstTropicalTransducerTransitionData::operator<(const HfstTropicalTransducerTransitionData &another)
+ const {
+ if (input_number < another.input_number )
+ return true;
+ if (input_number > another.input_number)
+ return false;
+ if (output_number < another.output_number)
+ return true;
+ if (output_number > another.output_number)
+ return false;
+ return (weight < another.weight);
+ }
+
+ bool HfstTropicalTransducerTransitionData::less_than_ignore_weight(const HfstTropicalTransducerTransitionData &another)
+ const {
+ if (input_number < another.input_number )
+ return true;
+ if (input_number > another.input_number)
+ return false;
+ if (output_number < another.output_number)
+ return true;
+ if (output_number > another.output_number)
+ return false;
+ return false;
+ }
+
+ void HfstTropicalTransducerTransitionData::operator=(const HfstTropicalTransducerTransitionData &another)
+ {
+ input_number = another.input_number;
+ output_number = another.output_number;
+ weight = another.weight;
+ }
+
+ Number2SymbolVectorInitializer::Number2SymbolVectorInitializer
+ (HfstTropicalTransducerTransitionData::Number2SymbolVector &vect) {
+ vect.push_back(std::string("@_EPSILON_SYMBOL_@"));
+ vect.push_back(std::string("@_UNKNOWN_SYMBOL_@"));
+ vect.push_back(std::string("@_IDENTITY_SYMBOL_@"));
+ }
+
+ Symbol2NumberMapInitializer::Symbol2NumberMapInitializer
+ (HfstTropicalTransducerTransitionData::Symbol2NumberMap &map) {
+ map["@_EPSILON_SYMBOL_@"] = 0;
+ map["@_UNKNOWN_SYMBOL_@"] = 1;
+ map["@_IDENTITY_SYMBOL_@"] = 2;
+ }
+
+ } // namespace implementations
-}
+} // namespace hfst
diff --git a/libhfst/src/implementations/HfstTropicalTransducerTransitionData.h b/libhfst/src/implementations/HfstTropicalTransducerTransitionData.h
index 10516e4..cd106f7 100644
--- a/libhfst/src/implementations/HfstTropicalTransducerTransitionData.h
+++ b/libhfst/src/implementations/HfstTropicalTransducerTransitionData.h
@@ -1,18 +1,23 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
+#ifndef _HFST_TROPICAL_TRANSDUCER_TRANSITION_DATA_H_
+#define _HFST_TROPICAL_TRANSDUCER_TRANSITION_DATA_H_
+
+
#include <string>
#include <map>
#include <set>
#include <cassert>
#include <cstdio>
-#include <iostream>
+#include <iosfwd>
#include <vector>
#include "../HfstExceptionDefs.h"
@@ -28,15 +33,15 @@ namespace hfst {
}
};
- /** @brief One implementation of template class C in
- HfstTransition.
+ /** @brief One implementation of template class C in
+ HfstTransition.
- A HfstTropicalTransducerTransitionData has an input symbol and an
- output symbol of type SymbolType (string) and a weight of type
+ A HfstTropicalTransducerTransitionData has an input symbol and an
+ output symbol of type SymbolType (string) and a weight of type
WeightType (float).
- \internal Actually a HfstTropicalTransducerTransitionData has an
- input and an output number of type unsigned int, but this
+ \internal Actually a HfstTropicalTransducerTransitionData has an
+ input and an output number of type unsigned int, but this
implementation is hidden from the user.
The class has two static maps and functions that take care of conversion
between strings and internal numbers.
@@ -51,28 +56,17 @@ namespace hfst {
/** @brief A set of symbols. */
typedef std::set<SymbolType> SymbolTypeSet;
- typedef std::vector<SymbolType>
+ typedef std::vector<SymbolType>
Number2SymbolVector;
- typedef std::map<SymbolType, unsigned int, string_comparison>
+ typedef std::map<SymbolType, unsigned int, string_comparison>
Symbol2NumberMap;
- HFSTDLL static SymbolType get_epsilon()
- {
- return SymbolType("@_EPSILON_SYMBOL_@");
- }
-
- HFSTDLL static SymbolType get_unknown()
- {
- return SymbolType("@_UNKNOWN_SYMBOL_@");
- }
-
- HFSTDLL static SymbolType get_identity()
- {
- return SymbolType("@_IDENTITY_SYMBOL_@");
- }
+ HFSTDLL static SymbolType get_epsilon();
+ HFSTDLL static SymbolType get_unknown();
+ HFSTDLL static SymbolType get_identity();
public: /* FIXME: Should be private. */
- /* Maps that contain information of the mappings between strings
+ /* Maps that contain information of the mappings between strings
and numbers */
HFSTDLL static Number2SymbolVector number2symbol_map;
HFSTDLL static Symbol2NumberMap symbol2number_map;
@@ -80,18 +74,16 @@ namespace hfst {
HFSTDLL static unsigned int max_number;
/* Get the biggest number used to represent a symbol. */
- HFSTDLL static unsigned int get_max_number() {
- return max_number;
- }
+ HFSTDLL static unsigned int get_max_number();
- /*
+ /*
Get a vector that defines how numbers of a transducer must
- be changed, i.e. harmonized, so that it follows the same
+ be changed, i.e. harmonized, so that it follows the same
number-to-string encoding as all transducers that use the datatype
HfstTropicalTransducerTransitionData.
\a symbols defines how numbers are mapped to strings in the
- original transducer so that each index in \a symbols
+ original transducer so that each index in \a symbols
is the number that corresponds to the string at that index.
An empty string at an index means that the index is not
used in the original transducer.
@@ -102,78 +94,17 @@ namespace hfst {
transducer, the result will contain a zero at that index.
*/
static std::vector<unsigned int> get_harmonization_vector
- (const std::vector<SymbolType> &symbols)
- {
- std::vector<unsigned int> harmv;
- harmv.reserve(symbols.size());
- harmv.resize(symbols.size(), 0);
- for (unsigned int i=0; i<symbols.size(); i++)
- {
- if (symbols.at(i) != "")
- harmv.at(i) = get_number(symbols.at(i));
- }
- return harmv;
- }
+ (const std::vector<SymbolType> &symbols);
static std::vector<unsigned int> get_reverse_harmonization_vector
- (const std::map<SymbolType, unsigned int> &symbols)
- {
- std::vector<unsigned int> harmv;
- harmv.reserve(max_number+1);
- harmv.resize(max_number+1, 0);
- for (unsigned int i=0; i<harmv.size(); i++)
- {
- std::map<SymbolType, unsigned int>::const_iterator it
- = symbols.find(get_symbol(i));
- if (it != symbols.end())
- harmv.at(i) = it->second;
- }
- return harmv;
- }
+ (const std::map<SymbolType, unsigned int> &symbols);
protected:
/* Get the symbol that is mapped as \a number */
- static const std::string &get_symbol(unsigned int number)
- {
- if (number >= number2symbol_map.size()) {
- std::string message("HfstTropicalTransducerTransitionData: "
- "number ");
- std::ostringstream oss;
- oss << number;
- message.append(oss.str());
- message.append(" is not mapped to any symbol");
- HFST_THROW_MESSAGE
- (HfstFatalException, message);
- }
- return number2symbol_map[number];
- }
+ static const std::string &get_symbol(unsigned int number);
/* Get the number that is used to represent \a symbol */
- static unsigned int get_number(const std::string &symbol)
- {
- if(symbol == "") { // FAIL
- Symbol2NumberMap::iterator it = symbol2number_map.find(symbol);
- if (it == symbol2number_map.end()) {
- std::cerr << "ERROR: No number for the empty symbol\n"
- << std::endl;
- }
- else {
- std::cerr << "ERROR: The empty symbol corresdponds to number "
- << it->second << std::endl;
- }
- assert(false);
- }
-
- Symbol2NumberMap::iterator it = symbol2number_map.find(symbol);
- if (it == symbol2number_map.end())
- {
- max_number++;
- symbol2number_map[symbol] = max_number;
- number2symbol_map.push_back(symbol);
- return max_number;
- }
- return it->second;
- }
+ static unsigned int get_number(const std::string &symbol);
//private: TEST
public:
@@ -183,142 +114,69 @@ namespace hfst {
WeightType weight;
public:
- HFSTDLL void print_transition_data()
- {
- fprintf(stderr, "%i:%i %f\n",
- input_number, output_number, weight);
- }
+ HFSTDLL void print_transition_data();
public:
- /** @brief Create a HfstTropicalTransducerTransitionData with
+ /** @brief Create a HfstTropicalTransducerTransitionData with
epsilon input and output strings and weight zero. */
- HFSTDLL HfstTropicalTransducerTransitionData():
- input_number(0), output_number(0), weight(0) {}
+ HFSTDLL HfstTropicalTransducerTransitionData();
- /** @brief Create a deep copy of HfstTropicalTransducerTransitionData
+ /** @brief Create a deep copy of HfstTropicalTransducerTransitionData
\a data. */
HFSTDLL HfstTropicalTransducerTransitionData
- (const HfstTropicalTransducerTransitionData &data) {
- input_number = data.input_number;
- output_number = data.output_number;
- weight = data.weight;
- }
+ (const HfstTropicalTransducerTransitionData &data);
- /** @brief Create a HfstTropicalTransducerTransitionData with
- input symbol \a isymbol, output symbol \a osymbol
+ /** @brief Create a HfstTropicalTransducerTransitionData with
+ input symbol \a isymbol, output symbol \a osymbol
and weight \a weight. */
HFSTDLL HfstTropicalTransducerTransitionData(SymbolType isymbol,
- SymbolType osymbol,
- WeightType weight) {
- if (isymbol == "" || osymbol == "")
- HFST_THROW_MESSAGE
- (EmptyStringException,
- "HfstTropicalTransducerTransitionData"
- "(SymbolType, SymbolType, WeightType)");
-
- input_number = get_number(isymbol);
- output_number = get_number(osymbol);
- this->weight = weight;
- }
+ SymbolType osymbol,
+ WeightType weight);
HFSTDLL HfstTropicalTransducerTransitionData
(unsigned int inumber,
unsigned int onumber,
- WeightType weight) {
- input_number = inumber;
- output_number = onumber;
- this->weight = weight;
- }
+ WeightType weight);
/** @brief Get the input symbol. */
- HFSTDLL const SymbolType &get_input_symbol() const {
- return get_symbol(input_number);
- }
+ HFSTDLL const SymbolType &get_input_symbol() const;
+
+ /** @brief Set the input symbol. */
+ HFSTDLL void set_input_symbol(const SymbolType & symbol);
/** @brief Get the output symbol. */
- HFSTDLL const SymbolType &get_output_symbol() const {
- return get_symbol(output_number);
- }
+ HFSTDLL const SymbolType &get_output_symbol() const;
- HFSTDLL unsigned int get_input_number() const {
- return input_number;
- }
+ /** @brief Set the output symbol. */
+ HFSTDLL void set_output_symbol(const SymbolType & symbol);
- HFSTDLL unsigned int get_output_number() const {
- return output_number;
- }
+ HFSTDLL unsigned int get_input_number() const;
+
+ HFSTDLL unsigned int get_output_number() const;
/** @brief Get the weight. */
- HFSTDLL WeightType get_weight() const {
- return weight;
- }
+ HFSTDLL WeightType get_weight() const;
/** @brief Set the weight. */
- HFSTDLL void set_weight(WeightType w) {
- weight = w;
- }
-
+ HFSTDLL void set_weight(WeightType w);
/* Are these needed? */
- HFSTDLL static bool is_epsilon(const SymbolType &symbol) {
- return (symbol.compare("@_EPSILON_SYMBOL_@") == 0);
- }
- HFSTDLL static bool is_unknown(const SymbolType &symbol) {
- return (symbol.compare("@_UNKNOWN_SYMBOL_@") == 0);
- }
- HFSTDLL static bool is_identity(const SymbolType &symbol) {
- return (symbol.compare("@_IDENTITY_SYMBOL_@") == 0);
- }
- HFSTDLL static bool is_valid_symbol(const SymbolType &symbol) {
- if (symbol == "")
- return false;
- return true;
- }
-
- HFSTDLL static SymbolType get_marker(const SymbolTypeSet &sts) {
- (void)sts;
- return SymbolType("@_MARKER_SYMBOL_@");
- }
-
- /** @brief Whether this transition is less than transition
- \a another.
+ HFSTDLL static bool is_epsilon(const SymbolType &symbol);
+ HFSTDLL static bool is_unknown(const SymbolType &symbol);
+ HFSTDLL static bool is_identity(const SymbolType &symbol);
+ HFSTDLL static bool is_valid_symbol(const SymbolType &symbol);
+ HFSTDLL static SymbolType get_marker(const SymbolTypeSet &sts);
+
+ /** @brief Whether this transition is less than transition
+ \a another.
/internal is it too slow if string comparison is used instead?
*/
- HFSTDLL bool operator<(const HfstTropicalTransducerTransitionData &another)
- const {
- if (input_number < another.input_number )
- return true;
- if (input_number > another.input_number)
- return false;
- if (output_number < another.output_number)
- return true;
- if (output_number > another.output_number)
- return false;
- return (weight < another.weight);
- }
-
+ HFSTDLL bool operator<(const HfstTropicalTransducerTransitionData &another) const;
// same as operator< but weight is ignored
- HFSTDLL bool less_than_ignore_weight(const HfstTropicalTransducerTransitionData &another)
- const {
- if (input_number < another.input_number )
- return true;
- if (input_number > another.input_number)
- return false;
- if (output_number < another.output_number)
- return true;
- if (output_number > another.output_number)
- return false;
- return false;
- }
-
- HFSTDLL void operator=(const HfstTropicalTransducerTransitionData &another)
- {
- input_number = another.input_number;
- output_number = another.output_number;
- weight = another.weight;
- }
+ HFSTDLL bool less_than_ignore_weight(const HfstTropicalTransducerTransitionData &another) const;
+ HFSTDLL void operator=(const HfstTropicalTransducerTransitionData &another);
friend class Number2SymbolVectorInitializer;
friend class Symbol2NumberMapInitializer;
@@ -327,32 +185,26 @@ namespace hfst {
friend class ComposeIntersectLexicon;
friend class ComposeIntersectRule;
friend class ComposeIntersectRulePair;
- template <class C> friend class HfstTransitionGraph;
+ friend class HfstBasicTransducer;
};
- // Initialization of static members in class
+ // Initialization of static members in class
// HfstTropicalTransducerTransitionData..
class Number2SymbolVectorInitializer {
public:
HFSTDLL Number2SymbolVectorInitializer
- (HfstTropicalTransducerTransitionData::Number2SymbolVector &vect) {
- vect.push_back(std::string("@_EPSILON_SYMBOL_@"));
- vect.push_back(std::string("@_UNKNOWN_SYMBOL_@"));
- vect.push_back(std::string("@_IDENTITY_SYMBOL_@"));
- }
+ (HfstTropicalTransducerTransitionData::Number2SymbolVector &vect);
};
class Symbol2NumberMapInitializer {
public:
HFSTDLL Symbol2NumberMapInitializer
- (HfstTropicalTransducerTransitionData::Symbol2NumberMap &map) {
- map["@_EPSILON_SYMBOL_@"] = 0;
- map["@_UNKNOWN_SYMBOL_@"] = 1;
- map["@_IDENTITY_SYMBOL_@"] = 2;
- }
+ (HfstTropicalTransducerTransitionData::Symbol2NumberMap &map);
};
} // namespace implementations
} // namespace hfst
+
+#endif // #ifndef _HFST_TROPICAL_TRANSDUCER_TRANSITION_DATA_H_
diff --git a/libhfst/src/implementations/LogWeightTransducer.cc b/libhfst/src/implementations/LogWeightTransducer.cc
index 34b3590..d6cdda8 100644
--- a/libhfst/src/implementations/LogWeightTransducer.cc
+++ b/libhfst/src/implementations/LogWeightTransducer.cc
@@ -1,15 +1,22 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "LogWeightTransducer.h"
#include "HfstSymbolDefs.h"
+#ifdef _MSC_VER
+#include "back-ends/openfstwin/src/include/fst/fstlib.h"
+#else
+#include "back-ends/openfst/src/include/fst/fstlib.h"
+#endif // _MSC_VER
+
+
#ifndef MAIN_TEST
namespace hfst { namespace implementations
{
@@ -66,8 +73,8 @@ namespace hfst { namespace implementations
(LogFst *t, const std::string &symbol)
{
assert(t->InputSymbols() != NULL);
- fst::SymbolTable st(t->InputSymbols()->Name());
- for ( fst::SymbolTableIterator it
+ fst::SymbolTable st(t->InputSymbols()->Name());
+ for ( fst::SymbolTableIterator it
= fst::SymbolTableIterator(*(t->InputSymbols()));
! it.Done(); it.Next() ) {
if (it.Symbol() != symbol) {
@@ -81,7 +88,7 @@ namespace hfst { namespace implementations
{
assert(t->InputSymbols() != NULL);
StringSet s;
- for ( fst::SymbolTableIterator it
+ for ( fst::SymbolTableIterator it
= fst::SymbolTableIterator(*(t->InputSymbols()));
! it.Done(); it.Next() ) {
s.insert( std::string(it.Symbol()) );
@@ -90,7 +97,7 @@ namespace hfst { namespace implementations
}
unsigned int LogWeightTransducer::get_symbol_number
- (LogFst *t,
+ (LogFst *t,
const std::string &symbol)
{
assert(t->InputSymbols() != NULL);
@@ -100,18 +107,18 @@ namespace hfst { namespace implementations
return (unsigned int)i;
}
- /* Find the number-to-number mappings needed to be performed to t1
+ /* Find the number-to-number mappings needed to be performed to t1
so that it will follow the same symbol-to-number encoding as t2.
- @pre t2's symbol table must contain all symbols in t1's symbol table.
+ @pre t2's symbol table must contain all symbols in t1's symbol table.
*/
NumberNumberMap LogWeightTransducer::create_mapping(LogFst *t1, LogFst *t2)
{
NumberNumberMap km;
// find the number-to-number mappings for transducer t1
- for ( fst::SymbolTableIterator it
+ for ( fst::SymbolTableIterator it
= fst::SymbolTableIterator(*(t1->InputSymbols()));
- ! it.Done(); it.Next() ) {
- km [ (unsigned int)it.Value() ]
+ ! it.Done(); it.Next() ) {
+ km [ (unsigned int)it.Value() ]
= (unsigned int) t2->InputSymbols()->Find( it.Symbol() );
}
return km;
@@ -119,13 +126,13 @@ namespace hfst { namespace implementations
/* Recode the symbol numbers in this transducer as indicated in KeyMap km. */
void LogWeightTransducer::recode_symbol_numbers
- (LogFst *t, NumberNumberMap &km)
+ (LogFst *t, NumberNumberMap &km)
{
- for (fst::StateIterator<LogFst> siter(*t);
+ for (fst::StateIterator<LogFst> siter(*t);
! siter.Done(); siter.Next())
{
StateId s = siter.Value();
- for (fst::MutableArcIterator<LogFst> aiter(t,s);
+ for (fst::MutableArcIterator<LogFst> aiter(t,s);
!aiter.Done(); aiter.Next())
{
const LogArc &arc = aiter.Value();
@@ -142,7 +149,7 @@ namespace hfst { namespace implementations
LogFst * LogWeightTransducer::set_final_weights(LogFst * t, float weight)
{
- for (fst::StateIterator<LogFst> siter(*t);
+ for (fst::StateIterator<LogFst> siter(*t);
! siter.Done(); siter.Next())
{
StateId s = siter.Value();
@@ -167,13 +174,13 @@ namespace hfst { namespace implementations
LogFst * LogWeightTransducer::transform_weights
(LogFst * t,float (*func)(float f))
{
- for (fst::StateIterator<LogFst> siter(*t);
+ for (fst::StateIterator<LogFst> siter(*t);
! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if ( t->Final(s) != LogWeight::Zero() )
t->SetFinal( s, func(t->Final(s).Value()) );
- for (fst::MutableArcIterator<LogFst> aiter(t,s);
+ for (fst::MutableArcIterator<LogFst> aiter(t,s);
!aiter.Done(); aiter.Next())
{
const LogArc &arc = aiter.Value();
@@ -204,8 +211,8 @@ namespace hfst { namespace implementations
zero_print = initial_state;
}
- for (fst::StateIterator<LogFst> siter(*t);
- ! siter.Done(); siter.Next())
+ for (fst::StateIterator<LogFst> siter(*t);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s == initial_state) {
@@ -227,7 +234,7 @@ namespace hfst { namespace implementations
else
target = (int)arc.nextstate;
fprintf(ofile, "%i\t%i\t%s\t%s\t%f\n", origin, target,
- sym->Find(arc.ilabel).c_str(),
+ sym->Find(arc.ilabel).c_str(),
sym->Find(arc.olabel).c_str(),
arc.weight.Value());
}
@@ -237,7 +244,7 @@ namespace hfst { namespace implementations
}
}
- for (fst::StateIterator<LogFst> siter(*t);
+ for (fst::StateIterator<LogFst> siter(*t);
! siter.Done(); siter.Next())
{
StateId s = siter.Value();
@@ -249,7 +256,7 @@ namespace hfst { namespace implementations
origin = 0;
else
origin = (int)s;
- for (fst::ArcIterator<LogFst> aiter(*t,s);
+ for (fst::ArcIterator<LogFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const LogArc &arc = aiter.Value();
@@ -261,7 +268,7 @@ namespace hfst { namespace implementations
else
target = (int)arc.nextstate;
fprintf(ofile, "%i\t%i\t%s\t%s\t%f\n", origin, target,
- sym->Find(arc.ilabel).c_str(),
+ sym->Find(arc.ilabel).c_str(),
sym->Find(arc.olabel).c_str(),
arc.weight.Value());
}
@@ -285,8 +292,8 @@ namespace hfst { namespace implementations
zero_print = initial_state;
}
- for (fst::StateIterator<LogFst> siter(*t);
- ! siter.Done(); siter.Next())
+ for (fst::StateIterator<LogFst> siter(*t);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s == initial_state) {
@@ -317,7 +324,7 @@ namespace hfst { namespace implementations
}
}
- for (fst::StateIterator<LogFst> siter(*t);
+ for (fst::StateIterator<LogFst> siter(*t);
! siter.Done(); siter.Next())
{
StateId s = siter.Value();
@@ -329,7 +336,7 @@ namespace hfst { namespace implementations
origin = 0;
else
origin = (int)s;
- for (fst::ArcIterator<LogFst> aiter(*t,s);
+ for (fst::ArcIterator<LogFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const LogArc &arc = aiter.Value();
@@ -367,8 +374,8 @@ namespace hfst { namespace implementations
zero_print = initial_state;
}
- for (fst::StateIterator<LogFst> siter(*t);
- ! siter.Done(); siter.Next())
+ for (fst::StateIterator<LogFst> siter(*t);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s == initial_state) {
@@ -389,10 +396,10 @@ namespace hfst { namespace implementations
target = 0;
else
target = (int)arc.nextstate;
- os << origin << "\t"
- << target << "\t"
+ os << origin << "\t"
+ << target << "\t"
<< sym->Find(arc.ilabel).c_str() << "\t"
- << sym->Find(arc.olabel).c_str() << "\t"
+ << sym->Find(arc.olabel).c_str() << "\t"
<< arc.weight.Value() << "\n";
}
if (t->Final(s) != LogWeight::Zero())
@@ -402,7 +409,7 @@ namespace hfst { namespace implementations
}
}
- for (fst::StateIterator<LogFst> siter(*t);
+ for (fst::StateIterator<LogFst> siter(*t);
! siter.Done(); siter.Next())
{
StateId s = siter.Value();
@@ -414,7 +421,7 @@ namespace hfst { namespace implementations
origin = 0;
else
origin = (int)s;
- for (fst::ArcIterator<LogFst> aiter(*t,s);
+ for (fst::ArcIterator<LogFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const LogArc &arc = aiter.Value();
@@ -425,7 +432,7 @@ namespace hfst { namespace implementations
target = 0;
else
target = (int)arc.nextstate;
- os << origin << "\t"
+ os << origin << "\t"
<< target << "\t"
<< sym->Find(arc.ilabel).c_str() << "\t"
<< sym->Find(arc.olabel).c_str() << "\t"
@@ -452,8 +459,8 @@ namespace hfst { namespace implementations
zero_print = initial_state;
}
- for (fst::StateIterator<LogFst> siter(*t);
- ! siter.Done(); siter.Next())
+ for (fst::StateIterator<LogFst> siter(*t);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s == initial_state) {
@@ -474,10 +481,10 @@ namespace hfst { namespace implementations
target = 0;
else
target = (int)arc.nextstate;
- os << origin << "\t"
- << target << "\t"
+ os << origin << "\t"
+ << target << "\t"
<< "\\" << arc.ilabel << "\t"
- << "\\" << arc.olabel << "\t"
+ << "\\" << arc.olabel << "\t"
<< arc.weight.Value() << "\n";
}
if (t->Final(s) != LogWeight::Zero())
@@ -487,7 +494,7 @@ namespace hfst { namespace implementations
}
}
- for (fst::StateIterator<LogFst> siter(*t);
+ for (fst::StateIterator<LogFst> siter(*t);
! siter.Done(); siter.Next())
{
StateId s = siter.Value();
@@ -499,7 +506,7 @@ namespace hfst { namespace implementations
origin = 0;
else
origin = (int)s;
- for (fst::ArcIterator<LogFst> aiter(*t,s);
+ for (fst::ArcIterator<LogFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const LogArc &arc = aiter.Value();
@@ -510,7 +517,7 @@ namespace hfst { namespace implementations
target = 0;
else
target = (int)arc.nextstate;
- os << origin << "\t"
+ os << origin << "\t"
<< target << "\t"
<< "\\" << arc.ilabel << "\t"
<< "\\" << arc.olabel << "\t"
@@ -544,8 +551,8 @@ namespace hfst { namespace implementations
}
// FIXME?: atof and atoi
- /* Reads a description of a transducer in AT&T text format
- and returns a corresponding binary transducer.
+ /* Reads a description of a transducer in AT&T text format
+ and returns a corresponding binary transducer.
@note The initial state must be numbered as zero. */
LogFst * LogWeightTransducer::read_in_att_format(FILE * ifile)
{
@@ -560,12 +567,12 @@ namespace hfst { namespace implementations
t->SetStart(initial_state);
- while ( fgets(line, 255, ifile) != NULL )
+ while ( fgets(line, 255, ifile) != NULL )
{
if (*line == '-') // transducer separator
return t;
//printf("read line: %s", line);
- char a1 [100]; char a2 [100]; char a3 [100];
+ char a1 [100]; char a2 [100]; char a3 [100];
char a4 [100]; char a5 [100];
int n = sscanf(line, "%s\t%s\t%s\t%s\t%s", a1, a2, a3, a4, a5);
//printf("number of arguments: (%i)\n", n);
@@ -588,16 +595,16 @@ namespace hfst { namespace implementations
{
int origin_number = atoi(a1);
int target_number = atoi(a2);
- StateId origin_state
+ StateId origin_state
= add_and_map_state(t, origin_number, state_map);
- StateId target_state
+ StateId target_state
= add_and_map_state(t, target_number, state_map);
int input_number = st.AddSymbol(std::string(a3));
int output_number = st.AddSymbol(std::string(a4));
- t->AddArc(origin_state,
- LogArc(input_number, output_number,
+ t->AddArc(origin_state,
+ LogArc(input_number, output_number,
weight, target_state));
}
@@ -617,10 +624,10 @@ namespace hfst { namespace implementations
return t;
}
- /*
- Create a copy of this transducer where all transitions of type
+ /*
+ Create a copy of this transducer where all transitions of type
"?:?", "?:x" and "x:?"
- are expanded according to the StringSymbolSet 'unknown' that lists
+ are expanded according to the StringSymbolSet 'unknown' that lists
all symbols previously
unknown to this transducer.
*/
@@ -630,12 +637,12 @@ namespace hfst { namespace implementations
LogFst * result = new LogFst();
- for (fst::StateIterator<LogFst> siter(*t);
+ for (fst::StateIterator<LogFst> siter(*t);
! siter.Done(); siter.Next())
result->AddState();
// go through all states in this
- for (fst::StateIterator<LogFst> siter(*t);
+ for (fst::StateIterator<LogFst> siter(*t);
! siter.Done(); siter.Next())
{
// create new state in result, if needed
@@ -656,7 +663,7 @@ namespace hfst { namespace implementations
{
const LogArc &arc = aiter.Value();
- // find the corresponding target state in result or,
+ // find the corresponding target state in result or,
// if not found, create a new state
StateId result_nextstate=arc.nextstate;
@@ -664,71 +671,71 @@ namespace hfst { namespace implementations
if (unknown_symbols_in_use) {
- const fst::SymbolTable *is = t->InputSymbols();
+ const fst::SymbolTable *is = t->InputSymbols();
if ( arc.ilabel == 1 && // cross-product "?:?"
arc.olabel == 1 )
{
- for (StringSet::iterator it1 = unknown.begin();
- it1 != unknown.end(); it1++)
+ for (StringSet::iterator it1 = unknown.begin();
+ it1 != unknown.end(); it1++)
{
int64 inumber = is->Find(*it1);
- for (StringSet::iterator it2 = unknown.begin();
- it2 != unknown.end(); it2++)
+ for (StringSet::iterator it2 = unknown.begin();
+ it2 != unknown.end(); it2++)
{
int64 onumber = is->Find(*it2);
if (inumber != onumber)
- result->AddArc(result_s,
- LogArc(inumber, onumber,
+ result->AddArc(result_s,
+ LogArc(inumber, onumber,
arc.weight, result_nextstate));
}
- result->AddArc(result_s,
- LogArc(inumber, 1, arc.weight,
+ result->AddArc(result_s,
+ LogArc(inumber, 1, arc.weight,
result_nextstate));
- result->AddArc(result_s,
- LogArc(1, inumber, arc.weight,
+ result->AddArc(result_s,
+ LogArc(1, inumber, arc.weight,
result_nextstate));
}
}
else if (arc.ilabel == 2 || // identity "?:?"
- arc.olabel == 2 )
+ arc.olabel == 2 )
{
- for (StringSet::iterator it = unknown.begin();
- it != unknown.end(); it++)
+ for (StringSet::iterator it = unknown.begin();
+ it != unknown.end(); it++)
{
int64 number = is->Find(*it);
- result->AddArc(result_s,
- LogArc(number, number, arc.weight,
+ result->AddArc(result_s,
+ LogArc(number, number, arc.weight,
result_nextstate));
}
}
else if (arc.ilabel == 1) // "?:x"
{
- for (StringSet::iterator it = unknown.begin();
- it != unknown.end(); it++)
+ for (StringSet::iterator it = unknown.begin();
+ it != unknown.end(); it++)
{
int64 number = is->Find(*it);
- result->AddArc(result_s,
+ result->AddArc(result_s,
LogArc(number, arc.olabel, arc.weight,
result_nextstate));
}
}
else if (arc.olabel == 1) // "x:?"
{
- for (StringSet::iterator it = unknown.begin();
- it != unknown.end(); it++)
+ for (StringSet::iterator it = unknown.begin();
+ it != unknown.end(); it++)
{
int64 number = is->Find(*it);
- result->AddArc(result_s,
- LogArc(arc.ilabel, number,
+ result->AddArc(result_s,
+ LogArc(arc.ilabel, number,
arc.weight, result_nextstate));
}
}
}
// the original transition is copied in all cases
- result->AddArc(result_s,
- LogArc(arc.ilabel, arc.olabel,
+ result->AddArc(result_s,
+ LogArc(arc.ilabel, arc.olabel,
arc.weight, result_nextstate));
}
@@ -741,7 +748,7 @@ namespace hfst { namespace implementations
unsigned int LogWeightTransducer::number_of_states(const LogFst *t)
{
unsigned int retval=0;
- for (fst::StateIterator<LogFst> siter(*t);
+ for (fst::StateIterator<LogFst> siter(*t);
! siter.Done(); siter.Next())
retval++;
return retval;
@@ -762,7 +769,7 @@ namespace hfst { namespace implementations
hfst::symbols::collect_unknown_sets(t1_symbols, unknown_t1,
t2_symbols, unknown_t2);
- // 2. Add new symbols from transducer t1 to the symbol table
+ // 2. Add new symbols from transducer t1 to the symbol table
// of transducer t2...
SymbolTable * st2 = t2->InputSymbols()->Copy();
@@ -805,7 +812,7 @@ namespace hfst { namespace implementations
clock_t endclock = clock();
- log_seconds_in_harmonize = log_seconds_in_harmonize +
+ log_seconds_in_harmonize = log_seconds_in_harmonize +
( (float)(endclock - startclock) / CLOCKS_PER_SEC);
return std::pair<LogFst*, LogFst*>(harmonized_t1, harmonized_t2);
@@ -847,7 +854,7 @@ namespace hfst { namespace implementations
if (filename == string())
{ return std::cin.bad(); }
else
- { return input_stream.bad(); }
+ { return input_stream.bad(); }
}
bool LogWeightInputStream::is_good(void) const
{
@@ -887,15 +894,15 @@ namespace hfst { namespace implementations
LogFst * LogWeightInputStream::read_transducer()
{
if (is_eof())
- { //throw StreamIsClosedException();
+ { //throw StreamIsClosedException();
HFST_THROW(StreamIsClosedException); }
LogFst * t;
FstHeader header;
- try
+ try
{
if (filename == string())
{
- header.Read(input_stream,"STDIN");
+ header.Read(input_stream,"STDIN");
t = static_cast<LogFst*>
(LogFst::Read(input_stream,
FstReadOptions("STDIN",
@@ -903,14 +910,14 @@ namespace hfst { namespace implementations
}
else
{
- header.Read(input_stream,filename);
+ header.Read(input_stream,filename);
t = static_cast<LogFst*>
(LogFst::Read(input_stream,
FstReadOptions(filename,
&header)));
}
if (t == NULL)
- { //throw TransducerHasWrongTypeException();
+ { //throw TransducerHasWrongTypeException();
HFST_THROW(TransducerHasWrongTypeException); }
}
//catch (TransducerHasWrongTypeException e)
@@ -926,7 +933,7 @@ namespace hfst { namespace implementations
{ throw e; }
}
-
+ /*
LogWeightStateIterator::LogWeightStateIterator(LogFst * t):
iterator(new StateIterator<LogFst>(*t))
{}
@@ -1002,7 +1009,7 @@ namespace hfst { namespace implementations
LogWeightTransition LogWeightTransitionIterator::value()
{
return LogWeightTransition(arc_iterator->Value(), this->t);
- }
+ }*/
fst::SymbolTable LogWeightTransducer::create_symbol_table(std::string name) {
@@ -1019,7 +1026,7 @@ namespace hfst { namespace implementations
}
LogFst * LogWeightTransducer::create_empty_transducer(void)
- {
+ {
LogFst * t = new LogFst;
initialize_symbol_tables(t);
StateId s = t->AddState();
@@ -1028,7 +1035,7 @@ namespace hfst { namespace implementations
}
LogFst * LogWeightTransducer::create_epsilon_transducer(void)
- {
+ {
LogFst * t = new LogFst;
initialize_symbol_tables(t);
StateId s = t->AddState();
@@ -1037,6 +1044,11 @@ namespace hfst { namespace implementations
return t;
}
+ void LogWeightTransducer::delete_transducer(LogFst * t)
+ {
+ delete t;
+ }
+
LogFst * LogWeightTransducer::define_transducer(unsigned int number)
{
LogFst * t = new LogFst;
@@ -1093,7 +1105,7 @@ namespace hfst { namespace implementations
return t;
}
- bool LogWeightTransducer::are_equivalent(LogFst *a, LogFst *b)
+ bool LogWeightTransducer::are_equivalent(LogFst *a, LogFst *b)
{
LogFst * mina = minimize(a);
LogFst * minb = minimize(b);
@@ -1121,7 +1133,7 @@ namespace hfst { namespace implementations
const LogArc &arc = aiter.Value();
if (arc.ilabel != arc.olabel)
return false;
- if (arc.ilabel == 1) // ?:?
+ if (arc.ilabel == 1) // ?:?
return false;
}
}
@@ -1196,7 +1208,7 @@ namespace hfst { namespace implementations
++it)
{
StateId s2 = t->AddState();
- for (StringPairSet::const_iterator it2 = (*it).begin();
+ for (StringPairSet::const_iterator it2 = (*it).begin();
it2 != (*it).end(); it2++ ) {
t->AddArc(s1,LogArc(st.AddSymbol(it2->first),
st.AddSymbol(it2->second),0,s2));
@@ -1263,7 +1275,7 @@ namespace hfst { namespace implementations
++it)
{
StateId s2 = t->AddState();
- for (NumberPairSet::const_iterator it2 = (*it).begin();
+ for (NumberPairSet::const_iterator it2 = (*it).begin();
it2 != (*it).end(); it2++ ) {
t->AddArc(s1,LogArc(it2->first,it2->second,0,s2));
}
@@ -1274,12 +1286,12 @@ namespace hfst { namespace implementations
}
- LogFst *
+ LogFst *
LogWeightTransducer::copy(LogFst * t)
{ return new LogFst(*t); }
- LogFst *
+ LogFst *
LogWeightTransducer::determinize(LogFst * t)
{
RmEpsilon<LogArc>(t);
@@ -1308,8 +1320,8 @@ namespace hfst { namespace implementations
void print_att_number(LogFst *t, FILE * ofile) {
fprintf(ofile, "initial state: %i\n", t->Start());
- for (fst::StateIterator<LogFst> siter(*t);
- ! siter.Done(); siter.Next())
+ for (fst::StateIterator<LogFst> siter(*t);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if ( t->Final(s) != LogWeight::Zero() )
@@ -1317,8 +1329,8 @@ namespace hfst { namespace implementations
for (fst::ArcIterator<LogFst> aiter(*t,s); !aiter.Done(); aiter.Next())
{
const LogArc &arc = aiter.Value();
- fprintf(ofile, "%i\t%i\t%i\t%i\t%f\n",
- s, arc.nextstate, arc.ilabel, arc.olabel,
+ fprintf(ofile, "%i\t%i\t%i\t%i\t%f\n",
+ s, arc.nextstate, arc.ilabel, arc.olabel,
arc.weight.Value());
}
}
@@ -1364,23 +1376,23 @@ namespace hfst { namespace implementations
/* For HfstMutableTransducer */
- StateId
+ StateId
LogWeightTransducer::add_state(LogFst *t)
- {
+ {
StateId s = t->AddState();
if (s == 0)
t->SetStart(s);
return s;
}
- void
+ void
LogWeightTransducer::set_final_weight(LogFst *t, StateId s, float w)
{
t->SetFinal(s, w);
return;
}
- void
+ void
LogWeightTransducer::add_transition
(LogFst *t, StateId source, std::string &isymbol, std::string &osymbol,
float w, StateId target)
@@ -1388,8 +1400,8 @@ namespace hfst { namespace implementations
SymbolTable *st = t->InputSymbols()->Copy();
/*if (t->InputSymbols() != t->OutputSymbols()) {
fprintf(stderr, "ERROR: LogWeightTransducer::add_transition:"
- " input and output symbols are not the same\n");
- throw ErrorException();
+ " input and output symbols are not the same\n");
+ throw ErrorException();
}*/
unsigned int ilabel = st->AddSymbol(isymbol);
unsigned int olabel = st->AddSymbol(osymbol);
@@ -1399,13 +1411,13 @@ namespace hfst { namespace implementations
return;
}
- float
+ float
LogWeightTransducer::get_final_weight(LogFst *t, StateId s)
{
return t->Final(s).Value();
}
- float
+ float
LogWeightTransducer::is_final(LogFst *t, StateId s)
{
return ( t->Final(s) != LogWeight::Zero() );
@@ -1417,23 +1429,23 @@ namespace hfst { namespace implementations
return t->Start();
}
- LogFst *
+ LogFst *
LogWeightTransducer::remove_epsilons(LogFst * t)
{ return new LogFst(RmEpsilonFst<LogArc>(*t)); }
- LogFst *
+ LogFst *
LogWeightTransducer::n_best(LogFst * t, unsigned int n)
- {
- LogFst * n_best_fst = new LogFst();
+ {
+ LogFst * n_best_fst = new LogFst();
fst::ShortestPath(*t,n_best_fst,(size_t)n);
return n_best_fst;
}
- LogFst *
+ LogFst *
LogWeightTransducer::repeat_star(LogFst * t)
{ return new LogFst(ClosureFst<LogArc>(*t,CLOSURE_STAR)); }
- LogFst *
+ LogFst *
LogWeightTransducer::repeat_plus(LogFst * t)
{ return new LogFst(ClosureFst<LogArc>(*t,CLOSURE_PLUS)); }
@@ -1467,7 +1479,7 @@ namespace hfst { namespace implementations
return repetition;
}
- LogFst *
+ LogFst *
LogWeightTransducer::optionalize(LogFst * t)
{
LogFst * eps = create_epsilon_transducer();
@@ -1477,7 +1489,7 @@ namespace hfst { namespace implementations
}
- LogFst *
+ LogFst *
LogWeightTransducer::invert(LogFst * t)
{
LogFst * inverse = copy(t);
@@ -1487,7 +1499,7 @@ namespace hfst { namespace implementations
}
/* Makes valgrind angry... */
- LogFst *
+ LogFst *
LogWeightTransducer::reverse(LogFst * t)
{
LogFst * reversed = new LogFst;
@@ -1498,13 +1510,13 @@ namespace hfst { namespace implementations
LogFst * LogWeightTransducer::extract_input_language
(LogFst * t)
- { LogFst * retval = new LogFst(ProjectFst<LogArc>(*t,PROJECT_INPUT));
+ { LogFst * retval = new LogFst(ProjectFst<LogArc>(*t,PROJECT_INPUT));
retval->SetInputSymbols(t->InputSymbols());
return retval; }
LogFst * LogWeightTransducer::extract_output_language
(LogFst * t)
- { LogFst * retval = new LogFst(ProjectFst<LogArc>(*t,PROJECT_OUTPUT));
+ { LogFst * retval = new LogFst(ProjectFst<LogArc>(*t,PROJECT_OUTPUT));
retval->SetInputSymbols(t->InputSymbols());
return retval; }
@@ -1527,8 +1539,8 @@ namespace hfst { namespace implementations
assert(st != NULL);
for (fst::StateIterator<LogFst> siter(*t); !siter.Done(); siter.Next()) {
StateId state_id = siter.Value();
- t->AddArc(state_id, fst::LogArc(st->AddSymbol(symbol_pair.first),
- st->AddSymbol(symbol_pair.second),
+ t->AddArc(state_id, fst::LogArc(st->AddSymbol(symbol_pair.first),
+ st->AddSymbol(symbol_pair.second),
0, state_id));
}
t->SetInputSymbols(st);
@@ -1536,11 +1548,11 @@ namespace hfst { namespace implementations
return t;
}
- /*static LogFst * insert_freely(LogFst * t, const NumberPair &number_pair)
+ /*static LogFst * insert_freely(LogFst * t, const NumberPair &number_pair)
{
for (fst::StateIterator<LogFst> siter(*t); !siter.Done(); siter.Next()) {
StateId state_id = siter.Value();
- t->AddArc(state_id, fst::LogArc(number_pair.first, number_pair.second,
+ t->AddArc(state_id, fst::LogArc(number_pair.first, number_pair.second,
0, state_id));
}
return t;
@@ -1548,17 +1560,17 @@ namespace hfst { namespace implementations
/*
LogFst * LogWeightTransducer::substitute
- (LogFst *t, void (*func)(std::string &isymbol, std::string &osymbol) )
+ (LogFst *t, void (*func)(std::string &isymbol, std::string &osymbol) )
{
LogFst * tc = t->Copy();
SymbolTable * st = tc->InputSymbols()->Copy();
assert(st != NULL);
- for (fst::StateIterator<LogFst> siter(*tc);
+ for (fst::StateIterator<LogFst> siter(*tc);
not siter.Done(); siter.Next())
{
StateId s = siter.Value();
- for (fst::MutableArcIterator<LogFst> aiter(tc,s);
+ for (fst::MutableArcIterator<LogFst> aiter(tc,s);
!aiter.Done(); aiter.Next())
{
const LogArc &arc = aiter.Value(); // current values
@@ -1577,7 +1589,7 @@ namespace hfst { namespace implementations
}
tc->SetInputSymbols(st);
delete st;
- return tc;
+ return tc;
}
*/
@@ -1598,16 +1610,16 @@ namespace hfst { namespace implementations
EncodeMapper<LogArc> encode_mapper(0x0001,ENCODE);
EncodeFst<LogArc> enc(*t,&encode_mapper);
- LogArc old_pair_code =
+ LogArc old_pair_code =
encode_mapper(LogArc(old_key_pair.first,old_key_pair.second,0,0));
LogArc new_pair_code =
encode_mapper(LogArc(new_key_pair.first,new_key_pair.second,0,0));
// First cast up, then cast down... For some reason dynamic_cast<LogFst*>
- // doesn't work although both EncodeFst<LogArc> and LogFst extend
- // Fst<LogArc>.
+ // doesn't work although both EncodeFst<LogArc> and LogFst extend
+ // Fst<LogArc>.
// reinterpret_cast worked, but that is apparently unsafe...
- LogFst * subst =
+ LogFst * subst =
substitute(static_cast<LogFst*>(static_cast<Fst<LogArc>*>(&enc)),
static_cast<unsigned int>(old_pair_code.ilabel),
static_cast<unsigned int>(new_pair_code.ilabel));
@@ -1617,9 +1629,9 @@ namespace hfst { namespace implementations
return new LogFst(dec);
}
- /* It is not certain whether the transition iterator goes through all
+ /* It is not certain whether the transition iterator goes through all
the transitions that are added
- during the substitution. In that case, this function should build
+ during the substitution. In that case, this function should build
a new transducer instead of
modifying the original one. */
LogFst * LogWeightTransducer::substitute
@@ -1630,21 +1642,21 @@ namespace hfst { namespace implementations
LogFst * tc = t->Copy();
fst::SymbolTable * st = tc->InputSymbols()->Copy();
assert(st != NULL);
- for (fst::StateIterator<LogFst> siter(*tc);
- ! siter.Done(); siter.Next())
+ for (fst::StateIterator<LogFst> siter(*tc);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
- for (fst::MutableArcIterator<LogFst> aiter(tc,s);
+ for (fst::MutableArcIterator<LogFst> aiter(tc,s);
!aiter.Done(); aiter.Next())
{
const LogArc &arc = aiter.Value();
- if ( strcmp( st->Find(arc.ilabel).c_str(),
+ if ( strcmp( st->Find(arc.ilabel).c_str(),
old_symbol_pair.first.c_str() ) == 0 &&
- strcmp( st->Find(arc.olabel).c_str(),
+ strcmp( st->Find(arc.olabel).c_str(),
old_symbol_pair.second.c_str() ) == 0 )
{
bool first_substitution=true;
- for (StringPairSet::iterator it = new_symbol_pair_set.begin();
+ for (StringPairSet::iterator it = new_symbol_pair_set.begin();
it != new_symbol_pair_set.end(); it++)
{
if (first_substitution) {
@@ -1653,12 +1665,12 @@ namespace hfst { namespace implementations
new_arc.olabel = st->AddSymbol(it->second);
new_arc.weight = arc.weight.Value();
new_arc.nextstate = arc.nextstate;
- aiter.SetValue(new_arc);
+ aiter.SetValue(new_arc);
first_substitution=false; }
else
- tc->AddArc(s, LogArc(st->AddSymbol(it->first),
- st->AddSymbol(it->second),
- arc.weight.Value(),
+ tc->AddArc(s, LogArc(st->AddSymbol(it->first),
+ st->AddSymbol(it->second),
+ arc.weight.Value(),
arc.nextstate));
}
}
@@ -1675,7 +1687,7 @@ namespace hfst { namespace implementations
{
assert(t->InputSymbols() != NULL);
SymbolTable * st = t->InputSymbols()->Copy();
- LogFst * retval = substitute(t, st->AddSymbol(old_symbol),
+ LogFst * retval = substitute(t, st->AddSymbol(old_symbol),
st->AddSymbol(new_symbol));
retval->SetInputSymbols(st);
delete st;
@@ -1719,18 +1731,18 @@ namespace hfst { namespace implementations
fst::LogArc arc = it.Value();
// find arcs that must be replaced
- if ( arc.ilabel == st->AddSymbol(old_symbol_pair.first) &&
- arc.olabel == st->AddSymbol(old_symbol_pair.second) )
+ if ( arc.ilabel == st->AddSymbol(old_symbol_pair.first) &&
+ arc.olabel == st->AddSymbol(old_symbol_pair.second) )
{
StateId destination_state = arc.nextstate;
StateId start_state = t->AddState();
- // change the label of the arc to epsilon and point the arc
+ // change the label of the arc to epsilon and point the arc
// to a new state
arc.ilabel = 0;
arc.olabel = 0;
- arc.nextstate = start_state;
+ arc.nextstate = start_state;
// weight remains the same
it.SetValue(arc);
@@ -1742,7 +1754,7 @@ namespace hfst { namespace implementations
// go through all states and arcs in replace transducer tr
- for (fst::StateIterator<LogFst> siter(*transducer);
+ for (fst::StateIterator<LogFst> siter(*transducer);
!siter.Done(); siter.Next()) {
StateId tr_state_id = siter.Value();
@@ -1754,28 +1766,28 @@ namespace hfst { namespace implementations
t->AddArc( tr_state_id + start_state,
fst::LogArc( 0,
0,
- /* final weight is copied
+ /* final weight is copied
to the epsilon transition */
- transducer->Final(tr_state_id),
+ transducer->Final(tr_state_id),
destination_state
)
- );
+ );
- for (fst::ArcIterator<LogFst> aiter(*transducer, tr_state_id);
+ for (fst::ArcIterator<LogFst> aiter(*transducer, tr_state_id);
!aiter.Done(); aiter.Next()) {
const fst::LogArc &tr_arc = aiter.Value();
- // adding arc from state 'tr_state_id+start_state'
+ // adding arc from state 'tr_state_id+start_state'
// to state 'tr_arc.nextstate'
// copy arcs from tr to t
- t->AddArc( tr_state_id + start_state,
- fst::LogArc( tr_arc.ilabel,
- tr_arc.olabel,
- tr_arc.weight, /* weight remains
- the same */
- tr_arc.nextstate + start_state
- )
+ t->AddArc( tr_state_id + start_state,
+ fst::LogArc( tr_arc.ilabel,
+ tr_arc.olabel,
+ tr_arc.weight, /* weight remains
+ the same */
+ tr_arc.nextstate + start_state
+ )
);
}
@@ -1803,18 +1815,18 @@ namespace hfst { namespace implementations
fst::LogArc arc = it.Value();
// find arcs that must be replaced
- if ( arc.ilabel == (int)old_number_pair.first &&
- arc.olabel == (int)old_number_pair.second )
+ if ( arc.ilabel == (int)old_number_pair.first &&
+ arc.olabel == (int)old_number_pair.second )
{
StateId destination_state = arc.nextstate;
StateId start_state = t->AddState();
- // change the label of the arc to epsilon and point
+ // change the label of the arc to epsilon and point
// the arc to a new state
arc.ilabel = 0;
arc.olabel = 0;
- arc.nextstate = start_state;
+ arc.nextstate = start_state;
// weight remains the same
it.SetValue(arc);
@@ -1826,7 +1838,7 @@ namespace hfst { namespace implementations
// go through all states and arcs in replace transducer tr
- for (fst::StateIterator<LogFst> siter(*transducer);
+ for (fst::StateIterator<LogFst> siter(*transducer);
!siter.Done(); siter.Next()) {
StateId tr_state_id = siter.Value();
@@ -1838,28 +1850,28 @@ namespace hfst { namespace implementations
t->AddArc( tr_state_id + start_state,
fst::LogArc( 0,
0,
- // final weight is copied
+ // final weight is copied
// to the epsilon transition
- transducer->Final(tr_state_id),
+ transducer->Final(tr_state_id),
destination_state
)
- );
+ );
- for (fst::ArcIterator<LogFst> aiter(*transducer, tr_state_id);
+ for (fst::ArcIterator<LogFst> aiter(*transducer, tr_state_id);
!aiter.Done(); aiter.Next()) {
const fst::LogArc &tr_arc = aiter.Value();
- // adding arc from state 'tr_state_id+start_state'
+ // adding arc from state 'tr_state_id+start_state'
// to state 'tr_arc.nextstate'
// copy arcs from tr to t
- t->AddArc( tr_state_id + start_state,
- fst::LogArc( tr_arc.ilabel,
- tr_arc.olabel,
- // weight remains the same
- tr_arc.weight,
- tr_arc.nextstate + start_state
- )
+ t->AddArc( tr_state_id + start_state,
+ fst::LogArc( tr_arc.ilabel,
+ tr_arc.olabel,
+ // weight remains the same
+ tr_arc.weight,
+ tr_arc.nextstate + start_state
+ )
);
}
@@ -1875,7 +1887,7 @@ namespace hfst { namespace implementations
LogFst * t2)
{
StringSet foo;
- // a copy of t2 is created so that its symbol table check sum is
+ // a copy of t2 is created so that its symbol table check sum is
// the same as t1's
// (else OpenFst complains about non-matching check sums... )
LogFst * t2_ = expand_arcs(t2, foo, false);
@@ -1919,8 +1931,8 @@ namespace hfst { namespace implementations
StateId s = t->Start();
- for (StringPairVector::const_iterator it = spv.begin();
- it != spv.end(); it++)
+ for (StringPairVector::const_iterator it = spv.begin();
+ it != spv.end(); it++)
{
unsigned int inumber = st->AddSymbol(it->first.c_str());
unsigned int onumber = st->AddSymbol(it->second.c_str());
@@ -1954,8 +1966,8 @@ namespace hfst { namespace implementations
{
StateId s = t->Start();
- for (NumberPairVector::const_iterator it = npv.begin();
- it != npv.end(); it++)
+ for (NumberPairVector::const_iterator it = npv.begin();
+ it != npv.end(); it++)
{
unsigned int inumber = it->first;
unsigned int onumber = it->second;
@@ -2052,7 +2064,7 @@ namespace hfst { namespace implementations
if (DEBUG) printf(" ..subtracted\n");
//DifferenceFst<LogArc> subtract(enc1,enc2);
- LogFst *result = new LogFst(subtract);
+ LogFst *result = new LogFst(subtract);
result->SetInputSymbols(t1->InputSymbols());
return result;
}
@@ -2060,7 +2072,7 @@ namespace hfst { namespace implementations
LogFst * LogWeightTransducer::set_weight(LogFst * t,float f)
{
LogFst * t_copy = new LogFst(*t);
- for (fst::StateIterator<LogFst> iter(*t);
+ for (fst::StateIterator<LogFst> iter(*t);
! iter.Done(); iter.Next())
{
if (t_copy->Final(iter.Value()) != fst::LogWeight::Zero())
@@ -2073,20 +2085,20 @@ namespace hfst { namespace implementations
// ----- TRIE FUNCTIONS BEGINS -----
int LogWeightTransducer::has_arc(LogFst &t,
- LogArc::StateId sourcestate,
- LogArc::Label ilabel,
+ LogArc::StateId sourcestate,
+ LogArc::Label ilabel,
LogArc::Label olabel)
{
for (ArcIterator<LogFst> aiter(t,sourcestate);
!aiter.Done();
aiter.Next())
{
- if ((aiter.Value().ilabel == ilabel) &&
+ if ((aiter.Value().ilabel == ilabel) &&
(aiter.Value().olabel == olabel))
{ return aiter.Position(); }
}
- return -1;
+ return -1;
}
void LogWeightTransducer::disjunct_as_tries(LogFst &t1,
@@ -2154,14 +2166,14 @@ namespace hfst { namespace implementations
static bool extract_paths
(LogFst * t, LogArc::StateId s,
- std::map<StateId,unsigned short> all_visitations,
+ std::map<StateId,unsigned short> all_visitations,
std::map<StateId, unsigned short> path_visitations,
- /*std::vector<char>& lbuffer, int lpos, std::vector<char>& ubuffer,
+ /*std::vector<char>& lbuffer, int lpos, std::vector<char>& ubuffer,
int upos,*/ float weight_sum,
hfst::ExtractStringsCb& callback, int cycles,
std::vector<hfst::FdState<int64> >* fd_state_stack,
bool filter_fd, /*bool include_spv,*/ StringPairVector &spv)
- {
+ {
if(cycles >= 0 && path_visitations[s] > cycles)
return true;
all_visitations[s]++;
@@ -2224,7 +2236,7 @@ namespace hfst { namespace implementations
bool added_fd_state = false;
if (fd_state_stack) {
- if(fd_state_stack->back().get_table().get_operation(arc.ilabel)
+ if(fd_state_stack->back().get_table().get_operation(arc.ilabel)
!= NULL) {
fd_state_stack->push_back(fd_state_stack->back());
if(fd_state_stack->back().apply_operation(arc.ilabel))
@@ -2236,11 +2248,11 @@ namespace hfst { namespace implementations
}
}
- /*
+ /*
int lp=lpos;
int up=upos;
- if (arc.ilabel != 0 &&
+ if (arc.ilabel != 0 &&
(!filter_fd || fd_state_stack->back().get_table().
get_operation(arc.ilabel)==NULL))
{
@@ -2250,7 +2262,7 @@ namespace hfst { namespace implementations
strcpy(&lbuffer[lpos], str.c_str());
lp += str.length();
}
- if (arc.olabel != 0 &&
+ if (arc.olabel != 0 &&
(!filter_fd || fd_state_stack->back().get_table().
get_operation(arc.olabel)==NULL))
{
@@ -2262,17 +2274,17 @@ namespace hfst { namespace implementations
}
*/
- /* Handle spv here. Special symbols (flags, epsilons) are
+ /* Handle spv here. Special symbols (flags, epsilons) are
always inserted. */
std::string istring("");
std::string ostring("");
- if (!filter_fd ||
+ if (!filter_fd ||
fd_state_stack->back().get_table().get_operation(arc.ilabel)==NULL)
istring = t->InputSymbols()->Find(arc.ilabel);
- if (!filter_fd ||
+ if (!filter_fd ||
fd_state_stack->back().get_table().get_operation(arc.olabel)==NULL)
ostring = t->InputSymbols()->Find(arc.olabel);
@@ -2330,7 +2342,7 @@ namespace hfst { namespace implementations
{
FdTable<int64>* table = new FdTable<int64>();
const fst::SymbolTable* symbols = t->InputSymbols();
- for(fst::SymbolTableIterator it=fst::SymbolTableIterator(*symbols);
+ for(fst::SymbolTableIterator it=fst::SymbolTableIterator(*symbols);
!it.Done(); it.Next())
{
if(FdOperation::is_diacritic(it.Symbol()))
@@ -2368,8 +2380,8 @@ namespace hfst { namespace implementations
output_stream.put(char(c));
}
- void LogWeightOutputStream::write_transducer(LogFst * transducer)
- {
+ void LogWeightOutputStream::write_transducer(LogFst * transducer)
+ {
if (!output_stream)
fprintf(stderr, "LogWeightOutputStream: ERROR: failbit set (1).\n");
/* When writing a transducer, both input and output symbol tables are
@@ -2378,7 +2390,7 @@ namespace hfst { namespace implementations
transducer->SetOutputSymbols(&output_st);
transducer->Write(output_stream,FstWriteOptions()); }
- void LogWeightOutputStream::close(void)
+ void LogWeightOutputStream::close(void)
{
if (filename != string())
{ o_stream.close(); }
diff --git a/libhfst/src/implementations/LogWeightTransducer.h b/libhfst/src/implementations/LogWeightTransducer.h
index 42b8e50..c607a33 100644
--- a/libhfst/src/implementations/LogWeightTransducer.h
+++ b/libhfst/src/implementations/LogWeightTransducer.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _LOG_WEIGHT_TRANSDUCER_H_
@@ -18,33 +18,64 @@
#include "../../../config.h"
#endif
-#ifdef _MSC_VER
- #include "back-ends/openfstwin/src/include/fst/fstlib.h"
-#else
- #include "back-ends/openfst/src/include/fst/fstlib.h"
-#endif // _MSC_VER
+//#ifdef _MSC_VER
+// #include "back-ends/openfstwin/src/include/fst/fstlib.h"
+//#else
+// #include "back-ends/openfst/src/include/fst/fstlib.h"
+//#endif // _MSC_VER
#include "HfstExtractStrings.h"
#include <cstdio>
#include <string>
#include <sstream>
-#include <iostream>
+#include <iosfwd>
+#include <fstream>
//#include "HfstAlphabet.h"
+#ifdef OPENFST_VERSION_1_5_4
+ #include "back-ends/openfst/src/include/fst/fst-decl.h"
+#else
+namespace fst
+{
+ template <class W> class TropicalWeightTpl;
+ typedef TropicalWeightTpl<float> TropicalWeight;
+
+ template <class W> class LogWeightTpl;
+ typedef LogWeightTpl<float> LogWeight;
+
+ template <class W> class ArcTpl;
+ typedef ArcTpl<TropicalWeight> StdArc;
+ typedef ArcTpl<LogWeight> LogArc;
+
+ template <class A> class VectorFst;
+ typedef VectorFst<StdArc> StdVectorFst;
+ typedef VectorFst<LogArc> LogFst;
+
+ class SymbolTable;
+}
+#endif
+
+#ifdef _MSC_VER
+typedef __int64 int64;
+#else
+typedef int64_t int64;
+#endif // _MSC_VER
+
/** @file LogWeightTransducer.h
\brief Declarations of functions and datatypes that form a bridge between
HFST API and OpenFst's transducers with logarithmic weights. */
-namespace hfst {
+namespace hfst {
namespace implementations
{
using namespace fst;
;
- typedef LogArc::StateId StateId;
+ //typedef LogArc::StateId StateId;
+ typedef unsigned int StateId;
typedef VectorFst<LogArc> LogFst;
typedef std::vector<LogArc> LogArcVector;
- struct LogArcLessThan {
+ struct LogArcLessThan {
bool operator() (const LogArc &arc1,const LogArc &arc2) const; };
using std::ostream;
@@ -53,12 +84,12 @@ namespace implementations
void openfst_log_set_hopcroft(bool value);
- class LogWeightInputStream
+ class LogWeightInputStream
{
private:
std::string filename;
- ifstream i_stream;
- istream &input_stream;
+ std::ifstream i_stream;
+ std::istream &input_stream;
void skip_identifier_version_3_0(void);
void skip_hfst_header(void);
public:
@@ -78,29 +109,29 @@ namespace implementations
void stream_unget(char c);
static bool is_fst(FILE * f);
- static bool is_fst(istream &s);
+ static bool is_fst(std::istream &s);
};
- class LogWeightOutputStream
+ class LogWeightOutputStream
{
private:
std::string filename;
- ofstream o_stream;
- ostream &output_stream;
+ std::ofstream o_stream;
+ std::ostream &output_stream;
//void write_3_0_library_header(std::ostream &out);
public:
- LogWeightOutputStream(void);
+ LogWeightOutputStream(void);
LogWeightOutputStream(const std::string &filename);
void close(void);
void write(const char &c);
void write_transducer(LogFst * transducer);
};
- class LogWeightTransitionIterator;
+ /* class LogWeightTransitionIterator;
typedef StateId LogWeightState;
- class LogWeightStateIterator
+ class LogWeightStateIterator
{
protected:
StateIterator<LogFst> * iterator;
@@ -140,13 +171,14 @@ namespace implementations
bool done(void);
LogWeightTransition value(void);
};
-
+ */
class LogWeightTransducer
{
public:
static LogFst * create_empty_transducer(void);
static LogFst * create_epsilon_transducer(void);
+ void delete_transducer(LogFst * t);
// string versions
static LogFst * define_transducer(const std::string &symbol);
@@ -222,7 +254,7 @@ namespace implementations
static bool are_equivalent(LogFst *one, LogFst *another);
static bool is_cyclic(LogFst * t);
- static bool is_automaton(LogFst * t);
+ static bool is_automaton(LogFst * t);
static FdTable<int64>* get_flag_diacritics(LogFst * t);
@@ -255,7 +287,7 @@ namespace implementations
static void remove_from_alphabet
(LogFst *t, const std::string &symbol);
static StringSet get_alphabet(LogFst *t);
- static unsigned int get_symbol_number(LogFst *t,
+ static unsigned int get_symbol_number(LogFst *t,
const std::string &symbol);
static NumberNumberMap create_mapping(LogFst * t1, LogFst * t2);
@@ -271,9 +303,9 @@ namespace implementations
private:
static fst::SymbolTable create_symbol_table(std::string name);
static void initialize_symbol_tables(LogFst *t);
- static void remove_symbol_table(LogFst *t);
+ static void remove_symbol_table(LogFst *t);
- /* Maps state numbers in AT&T text format to state ids used by
+ /* Maps state numbers in AT&T text format to state ids used by
OpenFst transducers. */
typedef std::map<int, StateId> StateMap;
@@ -281,9 +313,9 @@ namespace implementations
(LogFst *t, int state_number, StateMap &state_map);
static int has_arc(LogFst &t,
- LogArc::StateId sourcestate,
- LogArc::Label ilabel,
- LogArc::Label olabel);
+ /*LogArc::StateId*/ int sourcestate,
+ /*LogArc::Label*/ int ilabel,
+ /*LogArc::Label*/ int olabel);
static void disjunct_as_tries(LogFst &t1,
StateId t1_state,
const LogFst * t2,
diff --git a/libhfst/src/implementations/Makefile.am b/libhfst/src/implementations/Makefile.am
index 6876664..34ce58c 100644
--- a/libhfst/src/implementations/Makefile.am
+++ b/libhfst/src/implementations/Makefile.am
@@ -1,21 +1,21 @@
## Process this file with automake to produce Makefile.in
-# Copyright (c) 2016 University of Helsinki
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 3 of the License, or (at your option) any later version.
-# See the file COPYING included with this distribution for more
-# information.
+# Copyright (c) 2016 University of Helsinki
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+# See the file COPYING included with this distribution for more
+# information.
AUTOMAKE_OPTIONS=subdir-objects
noinst_LTLIBRARIES=libhfstimplementations.la
# HFST bridge specific stuff
-IMPLEMENTATION_SRCS=HfstTransitionGraph.cc \
- ConvertTransducerFormat.cc \
+IMPLEMENTATION_SRCS=ConvertTransducerFormat.cc \
HfstTropicalTransducerTransitionData.cc \
+ HfstBasicTransition.cc HfstBasicTransducer.cc \
ConvertSfstTransducer.cc ConvertTropicalWeightTransducer.cc \
ConvertLogWeightTransducer.cc ConvertFomaTransducer.cc \
ConvertOlTransducer.cc ConvertXfsmTransducer.cc \
@@ -90,6 +90,7 @@ implinclude_HEADERS = \
SfstTransducer.h TropicalWeightTransducer.h \
XfsmTransducer.h \
HfstOlTransducer.h HfstTransitionGraph.h HfstTransition.h \
+ HfstBasicTransition.h HfstBasicTransducer.h\
HfstTropicalTransducerTransitionData.h \
compose_intersect/ComposeIntersectRulePair.h \
compose_intersect/ComposeIntersectLexicon.h \
@@ -136,7 +137,7 @@ if WANT_XFSM
XFSM_TSTS=XfsmTransducer
endif
-LIBHFST_TSTS=HfstTransitionGraph ConvertTransducerFormat \
+LIBHFST_TSTS=HfstBasicTransducer ConvertTransducerFormat \
ConvertSfstTransducer ConvertTropicalWeightTransducer \
ConvertLogWeightTransducer ConvertFomaTransducer \
ConvertXfsmTransducer ConvertOlTransducer \
@@ -150,68 +151,68 @@ LIBHFST_TSTS=HfstTransitionGraph ConvertTransducerFormat \
check_PROGRAMS=$(FOMA_TSTS) $(HFSTOL_TSTS) $(OFST_TSTS) $(SFST_TSTS) $(LIBHFST_TSTS) $(XFSM_TSTS)
FomaTransducer_SOURCES=FomaTransducer.cc
-FomaTransducer_CXXFLAGS=-DMAIN_TEST
+FomaTransducer_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
FomaTransducer_LDADD=../libhfst.la
XfsmTransducer_SOURCES=XfsmTransducer.cc
-XfsmTransducer_CXXFLAGS=-DMAIN_TEST
+XfsmTransducer_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
XfsmTransducer_LDADD=../libhfst.la
HfstOlTransducer_SOURCES=HfstOlTransducer.cc
-HfstOlTransducer_CXXFLAGS=-DMAIN_TEST
+HfstOlTransducer_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
HfstOlTransducer_LDADD=../libhfst.la
LogWeightTransducer_SOURCES=LogWeightTransducer.cc
-LogWeightTransducer_CXXFLAGS=-DMAIN_TEST
+LogWeightTransducer_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
LogWeightTransducer_LDADD=../libhfst.la
TropicalWeightTransducer_SOURCES=TropicalWeightTransducer.cc
-TropicalWeightTransducer_CXXFLAGS=-DMAIN_TEST
+TropicalWeightTransducer_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
TropicalWeightTransducer_LDADD=../libhfst.la
SfstTransducer_SOURCES=SfstTransducer.cc
-SfstTransducer_CXXFLAGS=-DMAIN_TEST
+SfstTransducer_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
SfstTransducer_LDADD=../libhfst.la
-HfstTransitionGraph_SOURCES=HfstTransitionGraph.cc
-HfstTransitionGraph_CXXFLAGS=-DMAIN_TEST
-HfstTransitionGraph_LDADD=../libhfst.la
+HfstBasicTransducer_SOURCES=HfstBasicTransducer.cc
+HfstBasicTransducer_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
+HfstBasicTransducer_LDADD=../libhfst.la
ConvertTransducerFormat_SOURCES=ConvertTransducerFormat.cc
-ConvertTransducerFormat_CXXFLAGS=-DMAIN_TEST
+ConvertTransducerFormat_CXXFLAGS=-DMAIN_TEST -Wno-deprecated -Wno-deprecated
ConvertTransducerFormat_LDADD=../libhfst.la
ConvertSfstTransducer_SOURCES=ConvertSfstTransducer.cc
-ConvertSfstTransducer_CXXFLAGS=-DMAIN_TEST
+ConvertSfstTransducer_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
ConvertSfstTransducer_LDADD=../libhfst.la
ConvertTropicalWeightTransducer_SOURCES=ConvertTropicalWeightTransducer.cc
-ConvertTropicalWeightTransducer_CXXFLAGS=-DMAIN_TEST
+ConvertTropicalWeightTransducer_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
ConvertTropicalWeightTransducer_LDADD=../libhfst.la
ConvertLogWeightTransducer_SOURCES=ConvertLogWeightTransducer.cc
-ConvertLogWeightTransducer_CXXFLAGS=-DMAIN_TEST
+ConvertLogWeightTransducer_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
ConvertLogWeightTransducer_LDADD=../libhfst.la
ConvertFomaTransducer_SOURCES=ConvertFomaTransducer.cc
-ConvertFomaTransducer_CXXFLAGS=-DMAIN_TEST
+ConvertFomaTransducer_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
ConvertFomaTransducer_LDADD=../libhfst.la
ConvertXfsmTransducer_SOURCES=ConvertXfsmTransducer.cc
-ConvertXfsmTransducer_CXXFLAGS=-DMAIN_TEST
+ConvertXfsmTransducer_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
ConvertXfsmTransducer_LDADD=../libhfst.la
ConvertOlTransducer_SOURCES=ConvertOlTransducer.cc
-ConvertOlTransducer_CXXFLAGS=-DMAIN_TEST
+ConvertOlTransducer_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
ConvertOlTransducer_LDADD=../libhfst.la
optimized_lookup_transducer_SOURCES=optimized-lookup/transducer.cc \
optimized-lookup/ospell.cc optimized-lookup/pmatch.cc optimized-lookup/find_epsilon_loops.cc
-optimized_lookup_transducer_CXXFLAGS=-DMAIN_TEST
+optimized_lookup_transducer_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
optimized_lookup_transducer_LDADD=../libhfst.la
optimized_lookup_convert_SOURCES=optimized-lookup/convert.cc
-optimized_lookup_convert_CXXFLAGS=-DMAIN_TEST
+optimized_lookup_convert_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
optimized_lookup_convert_LDADD=../libhfst.la
compose_intersect_ComposeIntersectRulePair_SOURCES=compose_intersect/ComposeIntersectRulePair.cc
-compose_intersect_ComposeIntersectRulePair_CXXFLAGS=-DMAIN_TEST
+compose_intersect_ComposeIntersectRulePair_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
compose_intersect_ComposeIntersectRulePair_LDADD=../libhfst.la
compose_intersect_ComposeIntersectLexicon_SOURCES=compose_intersect/ComposeIntersectLexicon.cc
-compose_intersect_ComposeIntersectLexicon_CXXFLAGS=-DMAIN_TEST
+compose_intersect_ComposeIntersectLexicon_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
compose_intersect_ComposeIntersectLexicon_LDADD=../libhfst.la
compose_intersect_ComposeIntersectRule_SOURCES=compose_intersect/ComposeIntersectRule.cc
-compose_intersect_ComposeIntersectRule_CXXFLAGS=-DMAIN_TEST
+compose_intersect_ComposeIntersectRule_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
compose_intersect_ComposeIntersectRule_LDADD=../libhfst.la
compose_intersect_ComposeIntersectFst_SOURCES=compose_intersect/ComposeIntersectFst.cc
-compose_intersect_ComposeIntersectFst_CXXFLAGS=-DMAIN_TEST
+compose_intersect_ComposeIntersectFst_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
compose_intersect_ComposeIntersectFst_LDADD=../libhfst.la
compose_intersect_ComposeIntersectUtilities_SOURCES=compose_intersect/ComposeIntersectUtilities.cc
-compose_intersect_ComposeIntersectUtilities_CXXFLAGS=-DMAIN_TEST
+compose_intersect_ComposeIntersectUtilities_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
compose_intersect_ComposeIntersectUtilities_LDADD=../libhfst.la
#HfstConstantTransducer_SOURCES=HfstConstantTransducer.cc
#HfstConstantTransducer_CXXFLAGS=-DMAIN_TEST
diff --git a/libhfst/src/implementations/MyTransducerLibraryTransducer.cc b/libhfst/src/implementations/MyTransducerLibraryTransducer.cc
index f4c2abe..61ea8d5 100644
--- a/libhfst/src/implementations/MyTransducerLibraryTransducer.cc
+++ b/libhfst/src/implementations/MyTransducerLibraryTransducer.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
/* @file MyTransducerLibraryTransducer.cc
@@ -14,20 +14,20 @@
#include "MyTransducerLibraryTransducer.h"
#ifndef DEBUG_MAIN
-namespace hfst {
+namespace hfst {
namespace implementations
{
;
/* Create and open a stream to standard in. */
MyTransducerLibraryInputStream::MyTransducerLibraryInputStream(void) {
- HFST_THROW(FunctionNotImplementedException); }
+ HFST_THROW(FunctionNotImplementedException); }
/* Create and open a stream to file filename. */
MyTransducerLibraryInputStream::MyTransducerLibraryInputStream
(const std::string &filename) {
(void)filename;
- HFST_THROW(FunctionNotImplementedException); }
+ HFST_THROW(FunctionNotImplementedException); }
/* Close the stream. If it points to standard in, nothing is done. */
void MyTransducerLibraryInputStream::close(void) {
@@ -63,7 +63,7 @@ namespace implementations
(void)n;
HFST_THROW(FunctionNotImplementedException); }
- /* Optional: if you want to extract implementation specific data from the
+ /* Optional: if you want to extract implementation specific data from the
header. */
bool MyTransducerLibraryInputStream::set_implementation_specific_header_data
(StringPairVector &data, unsigned int index) {
@@ -94,7 +94,7 @@ namespace implementations
/* Write a char to the stream. */
void MyTransducerLibraryOutputStream::write(const char &c) {
(void)c;
- HFST_THROW(FunctionNotImplementedException); }
+ HFST_THROW(FunctionNotImplementedException); }
/* Optional: if you want to store implementation specific data to the header.
*/
@@ -140,7 +140,7 @@ namespace implementations
(void)spv;
HFST_THROW(FunctionNotImplementedException); }
- /* Create a transducer that recognises the union of symbol pairs in sps
+ /* Create a transducer that recognises the union of symbol pairs in sps
If cyclic is true, any number of consecutive symbol pairs is recognised. */
MyFst * MyTransducerLibraryTransducer::define_transducer
(const StringPairSet &sps, bool cyclic) {
@@ -175,26 +175,26 @@ namespace implementations
(void)t;
HFST_THROW(FunctionNotImplementedException); }
- /* Create a transducer that accepts any number of consecutive string pairs
+ /* Create a transducer that accepts any number of consecutive string pairs
accepted by transducer t. */
MyFst * MyTransducerLibraryTransducer::repeat_star(MyFst * t) {
(void)t;
HFST_THROW(FunctionNotImplementedException); }
- /* Create a transducer that accepts one or more consecutive string pairs
+ /* Create a transducer that accepts one or more consecutive string pairs
accepted by transducer t. */
MyFst * MyTransducerLibraryTransducer::repeat_plus(MyFst * t) {
(void)t;
HFST_THROW(FunctionNotImplementedException); }
- /* Create a transducer that accepts n consecutive string pairs accepted by
+ /* Create a transducer that accepts n consecutive string pairs accepted by
transducer t. */
MyFst * MyTransducerLibraryTransducer::repeat_n(MyFst * t,int n) {
(void)t;
(void)n;
HFST_THROW(FunctionNotImplementedException); }
- /* Create a transducer that accepts from zero to n consecutive string pairs
+ /* Create a transducer that accepts from zero to n consecutive string pairs
accepted by transducer t. */
MyFst * MyTransducerLibraryTransducer::repeat_le_n(MyFst * t,int n) {
(void)t;
@@ -207,31 +207,31 @@ namespace implementations
(void)t;
HFST_THROW(FunctionNotImplementedException); }
- /* Create a transducer that accepts string pair string2:string1 iff
+ /* Create a transducer that accepts string pair string2:string1 iff
transducer t accepts string pair string1:string2. */
MyFst * MyTransducerLibraryTransducer::invert(MyFst * t) {
(void)t;
HFST_THROW(FunctionNotImplementedException); }
- /* Create a transducer that accepts string pair 1gnirts:2gnirts iff
+ /* Create a transducer that accepts string pair 1gnirts:2gnirts iff
transducer t accepts string pair string1:string2. */
MyFst * MyTransducerLibraryTransducer::reverse(MyFst * t) {
(void)t;
HFST_THROW(FunctionNotImplementedException); }
- /* Create a transducer that accepts string pair string1:string1 iff
+ /* Create a transducer that accepts string pair string1:string1 iff
transducer t accepts string pair string1:string2. */
MyFst * MyTransducerLibraryTransducer::extract_input_language(MyFst * t) {
(void)t;
HFST_THROW(FunctionNotImplementedException); }
- /* Create a transducer that accepts string pair string2:string2 iff
+ /* Create a transducer that accepts string pair string2:string2 iff
transducer t accepts string pair string1:string2. */
MyFst * MyTransducerLibraryTransducer::extract_output_language(MyFst * t) {
(void)t;
HFST_THROW(FunctionNotImplementedException); }
- /* A vector of transducers that each accept one string pair accepted by
+ /* A vector of transducers that each accept one string pair accepted by
transducer t. t cannot be cyclic. */
std::vector<MyFst*> MyTransducerLibraryTransducer::extract_paths(MyFst *t) {
(void)t;
@@ -239,17 +239,17 @@ namespace implementations
/* TODO: document */
void MyTransducerLibraryTransducer::extract_paths
- (MyFst * t, hfst::ExtractStringsCb& callback,
+ (MyFst * t, hfst::ExtractStringsCb& callback,
int cycles, FdTable<unsigned int>* fd, bool filter_fd) {
(void)t;
(void)callback;
(void)cycles;
(void)fd;
(void)filter_fd;
- HFST_THROW(FunctionNotImplementedException); }
+ HFST_THROW(FunctionNotImplementedException); }
- /* Create a transducer that accepts string pair of
- [ A:B* s A:B* t A:B* r A:B* i A:B* n A:B* g A:B* 1:2 A:B* ]
+ /* Create a transducer that accepts string pair of
+ [ A:B* s A:B* t A:B* r A:B* i A:B* n A:B* g A:B* 1:2 A:B* ]
(where A and B are input and output symbol of symbol_pair) iff transducer
t accepts string pair string1:string2. */
MyFst * MyTransducerLibraryTransducer::insert_freely
@@ -258,7 +258,7 @@ namespace implementations
(void)symbol_pair;
HFST_THROW(FunctionNotImplementedException); }
- /* Create a transducer equivalent to transducer t but where all symbols
+ /* Create a transducer equivalent to transducer t but where all symbols
old_symbol are substituted with new_symbol. */
MyFst * MyTransducerLibraryTransducer::substitute
(MyFst * t, String old_symbol, String new_symbol) {
@@ -267,7 +267,7 @@ namespace implementations
(void)new_symbol;
HFST_THROW(FunctionNotImplementedException); }
- /* Create a transducer equivalent to transducer t but where all symbol pairs
+ /* Create a transducer equivalent to transducer t but where all symbol pairs
symbol_pair are substituted with a copy of transducer tr. */
MyFst * MyTransducerLibraryTransducer::substitute
(MyFst *t, const StringPair &symbol_pair, MyFst *tr) {
@@ -276,15 +276,15 @@ namespace implementations
(void)tr;
HFST_THROW(FunctionNotImplementedException); }
- /* Create a transducer that accepts string pair string1:string3 iff t1
- accepts string pair string1:string2 and t2 accepts string pair
+ /* Create a transducer that accepts string pair string1:string3 iff t1
+ accepts string pair string1:string2 and t2 accepts string pair
string2:string3, where string2 is any string. */
MyFst * MyTransducerLibraryTransducer::compose(MyFst * t1, MyFst * t2) {
(void)t1;
(void)t2;
HFST_THROW(FunctionNotImplementedException); }
- /* Create a transducer that accepts a concatenation of any string pair
+ /* Create a transducer that accepts a concatenation of any string pair
accepted by t1 and any string pair accepted by t2. */
MyFst * MyTransducerLibraryTransducer::concatenate(MyFst * t1, MyFst * t2) {
(void)t1;
@@ -297,14 +297,14 @@ namespace implementations
(void)t2;
HFST_THROW(FunctionNotImplementedException); }
- /* Create a transducer that accepts any string pair accepted by both t1 and
+ /* Create a transducer that accepts any string pair accepted by both t1 and
t2. */
MyFst * MyTransducerLibraryTransducer::intersect(MyFst * t1, MyFst * t2) {
(void)t1;
(void)t2;
HFST_THROW(FunctionNotImplementedException); }
- /* Create a transducer that accepts any string pair accepted by t1 but not
+ /* Create a transducer that accepts any string pair accepted by t1 but not
t2. */
MyFst * MyTransducerLibraryTransducer::subtract(MyFst * t1, MyFst * t2) {
(void)t1;
@@ -328,41 +328,41 @@ namespace implementations
/* Whether transducer t is cyclic. */
bool MyTransducerLibraryTransducer::is_cyclic(MyFst * t) {
(void)t;
- HFST_THROW(FunctionNotImplementedException); }
+ HFST_THROW(FunctionNotImplementedException); }
- /* A table of transducer numbers that represent flag
+ /* A table of transducer numbers that represent flag
diacritics in transducer t. */
FdTable<unsigned int>* MyTransducerLibraryTransducer::get_flag_diacritics
(MyFst * t) {
(void)t;
- HFST_THROW(FunctionNotImplementedException); }
+ HFST_THROW(FunctionNotImplementedException); }
/* Remove symbol symbol from the alphabet of transducer t. */
MyFst * MyTransducerLibraryTransducer::remove_from_alphabet
(MyFst *t, const std::string &symbol) {
(void)t;
(void)symbol;
- HFST_THROW(FunctionNotImplementedException); }
+ HFST_THROW(FunctionNotImplementedException); }
/* Disjunct t with a path transducer spv. */
MyFst * MyTransducerLibraryTransducer::disjunct
(MyFst * t, const StringPairVector &spv) {
(void)t;
(void)spv;
- HFST_THROW(FunctionNotImplementedException); }
+ HFST_THROW(FunctionNotImplementedException); }
/* Get all symbol pairs that occur in transitions of transducer t. */
StringPairSet MyTransducerLibraryTransducer::get_symbol_pairs(MyFst *t) {
(void)t;
- HFST_THROW(FunctionNotImplementedException); }
+ HFST_THROW(FunctionNotImplementedException); }
/* The number of states in transducer t. */
unsigned int MyTransducerLibraryTransducer::number_of_states(MyFst *t) {
(void)t;
HFST_THROW(FunctionNotImplementedException); }
- /* Add the following number-to-symbol correspondencies to the alphabet of
- transducer t:
+ /* Add the following number-to-symbol correspondencies to the alphabet of
+ transducer t:
0 : "@_EPSILON_SYMBOL_@", i.e. internal_epsilon
1 : "@_UNKNOWN_SYMBOL_@", i.e. internal_unknown
2 : "@_IDENTITY_SYMBOL_@", i.e. internal_identity */
diff --git a/libhfst/src/implementations/MyTransducerLibraryTransducer.h b/libhfst/src/implementations/MyTransducerLibraryTransducer.h
index b894c78..6f22f57 100644
--- a/libhfst/src/implementations/MyTransducerLibraryTransducer.h
+++ b/libhfst/src/implementations/MyTransducerLibraryTransducer.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _MY_TRANSDUCER_LIBRARY_TRANSDUCER_H_
@@ -12,11 +12,11 @@
/** @file MyTransducerLibraryTransducer.h
\brief Declarations of functions and datatypes that form a bridge between
- the HFST API and your transducer library.
+ the HFST API and your transducer library.
This file lists the classes and functions that a new library must implement
- before added under the HFST API. Write implementations in file
- MyTransducerLibraryTransducer.cc
+ before added under the HFST API. Write implementations in file
+ MyTransducerLibraryTransducer.cc
*/
#include "HfstExceptionDefs.h"
@@ -33,10 +33,10 @@
#include <cstdio>
#include <string>
#include <sstream>
-#include <iostream>
+#include <iosfwd>
-namespace hfst {
+namespace hfst {
namespace implementations
{
@@ -45,15 +45,15 @@ namespace implementations
using namespace my_namespace;
/** @brief A skeleton class for reading a new type of binary transducers
- from a stream.
+ from a stream.
If your transducer read function uses C FILEs, see class SfstInputStream
or FomaInputStream for an example how to implement the functions
in class MyTransducerLibraryInputStream. If your transducer read function
- uses C++ streams, see class TropicalWeightInputStream or
+ uses C++ streams, see class TropicalWeightInputStream or
LogWeightInputStream.
*/
- class MyTransducerLibraryInputStream
+ class MyTransducerLibraryInputStream
{
private:
// you probably need
@@ -71,7 +71,7 @@ namespace implementations
MyTransducerLibraryInputStream(void);
/** @brief Create and open a stream to file filename. */
MyTransducerLibraryInputStream(const std::string &filename);
- /** @brief Close the stream. If it points to standard in, nothing
+ /** @brief Close the stream. If it points to standard in, nothing
is done. */
void close(void);
/** @brief Whether the stream is at end. */
@@ -90,7 +90,7 @@ namespace implementations
/** @brief Extract n chars and ignore them. */
void ignore(unsigned int n);
- /** @brief Optional: if you want to extract implementation specific
+ /** @brief Optional: if you want to extract implementation specific
data from the header. */
bool set_implementation_specific_header_data
(StringPairVector &data, unsigned int index);
@@ -105,11 +105,11 @@ namespace implementations
If your transducer write function uses C FILEs, see class SfstOutputStream
or FomaOutputStream for an example how to implement the functions
- in class MyTransducerLibraryOutputStream. If your transducer write
- function uses C++ streams, see class TropicalWeightOutputStream or
+ in class MyTransducerLibraryOutputStream. If your transducer write
+ function uses C++ streams, see class TropicalWeightOutputStream or
LogWeightOutputStream.
*/
- class MyTransducerLibraryOutputStream
+ class MyTransducerLibraryOutputStream
{
private:
// you probably need this
@@ -124,7 +124,7 @@ namespace implementations
public:
/** @brief Create and open a stream to standard out. */
- MyTransducerLibraryOutputStream(void);
+ MyTransducerLibraryOutputStream(void);
/** @brief Create and open a stream to file filename. */
MyTransducerLibraryOutputStream(const std::string &filename);
/** @brief Close the stream. If it points to standard out, nothing
@@ -133,7 +133,7 @@ namespace implementations
/** @brief Write a char to the stream. */
void write(const char &c);
- /** @brief Optional: if you want to store implementation specific data
+ /** @brief Optional: if you want to store implementation specific data
to the binary transducer header. */
void append_implementation_specific_header_data
(std::vector<char> &header, MyFst *t);
@@ -145,7 +145,7 @@ namespace implementations
/** @brief A skeleton library class that contains operations for a new
transducer type that is added under the HFST interface.
- A transducer library that is added under HFST must implement these
+ A transducer library that is added under HFST must implement these
operations. */
class MyTransducerLibraryTransducer
{
@@ -154,22 +154,22 @@ namespace implementations
static MyFst * create_empty_transducer(void);
/** @brief Create a transducer that recognises the empty string. */
static MyFst * create_epsilon_transducer(void);
- /** @brief Create a transducer that recognises the symbol pair
+ /** @brief Create a transducer that recognises the symbol pair
symbol:symbol */
static MyFst * define_transducer(const std::string &symbol);
- /** @brief Create a transducer that recognises the symbol pair
+ /** @brief Create a transducer that recognises the symbol pair
isymbol:osymbol */
static MyFst * define_transducer
(const std::string &isymbol, const std::string &osymbol);
- /** @brief Create a transducer that recognises the concatenation of
+ /** @brief Create a transducer that recognises the concatenation of
symbol pairs in spv */
static MyFst * define_transducer(const StringPairVector &spv);
- /** @brief Create a transducer that recognises the union of symbol pairs
+ /** @brief Create a transducer that recognises the union of symbol pairs
in sps. If cyclic is true, any number of consecutive symbol pairs is
recognised. */
static MyFst * define_transducer
(const StringPairSet &sps, bool cyclic=false);
- /** @brief Crate a transducer that recognises the concatenation of
+ /** @brief Crate a transducer that recognises the concatenation of
symbol pair unions in spsv. */
static MyFst * define_transducer(const std::vector<StringPairSet> &spsv);
@@ -184,36 +184,36 @@ namespace implementations
*/
static MyFst * remove_epsilons(MyFst * t);
- /** @brief Create a transducer that accepts any number of consecutive
+ /** @brief Create a transducer that accepts any number of consecutive
string pairs accepted by transducer t. */
static MyFst * repeat_star(MyFst * t);
- /** @brief Create a transducer that accepts one or more consecutive
+ /** @brief Create a transducer that accepts one or more consecutive
string pairs accepted by transducer t. */
static MyFst * repeat_plus(MyFst * t);
- /** @brief Create a transducer that accepts n consecutive string pairs
+ /** @brief Create a transducer that accepts n consecutive string pairs
accepted by transducer t. */
static MyFst * repeat_n(MyFst * t, unsigned int n);
- /** @brief Create a transducer that accepts from zero to n consecutive
+ /** @brief Create a transducer that accepts from zero to n consecutive
string pairs accepted by transducer t. */
static MyFst * repeat_le_n(MyFst * t, unsigned int n);
- /** @brief Create a transducer that accepts string pairs accepted by
+ /** @brief Create a transducer that accepts string pairs accepted by
transducer t or an empty string. */
static MyFst * optionalize(MyFst * t);
- /** @brief Create a transducer that accepts string pair string2:string1
+ /** @brief Create a transducer that accepts string pair string2:string1
iff transducer t accepts string pair string1:string2. */
static MyFst * invert(MyFst * t);
- /** @brief Create a transducer that accepts string pair 1gnirts:2gnirts
+ /** @brief Create a transducer that accepts string pair 1gnirts:2gnirts
iff transducer t accepts string pair string1:string2. */
static MyFst * reverse(MyFst * t);
- /** @brief Create a transducer that accepts string pair string1:string1
+ /** @brief Create a transducer that accepts string pair string1:string1
iff transducer t accepts string pair string1:string2. */
static MyFst * extract_input_language(MyFst * t);
- /** @brief Create a transducer that accepts string pair string2:string2
+ /** @brief Create a transducer that accepts string pair string2:string2
iff transducer t accepts string pair string1:string2. */
static MyFst * extract_output_language(MyFst * t);
- /** @brief A vector of transducers that each accept one string pair
+ /** @brief A vector of transducers that each accept one string pair
accepted by transducer t. t cannot be cyclic. */
static std::vector<MyFst*> extract_paths(MyFst *t);
/** @brief TODO: document */
@@ -221,53 +221,53 @@ namespace implementations
(MyFst * t, hfst::ExtractStringsCb& callback,
int cycles=-1, FdTable<unsigned int>* fd=NULL, bool filter_fd=false);
- /** @brief Create a transducer that accepts string pair of
- [ A:B* s A:B* t A:B* r A:B* i A:B* n A:B* g A:B* 1:2 A:B* ]
- (where A and B are input and output symbol of symbol_pair) iff
+ /** @brief Create a transducer that accepts string pair of
+ [ A:B* s A:B* t A:B* r A:B* i A:B* n A:B* g A:B* 1:2 A:B* ]
+ (where A and B are input and output symbol of symbol_pair) iff
transducer t accepts string pair string1:string2. */
static MyFst * insert_freely(MyFst *t , const StringPair &symbol_pair);
- /** @brief Create a transducer equivalent to transducer t but where all
+ /** @brief Create a transducer equivalent to transducer t but where all
symbols old_symbol are substituted with new_symbol. */
static MyFst * substitute
(MyFst * t, String old_symbol, String new_symbol);
- /** @brief Create a transducer equivalent to transducer t but where all
- symbol pairs symbol_pair are substituted with a copy of transducer
+ /** @brief Create a transducer equivalent to transducer t but where all
+ symbol pairs symbol_pair are substituted with a copy of transducer
tr. */
static MyFst * substitute
(MyFst *t, const StringPair &symbol_pair, MyFst *tr);
- /** @brief Create a transducer that accepts string pair string1:string3
+ /** @brief Create a transducer that accepts string pair string1:string3
iff t1 accepts string pair string1:string2 and t2 accepts string pair
string2:string3, where string2 is any string. */
static MyFst * compose(MyFst * t1, MyFst * t2);
/** @brief Create a transducer that accepts a concatenation of any string
pair accepted by t1 and any string pair accepted by t2. */
static MyFst * concatenate(MyFst * t1, MyFst * t2);
- /** @brief Create a transducer that accepts any string pair accepted by
+ /** @brief Create a transducer that accepts any string pair accepted by
t1 or t2. */
static MyFst * disjunct(MyFst * t1, MyFst * t2);
- /** @brief Create a transducer that accepts any string pair accepted by
+ /** @brief Create a transducer that accepts any string pair accepted by
both t1 and t2. */
static MyFst * intersect(MyFst * t1, MyFst * t2);
- /** @brief Create a transducer that accepts any string pair accepted by
+ /** @brief Create a transducer that accepts any string pair accepted by
t1 but not t2. */
static MyFst * subtract(MyFst * t1, MyFst * t2);
- /** @brief Return a harmonized copy of transducers t1 and t2.
+ /** @brief Return a harmonized copy of transducers t1 and t2.
First, all string symbols that are found in t2's alphabet but not in
- t1's alphabet are added to the alphabet of t1 to the next free
- positions (numbers). Then a copy of t2 is created where the
+ t1's alphabet are added to the alphabet of t1 to the next free
+ positions (numbers). Then a copy of t2 is created where the
string-to-number mappings are the same as in t1.
- Next all unknown and identity symbols of t1 and t2 are expanded
- according to the alphabets of the transducers, i.e. an unknown or
- identity symbol in t2 is expanded to a set of all symbols known to t1
+ Next all unknown and identity symbols of t1 and t2 are expanded
+ according to the alphabets of the transducers, i.e. an unknown or
+ identity symbol in t2 is expanded to a set of all symbols known to t1
but unknown to t2 and vice versa.
- Unknown and identity symbols are explained in more detail in (Ref.).
+ Unknown and identity symbols are explained in more detail in (Ref.).
- A false value of unknown_symbols_in_use may be used for optimization
+ A false value of unknown_symbols_in_use may be used for optimization
as no unknown or identity expanding is needed in that case. */
static std::pair<MyFst*, MyFst*> harmonize
(MyFst *t1, MyFst *t2, bool unknown_symbols_in_use=true);
@@ -277,7 +277,7 @@ namespace implementations
/** @brief Whether transducer t is cyclic. */
static bool is_cyclic(MyFst * t);
- /** @brief A table of numbers that represent flag
+ /** @brief A table of numbers that represent flag
diacritics in transducer t. */
static FdTable<unsigned int>* get_flag_diacritics(MyFst * t);
@@ -287,7 +287,7 @@ namespace implementations
/** @brief Disjunct t with a path transducer spv. */
static MyFst * disjunct(MyFst * t, const StringPairVector &spv);
- /** @brief Get all symbol pairs that occur in transitions of transducer
+ /** @brief Get all symbol pairs that occur in transitions of transducer
t. */
static StringPairSet get_symbol_pairs(MyFst *t);
@@ -296,8 +296,8 @@ namespace implementations
// you probably need also these functions
protected:
- /** @brief Add the following number-to-symbol correspondencies to the
- alphabet of transducer t:
+ /** @brief Add the following number-to-symbol correspondencies to the
+ alphabet of transducer t:
0 : "@_EPSILON_SYMBOL_@"
1 : "@_UNKNOWN_SYMBOL_@"
2 : "@_IDENTITY_SYMBOL_@" */
diff --git a/libhfst/src/implementations/SfstTransducer.cc b/libhfst/src/implementations/SfstTransducer.cc
index 8316409..0964452 100644
--- a/libhfst/src/implementations/SfstTransducer.cc
+++ b/libhfst/src/implementations/SfstTransducer.cc
@@ -1,12 +1,14 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
+#include "back-ends/sfst/interface.h"
+#include "back-ends/sfst/fst.h"
#include "SfstTransducer.h"
#include "HfstSymbolDefs.h"
#include <time.h>
@@ -16,6 +18,32 @@ using namespace SFST;
#ifndef MAIN_TEST
namespace hfst { namespace implementations {
+ class HfstNode2Int {
+
+ struct hashf {
+ size_t operator()(const SFST::Node *node) const {
+ return (size_t)node;
+ }
+ };
+ struct equalf {
+ int operator()(const SFST::Node *n1, const SFST::Node *n2) const {
+ return (n1 == n2);
+ }
+ };
+ typedef SFST::hash_map<SFST::Node*, int, hashf, equalf> NL;
+
+ private:
+ NL number;
+
+ public:
+ int &operator[]( SFST::Node *node ) {
+ NL::iterator it=number.find(node);
+ if (it == number.end())
+ return number.insert(NL::value_type(node, 0)).first->second;
+ return it->second;
+ };
+ };
+
float sfst_seconds_in_harmonize=0;
float SfstTransducer::get_profile_seconds() {
@@ -39,9 +67,9 @@ namespace hfst { namespace implementations {
if (filename == std::string())
{ input_file = stdin; }
else {
- input_file = fopen(filename.c_str(),"r");
+ input_file = hfst::hfst_fopen(filename.c_str(),"r");
if (input_file == NULL)
- {
+ {
HFST_THROW(StreamNotReadableException); }
}
}
@@ -61,7 +89,7 @@ namespace hfst { namespace implementations {
char SfstInputStream::stream_get() {
return (char) fgetc(input_file); }
- short SfstInputStream::stream_get_short()
+ short SfstInputStream::stream_get_short()
{
short i;
assert(1 == fread(&i,sizeof(i),1,input_file));
@@ -112,12 +140,12 @@ namespace hfst { namespace implementations {
Character c,
Alphabet &alphabet)
{
- const char * string_symbol =
+ const char * string_symbol =
alphabet.code2symbol(c);
if (string_number_map.find(string_symbol) == string_number_map.end())
{ string_number_map[string_symbol] = c; }
else if (string_number_map[string_symbol] != c)
- {
+ {
HFST_THROW_MESSAGE
(HfstFatalException,
"SfstInputStream: symbol redefined"); }
@@ -131,8 +159,8 @@ namespace hfst { namespace implementations {
}
- std::pair<Transducer*, Transducer*> SfstTransducer::harmonize
- (Transducer *t1, Transducer *t2, bool unknown_symbols_in_use)
+ std::pair<Transducer*, Transducer*> SfstTransducer::harmonize
+ (Transducer *t1, Transducer *t2, bool unknown_symbols_in_use)
{
try {
@@ -153,7 +181,7 @@ namespace hfst { namespace implementations {
Transducer * new_t1 = &t1->copy(false, &t2->alphabet);
new_t1->alphabet.insert_symbols(t2->alphabet);
SFST::Alphabet::CharMap cm = t1->alphabet.get_char_map();
- for (SFST::Alphabet::CharMap::const_iterator it = cm.begin();
+ for (SFST::Alphabet::CharMap::const_iterator it = cm.begin();
it != cm.end(); it++) {
new_t1->alphabet.add_symbol(it->second);
}
@@ -191,7 +219,7 @@ namespace hfst { namespace implementations {
}
void SfstInputStream::ignore(unsigned int n)
- {
+ {
for (unsigned int i=0; i<n; i++)
fgetc(input_file);
}
@@ -218,7 +246,7 @@ namespace hfst { namespace implementations {
unsigned int SfstTransducer::number_of_states(Transducer* t)
{
std::vector<SFST::Node*> indexing;
- std::pair<size_t, size_t> number_of_nodes_and_transitions =
+ std::pair<size_t, size_t> number_of_nodes_and_transitions =
t->nodeindexing(&indexing);
return number_of_nodes_and_transitions.first;
}
@@ -226,7 +254,7 @@ namespace hfst { namespace implementations {
unsigned int SfstTransducer::number_of_arcs(Transducer* t)
{
std::vector<SFST::Node*> indexing;
- std::pair<size_t, size_t> number_of_nodes_and_transitions =
+ std::pair<size_t, size_t> number_of_nodes_and_transitions =
t->nodeindexing(&indexing);
return number_of_nodes_and_transitions.second;
}
@@ -235,10 +263,10 @@ namespace hfst { namespace implementations {
Transducer * SfstInputStream::read_transducer()
{
if (is_eof())
- {
+ {
HFST_THROW(StreamIsClosedException); }
Transducer * t = NULL;
- try
+ try
{
// DEBUGGING
assert (stream_get() == 'a');
@@ -271,15 +299,15 @@ namespace hfst { namespace implementations {
filename(std::string(str))
{
if (filename != std::string()) {
- ofile = fopen(filename.c_str(), "wb");
+ ofile = hfst::hfst_fopen(filename.c_str(), "wb");
if (ofile == NULL)
HFST_THROW(StreamNotReadableException);
- }
+ }
else
ofile = stdout;
}
- void SfstOutputStream::close(void)
+ void SfstOutputStream::close(void)
{
if (filename != std::string())
{ fclose(ofile); }
@@ -310,10 +338,10 @@ namespace hfst { namespace implementations {
}
void SfstOutputStream::write_transducer(Transducer * transducer)
- {
- transducer->store(ofile);
+ {
+ transducer->store(ofile);
if (fflush(ofile) != 0) {
- HFST_THROW_MESSAGE(HfstFatalException,
+ HFST_THROW_MESSAGE(HfstFatalException,
"An error happened when writing an SfstTransducer.");
}
}
@@ -327,7 +355,7 @@ namespace hfst { namespace implementations {
{
unsigned int biggest_number=0;
SFST::Alphabet::CharMap cm = t->alphabet.get_char_map();
- for (SFST::Alphabet::CharMap::const_iterator it = cm.begin();
+ for (SFST::Alphabet::CharMap::const_iterator it = cm.begin();
it != cm.end(); it++) {
if (it->first > biggest_number)
biggest_number = it->first;
@@ -367,7 +395,7 @@ namespace hfst { namespace implementations {
void SfstTransducer::print_alphabet(Transducer *t) {
fprintf(stderr, "alphabet..\n");
SFST::Alphabet::CharMap cm = t->alphabet.get_char_map();
- for (SFST::Alphabet::CharMap::const_iterator it = cm.begin();
+ for (SFST::Alphabet::CharMap::const_iterator it = cm.begin();
it != cm.end(); it++)
fprintf(stderr, "%i\t%s\n",it->first,it->second);
fprintf(stderr, "..alphabet\n");
@@ -382,13 +410,13 @@ namespace hfst { namespace implementations {
}
Transducer * SfstTransducer::create_empty_transducer(void)
- { Transducer * retval = new Transducer();
+ { Transducer * retval = new Transducer();
initialize_alphabet(retval);
return retval;
}
Transducer * SfstTransducer::create_epsilon_transducer(void)
- { Transducer * t = new Transducer;
+ { Transducer * t = new Transducer;
initialize_alphabet(t);
t->root_node()->set_final(1);
return t; }
@@ -406,7 +434,7 @@ namespace hfst { namespace implementations {
{ Transducer * t = new Transducer;
initialize_alphabet(t);
Node * n = t->new_node();
- t->root_node()->add_arc(Label(inumber,
+ t->root_node()->add_arc(Label(inumber,
onumber),n,t);
n->set_final(1);
return t; }
@@ -414,7 +442,7 @@ namespace hfst { namespace implementations {
Transducer * SfstTransducer::define_transducer(const std::string &symbol)
{ Transducer * t = new Transducer;
- initialize_alphabet(t);
+ initialize_alphabet(t);
Node * n = t->new_node();
unsigned int number;
@@ -458,7 +486,7 @@ namespace hfst { namespace implementations {
Node * temp = t->new_node();
unsigned int inumber,onumber;
- if (is_epsilon(it->first) ||
+ if (is_epsilon(it->first) ||
strcmp(it->first.c_str(),"<>") == 0 )
inumber=0;
else
@@ -515,11 +543,11 @@ namespace hfst { namespace implementations {
{
Node * temp = t->new_node();
- for (StringPairSet::const_iterator it2 = (*it).begin();
- it2 != (*it).end(); it2++ )
+ for (StringPairSet::const_iterator it2 = (*it).begin();
+ it2 != (*it).end(); it2++ )
{
unsigned int inumber,onumber;
- if (is_epsilon(it2->first) ||
+ if (is_epsilon(it2->first) ||
strcmp(it2->first.c_str(),"<>") == 0 )
inumber=0;
else
@@ -538,6 +566,9 @@ namespace hfst { namespace implementations {
n->set_final(1);
return t; }
+ void SfstTransducer::delete_transducer(Transducer * t)
+ { delete t; }
+
Transducer * SfstTransducer::copy(Transducer * t)
{ return &t->copy(); }
@@ -545,8 +576,8 @@ namespace hfst { namespace implementations {
{ return &t->determinise(); }
Transducer * SfstTransducer::minimize(Transducer * t)
- { Transducer * retval = &t->minimise(false);
- retval->alphabet.copy(t->alphabet);
+ { Transducer * retval = &t->minimise(false);
+ retval->alphabet.copy(t->alphabet);
return retval; }
Transducer * SfstTransducer::remove_epsilons(Transducer * t)
@@ -556,7 +587,7 @@ namespace hfst { namespace implementations {
{ return &t->kleene_star(); }
Transducer * SfstTransducer::repeat_plus(Transducer * t)
- { Transducer * star = repeat_star(t);
+ { Transducer * star = repeat_star(t);
t = &(*t + *star);
delete star;
return t; }
@@ -598,14 +629,14 @@ namespace hfst { namespace implementations {
{ return &t->reverse(); }
Transducer * SfstTransducer::extract_input_language(Transducer * t)
- {
+ {
Transducer * retval = &t->lower_level();
// projection includes in the alphabet only symbols that
// occur in the input side, which we do not want
SFST::Alphabet::CharMap _cm = t->alphabet.get_char_map();
- for (SFST::Alphabet::CharMap::const_iterator it
+ for (SFST::Alphabet::CharMap::const_iterator it
= _cm.begin(); it != _cm.end(); it++) {
retval->alphabet.add_symbol(it->second, it->first);
}
@@ -615,7 +646,7 @@ namespace hfst { namespace implementations {
retval = substitute(retval, internal_unknown, internal_identity);
delete tmp;
- return retval;
+ return retval;
}
Transducer * SfstTransducer::extract_output_language(Transducer * t)
@@ -627,7 +658,7 @@ namespace hfst { namespace implementations {
// occur in the output side, which we do not want
SFST::Alphabet::CharMap _cm = t->alphabet.get_char_map();
- for (SFST::Alphabet::CharMap::const_iterator it
+ for (SFST::Alphabet::CharMap::const_iterator it
= _cm.begin(); it != _cm.end(); it++) {
retval->alphabet.add_symbol(it->second, it->first);
}
@@ -701,7 +732,7 @@ namespace hfst { namespace implementations {
bool added_fd_state = false;
if (fd_state_stack) {
- if(fd_state_stack->back().get_table().get_operation(l.lower_char())
+ if(fd_state_stack->back().get_table().get_operation(l.lower_char())
!= NULL) {
fd_state_stack->push_back(fd_state_stack->back());
if(fd_state_stack->back().apply_operation(l.lower_char()))
@@ -713,18 +744,18 @@ namespace hfst { namespace implementations {
}
}
- /* Handle spv here. Special symbols (flags, epsilons) are always
+ /* Handle spv here. Special symbols (flags, epsilons) are always
inserted. */
Character lc=l.lower_char();
Character uc=l.upper_char();
std::string istring("");
std::string ostring("");
- if (!filter_fd ||
+ if (!filter_fd ||
fd_state_stack->back().get_table().get_operation(lc) == NULL)
istring = std::string(t->alphabet.write_char(lc));
- if (!filter_fd ||
+ if (!filter_fd ||
fd_state_stack->back().get_table().get_operation(uc) == NULL)
ostring = std::string(t->alphabet.write_char(uc));
@@ -735,9 +766,9 @@ namespace hfst { namespace implementations {
spv.push_back(StringPair(istring, ostring));
- res = extract_paths(t, arc[i]->target_node(), all_visitations,
+ res = extract_paths(t, arc[i]->target_node(), all_visitations,
path_visitations,
- callback, cycles,
+ callback, cycles,
fd_state_stack, filter_fd, spv);
spv.pop_back();
@@ -752,7 +783,7 @@ namespace hfst { namespace implementations {
static const int BUFFER_START_SIZE = 64;
void SfstTransducer::extract_paths
- (Transducer * t, hfst::ExtractStringsCb& callback, int cycles,
+ (Transducer * t, hfst::ExtractStringsCb& callback, int cycles,
FdTable<SFST::Character>* fd, bool filter_fd)
{
if(!t->root_node())
@@ -760,15 +791,15 @@ namespace hfst { namespace implementations {
HfstNode2Int all_visitations;
HfstNode2Int path_visitations;
- vector<hfst::FdState<Character> >* fd_state_stack =
- (fd==NULL) ? NULL :
+ vector<hfst::FdState<Character> >* fd_state_stack =
+ (fd==NULL) ? NULL :
new std::vector<hfst::FdState<Character> >
(1, hfst::FdState<Character>(*fd));
StringPairVector spv;
hfst::implementations::extract_paths
(t, t->root_node(), all_visitations, path_visitations,
- callback, cycles, fd_state_stack, filter_fd,
+ callback, cycles, fd_state_stack, filter_fd,
spv);
// add epsilon path, if needed
@@ -783,7 +814,7 @@ namespace hfst { namespace implementations {
/* Get a random path from transducer \a t. */
static HfstTwoLevelPath random_path(Transducer *t) {
- HfstTwoLevelPath path;
+ HfstTwoLevelPath path;
Node * current_t_node = t->root_node();
/* If we cannot proceed, all elements in \a path whose index is smaller
@@ -792,9 +823,9 @@ namespace hfst { namespace implementations {
int last_index=0;
std::vector<SFST::Node*> indexing;
- std::pair<size_t, size_t> number_of_nodes_and_transitions =
+ std::pair<size_t, size_t> number_of_nodes_and_transitions =
t->nodeindexing(&indexing);
- unsigned int number_of_nodes =
+ unsigned int number_of_nodes =
(unsigned int) number_of_nodes_and_transitions.first;
/* Whether a state has been visited. */
@@ -802,7 +833,7 @@ namespace hfst { namespace implementations {
visited.reserve(number_of_nodes);
/* Whether the state is marked as broken, i.e. we cannot proceed from
- that state. These arrays are used for giving more probability for
+ that state. These arrays are used for giving more probability for
shorter paths if \a t is cyclic. */
std::vector<int> broken;
broken.reserve(number_of_nodes);
@@ -824,7 +855,7 @@ namespace hfst { namespace implementations {
/* If we cannot proceed, return the longest path so far. */
if (t_transitions.empty() || broken[current_t_node->index]) {
for (int i=(int)path.second.size()-1; i>=last_index; i--) {
- path.second.pop_back();
+ path.second.pop_back();
}
return path;
}
@@ -838,9 +869,9 @@ namespace hfst { namespace implementations {
Node * t_target = arc.target_node();
- std::string istring
+ std::string istring
= t->alphabet.code2symbol(arc.label().lower_char());
- std::string ostring
+ std::string ostring
= t->alphabet.code2symbol(arc.label().upper_char());
if (istring.compare("<>") == 0)
istring = std::string(internal_epsilon);
@@ -855,18 +886,18 @@ namespace hfst { namespace implementations {
if ( (rand() % 4) == 0 ) { // randomly return the path so far,
return path;
} // or continue.
- last_index = (int)path.second.size();
- }
+ last_index = (int)path.second.size();
+ }
/* Give more probability for shorter paths. */
if ( broken[ t_target->index ] == 0 ) {
- if ( visited[ t_target->index ] == 1 )
+ if ( visited[ t_target->index ] == 1 )
if ( (rand() % 4) == 0 )
broken[ t_target->index ] = 1;
}
- if ( visited[ t_target->index ] == 1 ) {
+ if ( visited[ t_target->index ] == 1 ) {
if ( (rand() % 4) == 0 )
broken[ t_target->index ] = 1;
}
@@ -874,7 +905,7 @@ namespace hfst { namespace implementations {
/* Proceed to the target state. */
current_t_node = t_target;
break;
- }
+ }
}
return path;
};
@@ -918,7 +949,7 @@ namespace hfst { namespace implementations {
}
results.insert(path);
- --max_num;
+ --max_num;
}
}
@@ -948,7 +979,7 @@ namespace hfst { namespace implementations {
if (is_epsilon(new_symbol))
new_symbol_ = std::string("<>");
- Transducer * retval =
+ Transducer * retval =
&t->replace_char(t->alphabet.add_symbol(old_symbol_.c_str()),
t->alphabet.add_symbol(new_symbol_.c_str()));
retval->alphabet.copy(t->alphabet);
@@ -956,7 +987,7 @@ namespace hfst { namespace implementations {
Transducer * SfstTransducer::substitute
(Transducer *t, const StringPair &symbol_pair, Transducer *tr)
- {
+ {
std::string isymbol = symbol_pair.first;
std::string osymbol = symbol_pair.second;
if (is_epsilon(isymbol))
@@ -964,7 +995,7 @@ namespace hfst { namespace implementations {
if (is_epsilon(osymbol))
osymbol = std::string("<>");
- Transducer * retval
+ Transducer * retval
= &t->splice( Label(
t->alphabet.add_symbol(isymbol.c_str()),
t->alphabet.add_symbol(osymbol.c_str()) ), tr );
@@ -975,7 +1006,7 @@ namespace hfst { namespace implementations {
Transducer * SfstTransducer::compose
(Transducer * t1, Transducer * t2)
- {
+ {
return &t1->operator||(*t2); }
Transducer * SfstTransducer::concatenate
@@ -990,11 +1021,11 @@ namespace hfst { namespace implementations {
(Transducer * t, const StringPairVector &spv)
{
Node *node= t->root_node();
- for (StringPairVector::const_iterator it = spv.begin();
- it != spv.end(); it++)
+ for (StringPairVector::const_iterator it = spv.begin();
+ it != spv.end(); it++)
{
unsigned int inumber,onumber;
- if (is_epsilon(it->first) ||
+ if (is_epsilon(it->first) ||
strcmp(it->first.c_str(),"<>") == 0 )
inumber=0;
else
@@ -1024,21 +1055,21 @@ namespace hfst { namespace implementations {
Transducer * SfstTransducer::subtract
(Transducer * t1, Transducer * t2)
- {
- try {
+ {
+ try {
unsigned int t1_alphabet_size = t1->alphabet.size();
// This will cause an exception when SFST calculates the negation
// that is needed in subtraction.
if (t1_alphabet_size == 0) {
t1->alphabet.insert(Label(1,1)); // insert a dummy symbol pair
- }
- Transducer * retval = &t1->operator/(*t2);
+ }
+ Transducer * retval = &t1->operator/(*t2);
if (t1_alphabet_size == 0) {
t1->alphabet.clear_char_pairs(); // remove the dummy symbol pair
t1->complete_alphabet();
}
return retval;
- }
+ }
catch (const char *msg) {
fprintf(stderr, "ERROR: %s\n", msg);
HFST_THROW_MESSAGE(HfstFatalException, std::string(msg));
@@ -1058,14 +1089,14 @@ namespace hfst { namespace implementations {
bool SfstTransducer::is_automaton(Transducer * t)
{
return t->is_automaton();
- }
+ }
FdTable<SFST::Character>* SfstTransducer::get_flag_diacritics(Transducer * t)
{
FdTable<SFST::Character>* table = new FdTable<SFST::Character>();
SFST::Alphabet::CharMap cm = t->alphabet.get_char_map();
- for (SFST::Alphabet::CharMap::const_iterator it
+ for (SFST::Alphabet::CharMap::const_iterator it
= cm.begin(); it != cm.end(); it++) {
if(FdOperation::is_diacritic(it->second))
table->define_diacritic(it->first, it->second);
@@ -1090,7 +1121,7 @@ namespace hfst { namespace implementations {
std::vector<SFST::Label> label;
SFST::Alphabet::CharMap cm = alpha.get_char_map();
- for( SFST::Alphabet::CharMap::const_iterator it=cm.begin();
+ for( SFST::Alphabet::CharMap::const_iterator it=cm.begin();
it!=cm.end(); it++ ) {
SFST::Character c=it->first;
char *s=it->second;
@@ -1100,7 +1131,7 @@ namespace hfst { namespace implementations {
}
}
- for( std::set<SFST::Label>::const_iterator it=alpha.begin();
+ for( std::set<SFST::Label>::const_iterator it=alpha.begin();
it!=alpha.end(); it++ ) {
SFST::Label l=*it;
if (strcmp(alpha.code2symbol(l.upper_char()), symbol_to_remove) != 0 &&
@@ -1134,7 +1165,7 @@ namespace hfst { namespace implementations {
}
unsigned int SfstTransducer::get_symbol_number
- (Transducer *t,
+ (Transducer *t,
const std::string &symbol)
{
if (symbol == "@_EPSILON_SYMBOL_@")
@@ -1159,13 +1190,13 @@ namespace hfst { namespace implementations {
if (isymbol == NULL) {
HFST_THROW_MESSAGE(HfstFatalException, "input number not found");
- //fprintf(stderr, "ERROR: input number %i not found\n",
+ //fprintf(stderr, "ERROR: input number %i not found\n",
// it->lower_char());
//exit(1);
}
if (osymbol == NULL) {
HFST_THROW_MESSAGE(HfstFatalException, "output number not found");
- //fprintf(stderr, "ERROR: input number %i not found\n",
+ //fprintf(stderr, "ERROR: input number %i not found\n",
// it->upper_char());
//exit(1);
}
@@ -1190,7 +1221,7 @@ namespace hfst { namespace implementations {
{
if ( l.lower_char() == 1 && l.upper_char() == 1 ) // cross product "?:?"
{
- for (hfst::StringSet::iterator it1 = s.begin(); it1 != s.end(); it1++)
+ for (hfst::StringSet::iterator it1 = s.begin(); it1 != s.end(); it1++)
{
if (not FdOperation::is_diacritic(*it1)) {
@@ -1200,8 +1231,8 @@ namespace hfst { namespace implementations {
<< std::endl;
assert(false);
}
- for (hfst::StringSet::iterator it2 = s.begin();
- it2 != s.end(); it2++)
+ for (hfst::StringSet::iterator it2 = s.begin();
+ it2 != s.end(); it2++)
{
if (not FdOperation::is_diacritic(*it2)) {
int onumber = t->alphabet.symbol2code(it2->c_str());
@@ -1210,8 +1241,8 @@ namespace hfst { namespace implementations {
<< std::endl;
assert(false);
}
- if (inumber != onumber) {
- // add transitions of type x:y
+ if (inumber != onumber) {
+ // add transitions of type x:y
// (non-identity cross-product of symbols in s)
origin->add_arc( Label(inumber, onumber), target, t );
}
@@ -1225,7 +1256,7 @@ namespace hfst { namespace implementations {
}
else if (l.lower_char() == 2 || l.upper_char() == 2 ) // identity "?:?"
{
- for (hfst::StringSet::iterator it = s.begin(); it != s.end(); it++)
+ for (hfst::StringSet::iterator it = s.begin(); it != s.end(); it++)
{
if (not FdOperation::is_diacritic(*it)) {
int number = t->alphabet.symbol2code(it->c_str());
@@ -1241,7 +1272,7 @@ namespace hfst { namespace implementations {
}
else if (l.lower_char() == 1) // "?:x"
{
- for (hfst::StringSet::iterator it = s.begin(); it != s.end(); it++)
+ for (hfst::StringSet::iterator it = s.begin(); it != s.end(); it++)
{
if (not FdOperation::is_diacritic(*it)) {
int number = t->alphabet.symbol2code(it->c_str());
@@ -1256,7 +1287,7 @@ namespace hfst { namespace implementations {
}
else if (l.upper_char() == 1) // "x:?"
{
- for (hfst::StringSet::iterator it = s.begin(); it != s.end(); it++)
+ for (hfst::StringSet::iterator it = s.begin(); it != s.end(); it++)
{
if (not FdOperation::is_diacritic(*it)) {
int number = t->alphabet.symbol2code(it->c_str());
@@ -1268,7 +1299,7 @@ namespace hfst { namespace implementations {
origin->add_arc( Label(l.lower_char(), number), target, t );
}
}
- }
+ }
// keep the original transition in all cases
return;
}
@@ -1315,15 +1346,15 @@ using namespace hfst::implementations;
bool does_sfst_alphabet_contain(SFST::Transducer *t, const char *str)
{
SFST::Alphabet::CharMap cm = t->alphabet.get_char_map();
- for (SFST::Alphabet::CharMap::const_iterator it = cm.begin();
+ for (SFST::Alphabet::CharMap::const_iterator it = cm.begin();
it != cm.end(); it++) {
if (strcmp(str, it->second) == 0) {
return true; }
- }
+ }
return false;
}
-int main(int argc, char * argv[])
+int main(int argc, char * argv[])
{
std::cout << "Unit tests for " __FILE__ ":";
@@ -1331,26 +1362,25 @@ int main(int argc, char * argv[])
SFST::Transducer * t = SfstTransducer::define_transducer("a", "b");
SFST::Transducer * t_input = SfstTransducer::extract_input_language(t);
- std::cout << "#1.5" << std::endl;
- assert( does_sfst_alphabet_contain(t_input, "a") &&
+ assert( does_sfst_alphabet_contain(t_input, "a") &&
does_sfst_alphabet_contain(t_input, "b") );
SFST::Transducer * t_output = SfstTransducer::extract_output_language(t);
- assert( does_sfst_alphabet_contain(t_output, "a") &&
+ assert( does_sfst_alphabet_contain(t_output, "a") &&
does_sfst_alphabet_contain(t_output, "b") );
SFST::Transducer * t_min = SfstTransducer::minimize(t_input);
- assert( does_sfst_alphabet_contain(t_min, "a") &&
+ assert( does_sfst_alphabet_contain(t_min, "a") &&
does_sfst_alphabet_contain(t_min, "b") );
SFST::Transducer * t_eps_free = SfstTransducer::remove_epsilons(t_output);
- assert( does_sfst_alphabet_contain(t_eps_free, "a") &&
+ assert( does_sfst_alphabet_contain(t_eps_free, "a") &&
does_sfst_alphabet_contain(t_eps_free, "b") );
SFST::Transducer * t_subst = SfstTransducer::substitute(t, "a", "c");
- assert( does_sfst_alphabet_contain(t_subst, "a") &&
- does_sfst_alphabet_contain(t_subst, "b") &&
- does_sfst_alphabet_contain(t_subst, "c") );
+ assert( does_sfst_alphabet_contain(t_subst, "a") &&
+ does_sfst_alphabet_contain(t_subst, "b") &&
+ does_sfst_alphabet_contain(t_subst, "c") );
std::cout << std::endl << "ok" << std::endl;
return EXIT_SUCCESS;
diff --git a/libhfst/src/implementations/SfstTransducer.h b/libhfst/src/implementations/SfstTransducer.h
index 6356f8f..ae4f67d 100644
--- a/libhfst/src/implementations/SfstTransducer.h
+++ b/libhfst/src/implementations/SfstTransducer.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _SFST_TRANSDUCER_H_
@@ -14,18 +14,21 @@
#include "HfstFlagDiacritics.h"
#include "HfstSymbolDefs.h"
#include "HfstExtractStrings.h"
-#include "back-ends/sfst/interface.h"
-#include "back-ends/sfst/fst.h"
+//#include "back-ends/sfst/interface.h"
+//#include "back-ends/sfst/fst.h"
+
+namespace SFST { class Transducer; typedef short unsigned int Character; class Alphabet; class Label; }
+
#include <cstdio>
#include <string>
#include <sstream>
-#include <iostream>
+#include <iosfwd>
/** @file SfstTransducer.h
\brief Declarations of functions and datatypes that form a bridge between
HFST API and SFST. */
-namespace hfst {
+namespace hfst {
namespace implementations
{
//using namespace SFST;
@@ -36,7 +39,7 @@ namespace implementations
void sfst_set_hopcroft(bool);
- class SfstInputStream
+ class SfstInputStream
{
private:
std::string filename;
@@ -69,14 +72,14 @@ namespace implementations
static bool is_fst(std::istream &s);
};
- class SfstOutputStream
+ class SfstOutputStream
{
private:
std::string filename;
FILE *ofile;
//void write_3_0_library_header(FILE *file, bool is_minimal);
public:
- SfstOutputStream(void);
+ SfstOutputStream(void);
SfstOutputStream(const std::string &filename);
void close(void);
void write(const char &c);
@@ -85,6 +88,7 @@ namespace implementations
void write_transducer(SFST::Transducer * transducer);
};
+ /*
class HfstNode2Int {
struct hashf {
@@ -109,7 +113,7 @@ namespace implementations
return number.insert(NL::value_type(node, 0)).first->second;
return it->second;
};
- };
+ };*/
class SfstTransducer
@@ -122,6 +126,8 @@ namespace implementations
static SFST::Transducer * define_transducer
(unsigned int inumber, unsigned int onumber);
+ static void delete_transducer(SFST::Transducer * t);
+
static SFST::Transducer * define_transducer(const std::string &symbol);
static SFST::Transducer * define_transducer
(const std::string &isymbol, const std::string &osymbol);
@@ -200,14 +206,14 @@ namespace implementations
static void insert_to_alphabet(SFST::Transducer *t, const std::string &symbol);
static void remove_from_alphabet
(SFST::Transducer *t, const std::string &symbol);
- static unsigned int get_symbol_number(SFST::Transducer *t,
+ static unsigned int get_symbol_number(SFST::Transducer *t,
const std::string &symbol);
protected:
static void initialize_alphabet(SFST::Transducer *t);
static SFST::Transducer * expand_arcs(SFST::Transducer * t, StringSet &unknown);
- static void expand_node( SFST::Transducer *t, SFST::Node *origin, SFST::Label &l,
+ static void expand_node( SFST::Transducer *t, SFST::Node *origin, SFST::Label &l,
SFST::Node *target, hfst::StringSet &s );
static void expand2
( SFST::Transducer *t, SFST::Node *node,
diff --git a/libhfst/src/implementations/TropicalWeightTransducer.cc b/libhfst/src/implementations/TropicalWeightTransducer.cc
index 5c87a0d..1314db1 100644
--- a/libhfst/src/implementations/TropicalWeightTransducer.cc
+++ b/libhfst/src/implementations/TropicalWeightTransducer.cc
@@ -1,23 +1,30 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "TropicalWeightTransducer.h"
#include "HfstSymbolDefs.h"
#include "HfstLookupFlagDiacritics.h"
-#include "HfstTransitionGraph.h"
+#include "HfstBasicTransducer.h"
#include "ConvertTransducerFormat.h"
+#ifdef _MSC_VER
+#include "back-ends/openfstwin/src/include/fst/fstlib.h"
+#else
+#include "back-ends/openfst/src/include/fst/fstlib.h"
+#endif // _MSC_VER
+
+
#ifndef MAIN_TEST
#define CHECK_EPSILON_CYCLES(x, y) { hfst::implementations::HfstBasicTransducer * fsm = hfst::implementations::ConversionFunctions::tropical_ofst_to_hfst_basic_transducer( x ); if (fsm->has_negative_epsilon_cycles()) { if (warning_stream != NULL) { *warning_stream << y << ": warning: transducer has epsilon cycles with a negative weight" << std::endl; } } delete fsm; }
-namespace hfst {
+namespace hfst {
bool get_encode_weights();
namespace implementations {
@@ -54,11 +61,11 @@ namespace hfst {
void TropicalWeightTransducer::add_to_weights(StdVectorFst * t, float w)
{
- for (fst::StateIterator<StdVectorFst> siter(*t);
- ! siter.Done(); siter.Next())
+ for (fst::StateIterator<StdVectorFst> siter(*t);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
- for (fst::MutableArcIterator<fst::StdVectorFst> aiter(t,s);
+ for (fst::MutableArcIterator<fst::StdVectorFst> aiter(t,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
@@ -84,12 +91,11 @@ namespace hfst {
// in case of an empty transducer, infinity is returned
// (empty in the sense of having no transitions or final states)
float retval = std::numeric_limits<float>::infinity();
- bool weight_found;
- for (fst::StateIterator<StdVectorFst> siter(*t);
- ! siter.Done(); siter.Next())
+ for (fst::StateIterator<StdVectorFst> siter(*t);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
- for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
@@ -116,10 +122,10 @@ namespace hfst {
RmEpsilon<StdArc>(t);
- float w = get_smallest_weight(t);
- if (w < 0)
- {
- add_to_weights(t, -w);
+ float w = get_smallest_weight(t);
+ if (w < 0)
+ {
+ add_to_weights(t, -w);
}
EncodeMapper<StdArc> encode_mapper
@@ -131,9 +137,9 @@ namespace hfst {
Minimize<StdArc>(det);
Decode(det, encode_mapper);
- if (w < 0)
- {
- add_to_weights(det, w);
+ if (w < 0)
+ {
+ add_to_weights(det, w);
}
return det;
@@ -195,8 +201,8 @@ namespace hfst {
(StdVectorFst *t, const std::string &symbol)
{
assert(t->InputSymbols() != NULL);
- fst::SymbolTable st(t->InputSymbols()->Name());
- for ( fst::SymbolTableIterator it
+ fst::SymbolTable st(t->InputSymbols()->Name());
+ for ( fst::SymbolTableIterator it
= fst::SymbolTableIterator(*(t->InputSymbols()));
!it.Done(); it.Next() ) {
if (it.Symbol() != symbol) {
@@ -210,7 +216,7 @@ namespace hfst {
(StdVectorFst *t, StateId s, std::set<StateId> & visited_states, StringSet & symbols)
{
visited_states.insert(s);
- for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
@@ -243,7 +249,7 @@ namespace hfst {
{
assert(t->InputSymbols() != NULL);
StringSet s;
- for ( fst::SymbolTableIterator it
+ for ( fst::SymbolTableIterator it
= fst::SymbolTableIterator(*(t->InputSymbols()));
! it.Done(); it.Next() ) {
s.insert( std::string(it.Symbol()) );
@@ -252,7 +258,7 @@ namespace hfst {
}
unsigned int TropicalWeightTransducer::get_symbol_number
- (StdVectorFst *t,
+ (StdVectorFst *t,
const std::string &symbol)
{
assert(t->InputSymbols() != NULL);
@@ -266,12 +272,12 @@ namespace hfst {
(StdVectorFst *t)
{
unsigned int biggest_number=0;
- for ( fst::SymbolTableIterator it
+ for ( fst::SymbolTableIterator it
= fst::SymbolTableIterator(*(t->InputSymbols()));
! it.Done(); it.Next() ) {
- if (it.Value() > biggest_number)
+ if (it.Value() > biggest_number)
biggest_number = it.Value();
- }
+ }
return biggest_number;
}
@@ -292,19 +298,19 @@ namespace hfst {
return symbol_vector;
}
- /* Find the number-to-number mappings needed to be performed to t1
+ /* Find the number-to-number mappings needed to be performed to t1
so that it will follow the same symbol-to-number encoding as t2.
- @pre t2's symbol table must contain all symbols in t1's symbol table.
+ @pre t2's symbol table must contain all symbols in t1's symbol table.
*/
NumberNumberMap TropicalWeightTransducer::create_mapping
(fst::StdVectorFst *t1, fst::StdVectorFst *t2)
{
NumberNumberMap km;
// find the number-to-number mappings for transducer t1
- for ( fst::SymbolTableIterator it
+ for ( fst::SymbolTableIterator it
= fst::SymbolTableIterator(*(t1->InputSymbols()));
- ! it.Done(); it.Next() ) {
- km [ (unsigned int)it.Value() ]
+ ! it.Done(); it.Next() ) {
+ km [ (unsigned int)it.Value() ]
= (unsigned int) t2->InputSymbols()->Find( it.Symbol() );
assert(it.Value() >= 0);
@@ -316,13 +322,13 @@ namespace hfst {
/* Recode the symbol numbers in this transducer as indicated in KeyMap km. */
void TropicalWeightTransducer::recode_symbol_numbers
- (StdVectorFst *t, NumberNumberMap &km)
+ (StdVectorFst *t, NumberNumberMap &km)
{
- for (fst::StateIterator<StdVectorFst> siter(*t);
+ for (fst::StateIterator<StdVectorFst> siter(*t);
! siter.Done(); siter.Next())
{
StateId s = siter.Value();
- for (fst::MutableArcIterator<StdVectorFst> aiter(t,s);
+ for (fst::MutableArcIterator<StdVectorFst> aiter(t,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
@@ -344,7 +350,7 @@ namespace hfst {
StdVectorFst * TropicalWeightTransducer::set_final_weights
(StdVectorFst * t, float weight, bool increment)
{
- for (fst::StateIterator<StdVectorFst> siter(*t);
+ for (fst::StateIterator<StdVectorFst> siter(*t);
! siter.Done(); siter.Next())
{
StateId s = siter.Value();
@@ -381,13 +387,13 @@ namespace hfst {
StdVectorFst * TropicalWeightTransducer::transform_weights
(StdVectorFst * t,float (*func)(float f))
{
- for (fst::StateIterator<StdVectorFst> siter(*t);
+ for (fst::StateIterator<StdVectorFst> siter(*t);
! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if ( t->Final(s) != TropicalWeight::Zero() )
t->SetFinal( s, func(t->Final(s).Value()) );
- for (fst::MutableArcIterator<StdVectorFst> aiter(t,s);
+ for (fst::MutableArcIterator<StdVectorFst> aiter(t,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
@@ -419,8 +425,8 @@ namespace hfst {
zero_print = initial_state;
}
- for (fst::StateIterator<StdVectorFst> siter(*t);
- ! siter.Done(); siter.Next())
+ for (fst::StateIterator<StdVectorFst> siter(*t);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s == initial_state) {
@@ -431,7 +437,7 @@ namespace hfst {
origin = 0;
else
origin = (int)s;
- for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
@@ -443,7 +449,7 @@ namespace hfst {
else
target = (int)arc.nextstate;
fprintf(ofile, "%i\t%i\t%s\t%s\t%f\n", origin, target,
- sym->Find(arc.ilabel).c_str(),
+ sym->Find(arc.ilabel).c_str(),
sym->Find(arc.olabel).c_str(),
arc.weight.Value());
}
@@ -453,7 +459,7 @@ namespace hfst {
}
}
- for (fst::StateIterator<StdVectorFst> siter(*t);
+ for (fst::StateIterator<StdVectorFst> siter(*t);
! siter.Done(); siter.Next())
{
StateId s = siter.Value();
@@ -465,7 +471,7 @@ namespace hfst {
origin = 0;
else
origin = (int)s;
- for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
@@ -477,7 +483,7 @@ namespace hfst {
else
target = (int)arc.nextstate;
fprintf(ofile, "%i\t%i\t%s\t%s\t%f\n", origin, target,
- sym->Find(arc.ilabel).c_str(),
+ sym->Find(arc.ilabel).c_str(),
sym->Find(arc.olabel).c_str(),
arc.weight.Value());
}
@@ -502,8 +508,8 @@ namespace hfst {
zero_print = initial_state;
}
- for (fst::StateIterator<StdVectorFst> siter(*t);
- ! siter.Done(); siter.Next())
+ for (fst::StateIterator<StdVectorFst> siter(*t);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s == initial_state) {
@@ -514,7 +520,7 @@ namespace hfst {
origin = 0;
else
origin = (int)s;
- for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
@@ -535,7 +541,7 @@ namespace hfst {
}
}
- for (fst::StateIterator<StdVectorFst> siter(*t);
+ for (fst::StateIterator<StdVectorFst> siter(*t);
! siter.Done(); siter.Next())
{
StateId s = siter.Value();
@@ -547,7 +553,7 @@ namespace hfst {
origin = 0;
else
origin = (int)s;
- for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
@@ -586,8 +592,8 @@ namespace hfst {
zero_print = initial_state;
}
- for (fst::StateIterator<StdVectorFst> siter(*t);
- ! siter.Done(); siter.Next())
+ for (fst::StateIterator<StdVectorFst> siter(*t);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s == initial_state) {
@@ -598,7 +604,7 @@ namespace hfst {
origin = 0;
else
origin = (int)s;
- for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
@@ -609,10 +615,10 @@ namespace hfst {
target = 0;
else
target = (int)arc.nextstate;
- os << origin << "\t"
- << target << "\t"
+ os << origin << "\t"
+ << target << "\t"
<< sym->Find(arc.ilabel).c_str() << "\t"
- << sym->Find(arc.olabel).c_str() << "\t"
+ << sym->Find(arc.olabel).c_str() << "\t"
<< arc.weight.Value() << "\n";
}
if (t->Final(s) != TropicalWeight::Zero())
@@ -622,7 +628,7 @@ namespace hfst {
}
}
- for (fst::StateIterator<StdVectorFst> siter(*t);
+ for (fst::StateIterator<StdVectorFst> siter(*t);
! siter.Done(); siter.Next())
{
StateId s = siter.Value();
@@ -634,7 +640,7 @@ namespace hfst {
origin = 0;
else
origin = (int)s;
- for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
@@ -645,7 +651,7 @@ namespace hfst {
target = 0;
else
target = (int)arc.nextstate;
- os << origin << "\t"
+ os << origin << "\t"
<< target << "\t"
<< sym->Find(arc.ilabel).c_str() << "\t"
<< sym->Find(arc.olabel).c_str() << "\t"
@@ -672,8 +678,8 @@ namespace hfst {
zero_print = initial_state;
}
- for (fst::StateIterator<StdVectorFst> siter(*t);
- ! siter.Done(); siter.Next())
+ for (fst::StateIterator<StdVectorFst> siter(*t);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s == initial_state) {
@@ -684,7 +690,7 @@ namespace hfst {
origin = 0;
else
origin = (int)s;
- for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
@@ -695,10 +701,10 @@ namespace hfst {
target = 0;
else
target = (int)arc.nextstate;
- os << origin << "\t"
- << target << "\t"
+ os << origin << "\t"
+ << target << "\t"
<< "\\" << arc.ilabel << "\t"
- << "\\" << arc.olabel << "\t"
+ << "\\" << arc.olabel << "\t"
<< arc.weight.Value() << "\n";
}
if (t->Final(s) != TropicalWeight::Zero())
@@ -708,7 +714,7 @@ namespace hfst {
}
}
- for (fst::StateIterator<StdVectorFst> siter(*t);
+ for (fst::StateIterator<StdVectorFst> siter(*t);
! siter.Done(); siter.Next())
{
StateId s = siter.Value();
@@ -720,7 +726,7 @@ namespace hfst {
origin = 0;
else
origin = (int)s;
- for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
@@ -731,7 +737,7 @@ namespace hfst {
target = 0;
else
target = (int)arc.nextstate;
- os << origin << "\t"
+ os << origin << "\t"
<< target << "\t"
<< "\\" << arc.ilabel << "\t"
<< "\\" << arc.olabel << "\t"
@@ -747,7 +753,7 @@ namespace hfst {
// AT&T format is handled here ------------------------------
- /* Maps state numbers in AT&T text format to state ids used by
+ /* Maps state numbers in AT&T text format to state ids used by
OpenFst transducers. */
typedef std::map<int, StateId> StateMap;
@@ -756,7 +762,7 @@ namespace hfst {
state to t if state_number is encountered for the first time and
updates state_map accordingly. */
StateId TropicalWeightTransducer::
- add_and_map_state(StdVectorFst *t, int state_number,
+ add_and_map_state(StdVectorFst *t, int state_number,
StateMap &state_map)
{
StateMap::iterator it = state_map.find(state_number);
@@ -770,8 +776,8 @@ namespace hfst {
}
// FIXME?: atof and atoi
- /* Reads a description of a transducer in AT&T text format and returns
- a corresponding binary transducer.
+ /* Reads a description of a transducer in AT&T text format and returns
+ a corresponding binary transducer.
@note The initial state must be numbered as zero. */
StdVectorFst * TropicalWeightTransducer::read_in_att_format(FILE * ifile)
{
@@ -786,12 +792,12 @@ namespace hfst {
t->SetStart(initial_state);
- while ( fgets(line, 255, ifile) != NULL )
+ while ( fgets(line, 255, ifile) != NULL )
{
if (*line == '-') // transducer separator
return t;
- char a1 [100]; char a2 [100]; char a3 [100];
+ char a1 [100]; char a2 [100]; char a3 [100];
char a4 [100]; char a5 [100];
int n = sscanf(line, "%s\t%s\t%s\t%s\t%s", a1, a2, a3, a4, a5);
@@ -813,16 +819,16 @@ namespace hfst {
{
int origin_number = atoi(a1);
int target_number = atoi(a2);
- StateId origin_state
+ StateId origin_state
= add_and_map_state(t, origin_number, state_map);
- StateId target_state
+ StateId target_state
= add_and_map_state(t, target_number, state_map);
int input_number = st.AddSymbol(std::string(a3));
int output_number = st.AddSymbol(std::string(a4));
- t->AddArc(origin_state,
- StdArc(input_number, output_number,
+ t->AddArc(origin_state,
+ StdArc(input_number, output_number,
weight, target_state));
}
@@ -840,9 +846,9 @@ namespace hfst {
return t;
}
- /*
- Create a copy of this transducer where all transitions of type
- "?:?", "?:x" and "x:?" are expanded according to the StringSymbolSet
+ /*
+ Create a copy of this transducer where all transitions of type
+ "?:?", "?:x" and "x:?" are expanded according to the StringSymbolSet
'unknown' that lists all symbols previously unknown to this transducer.
*/
StdVectorFst * TropicalWeightTransducer::expand_arcs
@@ -852,12 +858,12 @@ namespace hfst {
StdVectorFst * result = new StdVectorFst();
- for (fst::StateIterator<StdVectorFst> siter(*t);
+ for (fst::StateIterator<StdVectorFst> siter(*t);
! siter.Done(); siter.Next())
result->AddState();
// go through all states in this
- for (fst::StateIterator<StdVectorFst> siter(*t);
+ for (fst::StateIterator<StdVectorFst> siter(*t);
! siter.Done(); siter.Next())
{
// create new state in result, if needed
@@ -874,7 +880,7 @@ namespace hfst {
// go through all the arcs in this
- for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
@@ -887,74 +893,74 @@ namespace hfst {
if (unknown_symbols_in_use) {
- const fst::SymbolTable *is = t->InputSymbols();
+ const fst::SymbolTable *is = t->InputSymbols();
if ( arc.ilabel == 1 && // cross-product "?:?"
arc.olabel == 1 )
{
- for (StringSet::iterator it1 = unknown.begin();
- it1 != unknown.end(); it1++)
+ for (StringSet::iterator it1 = unknown.begin();
+ it1 != unknown.end(); it1++)
{
if (! FdOperation::is_diacritic(*it1)) {
int64 inumber = is->Find(*it1);
- for (StringSet::iterator it2 = unknown.begin();
- it2 != unknown.end(); it2++)
+ for (StringSet::iterator it2 = unknown.begin();
+ it2 != unknown.end(); it2++)
{
if (! FdOperation::is_diacritic(*it2)) {
int64 onumber = is->Find(*it2);
if (inumber != onumber)
- result->AddArc(result_s,
- StdArc(inumber, onumber,
- arc.weight,
+ result->AddArc(result_s,
+ StdArc(inumber, onumber,
+ arc.weight,
result_nextstate));
}
}
- result->AddArc(result_s,
- StdArc(inumber, 1, arc.weight,
+ result->AddArc(result_s,
+ StdArc(inumber, 1, arc.weight,
result_nextstate));
- result->AddArc(result_s,
- StdArc(1, inumber, arc.weight,
+ result->AddArc(result_s,
+ StdArc(1, inumber, arc.weight,
result_nextstate));
}
}
}
else if (arc.ilabel == 2 || // identity "?:?"
- arc.olabel == 2 )
+ arc.olabel == 2 )
{
- for (StringSet::iterator it = unknown.begin();
- it != unknown.end(); it++)
+ for (StringSet::iterator it = unknown.begin();
+ it != unknown.end(); it++)
{
if (! FdOperation::is_diacritic(*it)) {
int64 number = is->Find(*it);
- result->AddArc(result_s,
- StdArc(number, number,
+ result->AddArc(result_s,
+ StdArc(number, number,
arc.weight, result_nextstate));
}
}
}
else if (arc.ilabel == 1) // "?:x"
{
- for (StringSet::iterator it = unknown.begin();
- it != unknown.end(); it++)
+ for (StringSet::iterator it = unknown.begin();
+ it != unknown.end(); it++)
{
if (! FdOperation::is_diacritic(*it)) {
int64 number = is->Find(*it);
- result->AddArc(result_s,
- StdArc(number, arc.olabel,
+ result->AddArc(result_s,
+ StdArc(number, arc.olabel,
arc.weight, result_nextstate));
}
}
}
else if (arc.olabel == 1) // "x:?"
{
- for (StringSet::iterator it = unknown.begin();
- it != unknown.end(); it++)
+ for (StringSet::iterator it = unknown.begin();
+ it != unknown.end(); it++)
{
if (! FdOperation::is_diacritic(*it)) {
int64 number = is->Find(*it);
- result->AddArc(result_s,
- StdArc(arc.ilabel, number,
+ result->AddArc(result_s,
+ StdArc(arc.ilabel, number,
arc.weight, result_nextstate));
}
}
@@ -962,8 +968,8 @@ namespace hfst {
}
// the original transition is copied in all cases
- result->AddArc(result_s, StdArc(arc.ilabel, arc.olabel,
- arc.weight, result_nextstate));
+ result->AddArc(result_s, StdArc(arc.ilabel, arc.olabel,
+ arc.weight, result_nextstate));
}
}
@@ -971,16 +977,16 @@ namespace hfst {
}
- static StdVectorFst * copy_fst(const StdVectorFst * t)
+ /* static StdVectorFst * copy_fst(const StdVectorFst * t)
{
StdVectorFst * result = new StdVectorFst();
- for (fst::StateIterator<StdVectorFst> siter(*t);
+ for (fst::StateIterator<StdVectorFst> siter(*t);
! siter.Done(); siter.Next())
result->AddState();
// go through all states in this
- for (fst::StateIterator<StdVectorFst> siter(*t);
+ for (fst::StateIterator<StdVectorFst> siter(*t);
! siter.Done(); siter.Next())
{
// create new state in result, if needed
@@ -997,7 +1003,7 @@ namespace hfst {
// go through all the arcs in this
- for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
@@ -1007,17 +1013,17 @@ namespace hfst {
StateId result_nextstate=arc.nextstate;
// copy the transition
- result->AddArc(result_s, StdArc(arc.ilabel, arc.olabel,
- arc.weight, result_nextstate));
+ result->AddArc(result_s, StdArc(arc.ilabel, arc.olabel,
+ arc.weight, result_nextstate));
}
}
return result;
- }
+ }*/
unsigned int TropicalWeightTransducer::number_of_states(const StdVectorFst *t)
{
unsigned int retval=0;
- for (fst::StateIterator<StdVectorFst> siter(*t);
+ for (fst::StateIterator<StdVectorFst> siter(*t);
! siter.Done(); siter.Next())
retval++;
return retval;
@@ -1026,10 +1032,10 @@ namespace hfst {
unsigned int TropicalWeightTransducer::number_of_arcs(const StdVectorFst *t)
{
unsigned int retval=0;
- for (fst::StateIterator<StdVectorFst> siter(*t);
+ for (fst::StateIterator<StdVectorFst> siter(*t);
! siter.Done(); siter.Next())
{
- for (fst::ArcIterator<StdVectorFst> aiter(*t,siter.Value());
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,siter.Value());
!aiter.Done(); aiter.Next())
retval++;
}
@@ -1058,19 +1064,19 @@ namespace hfst {
if (DEBUG)
{
fprintf(stderr, "New symbols for t1: ");
- for (StringSet::const_iterator it = unknown_t1.begin();
+ for (StringSet::const_iterator it = unknown_t1.begin();
it != unknown_t1.end(); it++)
fprintf(stderr, "'%s', ", it->c_str());
fprintf(stderr, "\n");
fprintf(stderr, "New symbols for t2: ");
- for (StringSet::const_iterator it = unknown_t2.begin();
+ for (StringSet::const_iterator it = unknown_t2.begin();
it != unknown_t2.end(); it++)
fprintf(stderr, "'%s', ", it->c_str());
fprintf(stderr, "\n");
}
- // 2. Add new symbols from transducer t1 to the symbol table of
+ // 2. Add new symbols from transducer t1 to the symbol table of
// transducer t2...
SymbolTable * st2 = t2->InputSymbols()->Copy();
@@ -1116,7 +1122,7 @@ namespace hfst {
#ifdef PROFILE_MINIMIZATION
clock_t endclock = clock();
- tropical_seconds_in_harmonize = tropical_seconds_in_harmonize +
+ tropical_seconds_in_harmonize = tropical_seconds_in_harmonize +
( (float)(endclock - startclock) / CLOCKS_PER_SEC);
#endif
@@ -1151,7 +1157,7 @@ namespace hfst {
if (filename == string())
{ return std::cin.bad(); }
else
- { return input_stream.bad(); }
+ { return input_stream.bad(); }
}
bool TropicalWeightInputStream::is_good(void) const
{
@@ -1195,11 +1201,11 @@ namespace hfst {
HFST_THROW(StreamIsClosedException); }
StdVectorFst * t;
FstHeader header;
- try
+ try
{
if (filename == string())
{
- header.Read(input_stream,"STDIN");
+ header.Read(input_stream,"STDIN");
t = static_cast<StdVectorFst*>
(StdVectorFst::Read(input_stream,
FstReadOptions("STDIN",
@@ -1207,14 +1213,14 @@ namespace hfst {
}
else
{
- header.Read(input_stream,filename);
+ header.Read(input_stream,filename);
t = static_cast<StdVectorFst*>
(StdVectorFst::Read(input_stream,
FstReadOptions(filename,
&header)));
}
if (t == NULL)
- {
+ {
HFST_THROW(TransducerHasWrongTypeException); }
}
@@ -1231,7 +1237,7 @@ namespace hfst {
}
- TropicalWeightStateIterator::TropicalWeightStateIterator(StdVectorFst * t):
+ /*TropicalWeightStateIterator::TropicalWeightStateIterator(StdVectorFst * t):
iterator(new StateIterator<StdVectorFst>(*t))
{}
@@ -1305,7 +1311,7 @@ namespace hfst {
TropicalWeightTransition TropicalWeightTransitionIterator::value()
{
return TropicalWeightTransition(arc_iterator->Value(), this->t);
- }
+ }*/
fst::SymbolTable TropicalWeightTransducer::create_symbol_table
@@ -1323,7 +1329,7 @@ namespace hfst {
}
StdVectorFst * TropicalWeightTransducer::create_empty_transducer(void)
- {
+ {
StdVectorFst * t = new StdVectorFst;
initialize_symbol_tables(t);
StateId s = t->AddState();
@@ -1332,7 +1338,7 @@ namespace hfst {
}
StdVectorFst * TropicalWeightTransducer::create_epsilon_transducer(void)
- {
+ {
StdVectorFst * t = new StdVectorFst;
initialize_symbol_tables(t);
StateId s = t->AddState();
@@ -1341,6 +1347,11 @@ namespace hfst {
return t;
}
+ void TropicalWeightTransducer::delete_transducer(StdVectorFst * t)
+ {
+ delete t;
+ }
+
StdVectorFst * TropicalWeightTransducer::define_transducer
(unsigned int number)
{
@@ -1407,7 +1418,7 @@ namespace hfst {
}
bool TropicalWeightTransducer::are_equivalent
- (StdVectorFst *a_, StdVectorFst *b_)
+ (StdVectorFst *a_, StdVectorFst *b_)
{
StdVectorFst * a = copy(a_);
StdVectorFst * b = copy(b_);
@@ -1443,11 +1454,11 @@ namespace hfst {
bool TropicalWeightTransducer::is_automaton(StdVectorFst * t)
{
- for (fst::StateIterator<StdVectorFst> siter(*t);
- ! siter.Done(); siter.Next())
+ for (fst::StateIterator<StdVectorFst> siter(*t);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
- for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
@@ -1534,7 +1545,7 @@ namespace hfst {
++it)
{
StateId s2 = t->AddState();
- for (StringPairSet::const_iterator it2 = (*it).begin();
+ for (StringPairSet::const_iterator it2 = (*it).begin();
it2 != (*it).end(); it2++ ) {
assert(! (it2->first == ""));
@@ -1605,7 +1616,7 @@ namespace hfst {
++it)
{
StateId s2 = t->AddState();
- for (NumberPairSet::const_iterator it2 = (*it).begin();
+ for (NumberPairSet::const_iterator it2 = (*it).begin();
it2 != (*it).end(); it2++ ) {
t->AddArc(s1,StdArc(it2->first,it2->second,0,s2));
}
@@ -1616,12 +1627,12 @@ namespace hfst {
}
- StdVectorFst *
+ StdVectorFst *
TropicalWeightTransducer::copy(StdVectorFst * t)
{ return new StdVectorFst(*t); }
- StdVectorFst *
+ StdVectorFst *
TropicalWeightTransducer::determinize(StdVectorFst * t)
{
@@ -1629,10 +1640,10 @@ namespace hfst {
RmEpsilon<StdArc>(t);
- float w = get_smallest_weight(t);
- if (w < 0)
- {
- add_to_weights(t, -w);
+ float w = get_smallest_weight(t);
+ if (w < 0)
+ {
+ add_to_weights(t, -w);
}
EncodeMapper<StdArc> encode_mapper
@@ -1642,9 +1653,9 @@ namespace hfst {
Determinize<StdArc>(*t, det);
Decode(det, encode_mapper);
- if (w < 0)
- {
- add_to_weights(det, w);
+ if (w < 0)
+ {
+ add_to_weights(det, w);
}
return det;
@@ -1671,7 +1682,7 @@ namespace hfst {
void TropicalWeightTransducer::print_alphabet(const StdVectorFst *t)
{
for(fst::SymbolTableIterator it=fst::SymbolTableIterator
- (*t->InputSymbols());
+ (*t->InputSymbols());
!it.Done(); it.Next())
{
fprintf(stderr, "'%s', ", it.Symbol().c_str());
@@ -1681,18 +1692,18 @@ namespace hfst {
void print_att_number(StdVectorFst *t, FILE * ofile) {
fprintf(ofile, "initial state: %i\n", t->Start());
- for (fst::StateIterator<StdVectorFst> siter(*t);
- ! siter.Done(); siter.Next())
+ for (fst::StateIterator<StdVectorFst> siter(*t);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if ( t->Final(s) != TropicalWeight::Zero() )
fprintf(ofile, "%i\t%f\n", s, t->Final(s).Value());
- for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
- fprintf(ofile, "%i\t%i\t%i\t%i\t%f\n",
- s, arc.nextstate, arc.ilabel, arc.olabel,
+ fprintf(ofile, "%i\t%i\t%i\t%i\t%f\n",
+ s, arc.nextstate, arc.ilabel, arc.olabel,
arc.weight.Value());
}
}
@@ -1736,16 +1747,16 @@ namespace hfst {
/* For HfstMutableTransducer */
- StateId
+ StateId
TropicalWeightTransducer::add_state(StdVectorFst *t)
- {
+ {
StateId s = t->AddState();
if (s == 0)
t->SetStart(s);
return s;
}
- void
+ void
TropicalWeightTransducer::set_final_weight
(StdVectorFst *t, StateId s, float w)
{
@@ -1753,9 +1764,9 @@ namespace hfst {
return;
}
- void
+ void
TropicalWeightTransducer::add_transition
- (StdVectorFst *t, StateId source, std::string &isymbol,
+ (StdVectorFst *t, StateId source, std::string &isymbol,
std::string &osymbol, float w, StateId target)
{
SymbolTable *st = t->InputSymbols()->Copy();
@@ -1767,13 +1778,13 @@ namespace hfst {
return;
}
- float
+ float
TropicalWeightTransducer::get_final_weight(StdVectorFst *t, StateId s)
{
return t->Final(s).Value();
}
- float
+ float
TropicalWeightTransducer::is_final(StdVectorFst *t, StateId s)
{
return ( t->Final(s) != TropicalWeight::Zero() );
@@ -1785,53 +1796,56 @@ namespace hfst {
return t->Start();
}
- StdVectorFst *
+ StdVectorFst *
TropicalWeightTransducer::remove_epsilons(StdVectorFst * t)
{
CHECK_EPSILON_CYCLES(t, "remove_epsilons");
return new StdVectorFst(RmEpsilonFst<StdArc>(*t)); }
- StdVectorFst *
+ StdVectorFst *
TropicalWeightTransducer::prune(StdVectorFst * t)
- { StdVectorFst * retval = new StdVectorFst();
+ { StdVectorFst * retval = new StdVectorFst();
fst::Prune(*t, retval, TropicalWeight::One());
return retval;
}
- StdVectorFst *
+ StdVectorFst *
TropicalWeightTransducer::n_best(StdVectorFst * t, unsigned int n)
- {
+ {
CHECK_EPSILON_CYCLES(t, "n_best");
- StdVectorFst * n_best_fst = new StdVectorFst();
+ StdVectorFst * n_best_fst = new StdVectorFst();
StdVectorFst * scaled = t->Copy();
RmEpsilon(scaled);
- float w = get_smallest_weight(scaled);
- if (w < 0)
- {
- add_to_weights(scaled, -w);
+ float w = get_smallest_weight(scaled);
+ if (w < 0)
+ {
+ add_to_weights(scaled, -w);
}
- try
+ try
{
fst::ShortestPath(*scaled,n_best_fst,(size_t)n);
}
catch (const std::bad_alloc & e)
{
+ (void)e;
+ delete n_best_fst; delete scaled;
HFST_THROW_MESSAGE(HfstFatalException, "TropicalWeightTransducer::nbest runs out of memory");
}
RmEpsilon(n_best_fst);
- if (w < 0)
- {
- add_to_weights(n_best_fst, w);
+ if (w < 0)
+ {
+ add_to_weights(n_best_fst, w);
}
+ delete scaled;
return n_best_fst;
}
- StdVectorFst *
+ StdVectorFst *
TropicalWeightTransducer::repeat_star(StdVectorFst * t)
{ return new StdVectorFst(ClosureFst<StdArc>(*t,CLOSURE_STAR)); }
- StdVectorFst *
+ StdVectorFst *
TropicalWeightTransducer::repeat_plus(StdVectorFst * t)
{ return new StdVectorFst(ClosureFst<StdArc>(*t,CLOSURE_PLUS)); }
@@ -1867,7 +1881,7 @@ namespace hfst {
return repetition;
}
- StdVectorFst *
+ StdVectorFst *
TropicalWeightTransducer::optionalize(StdVectorFst * t)
{
StdVectorFst * eps = create_epsilon_transducer();
@@ -1878,7 +1892,7 @@ namespace hfst {
}
- StdVectorFst *
+ StdVectorFst *
TropicalWeightTransducer::invert(StdVectorFst * t)
{
StdVectorFst * inverse = copy(t);
@@ -1888,7 +1902,7 @@ namespace hfst {
}
/* Makes valgrind angry... */
- StdVectorFst *
+ StdVectorFst *
TropicalWeightTransducer::reverse(StdVectorFst * t)
{
StdVectorFst * reversed = new StdVectorFst;
@@ -1901,7 +1915,7 @@ namespace hfst {
(StdVectorFst * t)
{ StdVectorFst * proj = new StdVectorFst(ProjectFst<StdArc>
(*t,PROJECT_INPUT));
- // substitute unknown with identity
+ // substitute unknown with identity
StdVectorFst * retval = substitute(proj, 1, 2);
delete proj;
retval->SetInputSymbols(t->InputSymbols());
@@ -1912,7 +1926,7 @@ namespace hfst {
(StdVectorFst * t)
{ StdVectorFst * proj = new StdVectorFst(ProjectFst<StdArc>
(*t,PROJECT_OUTPUT));
- // substitute unknown with identity
+ // substitute unknown with identity
StdVectorFst * retval = substitute(proj, 1, 2);
delete proj;
retval->SetInputSymbols(t->InputSymbols());
@@ -1927,11 +1941,11 @@ namespace hfst {
{
SymbolTable * st = t->InputSymbols()->Copy();
assert(st != NULL);
- for (fst::StateIterator<fst::StdFst> siter(*t);
+ for (fst::StateIterator<fst::StdFst> siter(*t);
!siter.Done(); siter.Next()) {
StateId state_id = siter.Value();
- t->AddArc(state_id,
- fst::StdArc(st->AddSymbol(symbol_pair.first),
+ t->AddArc(state_id,
+ fst::StdArc(st->AddSymbol(symbol_pair.first),
st->AddSymbol(symbol_pair.second), 0, state_id));
}
t->SetInputSymbols(st);
@@ -1949,13 +1963,13 @@ namespace hfst {
}
void TropicalWeightTransducer::set_symbol_table
- (StdVectorFst * t,
+ (StdVectorFst * t,
std::vector<std::pair<unsigned short, std::string> > symbol_mappings)
{
SymbolTable st = create_symbol_table("");
for (unsigned int i=0; i<symbol_mappings.size(); i++)
{
- st.AddSymbol(symbol_mappings[i].second,
+ st.AddSymbol(symbol_mappings[i].second,
symbol_mappings[i].first);
}
t->SetInputSymbols(&st);
@@ -1970,17 +1984,17 @@ namespace hfst {
EncodeMapper<StdArc> encode_mapper(0x0001,ENCODE);
EncodeFst<StdArc> enc(*t,&encode_mapper);
- StdArc old_pair_code =
+ StdArc old_pair_code =
encode_mapper(StdArc(old_key_pair.first,old_key_pair.second,0,0));
StdArc new_pair_code =
encode_mapper(StdArc(new_key_pair.first,new_key_pair.second,0,0));
- // First cast up, then cast down...
+ // First cast up, then cast down...
// For some reason dynamic_cast<StdVectorFst*>
- // doesn't work although both EncodeFst<StdArc> and StdVectorFst
- // extend Fst<StdArc>.
+ // doesn't work although both EncodeFst<StdArc> and StdVectorFst
+ // extend Fst<StdArc>.
// reinterpret_cast worked, but that is apparently unsafe...
- StdVectorFst * subst =
+ StdVectorFst * subst =
substitute(static_cast<StdVectorFst*>(static_cast<Fst<StdArc>*>(&enc)),
static_cast<unsigned int>(old_pair_code.ilabel),
static_cast<unsigned int>(new_pair_code.ilabel));
@@ -1998,21 +2012,21 @@ namespace hfst {
fst::StdVectorFst * tc = t->Copy();
fst::SymbolTable * st = tc->InputSymbols()->Copy();
assert(st != NULL);
- for (fst::StateIterator<fst::StdVectorFst> siter(*tc);
- ! siter.Done(); siter.Next())
+ for (fst::StateIterator<fst::StdVectorFst> siter(*tc);
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
- for (fst::MutableArcIterator<StdVectorFst> aiter(tc,s);
+ for (fst::MutableArcIterator<StdVectorFst> aiter(tc,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
- if ( strcmp( st->Find(arc.ilabel).c_str(),
+ if ( strcmp( st->Find(arc.ilabel).c_str(),
old_symbol_pair.first.c_str() ) == 0 &&
- strcmp( st->Find(arc.olabel).c_str(),
+ strcmp( st->Find(arc.olabel).c_str(),
old_symbol_pair.second.c_str() ) == 0 )
{
bool first_substitution=true;
- for (StringPairSet::iterator it = new_symbol_pair_set.begin();
+ for (StringPairSet::iterator it = new_symbol_pair_set.begin();
it != new_symbol_pair_set.end(); it++)
{
if (first_substitution) {
@@ -2021,12 +2035,12 @@ namespace hfst {
new_arc.olabel = st->AddSymbol(it->second);
new_arc.weight = arc.weight.Value();
new_arc.nextstate = arc.nextstate;
- aiter.SetValue(new_arc);
+ aiter.SetValue(new_arc);
first_substitution=false; }
else
- tc->AddArc(s, StdArc(st->AddSymbol(it->first),
- st->AddSymbol(it->second),
- arc.weight.Value(),
+ tc->AddArc(s, StdArc(st->AddSymbol(it->first),
+ st->AddSymbol(it->second),
+ arc.weight.Value(),
arc.nextstate));
}
}
@@ -2043,7 +2057,7 @@ namespace hfst {
{
assert(t->InputSymbols() != NULL);
SymbolTable * st = t->InputSymbols()->Copy();
- StdVectorFst * retval
+ StdVectorFst * retval
= substitute(t, st->AddSymbol(old_symbol), st->AddSymbol(new_symbol));
retval->SetInputSymbols(st);
delete st;
@@ -2089,18 +2103,18 @@ namespace hfst {
fst::StdArc arc = it.Value();
// find arcs that must be replaced
- if ( arc.ilabel == st->AddSymbol(old_symbol_pair.first) &&
- arc.olabel == st->AddSymbol(old_symbol_pair.second) )
+ if ( arc.ilabel == st->AddSymbol(old_symbol_pair.first) &&
+ arc.olabel == st->AddSymbol(old_symbol_pair.second) )
{
StateId destination_state = arc.nextstate;
StateId start_state = t->AddState();
- // change the label of the arc to epsilon and point
+ // change the label of the arc to epsilon and point
// the arc to a new state
arc.ilabel = 0;
arc.olabel = 0;
- arc.nextstate = start_state;
+ arc.nextstate = start_state;
// weight remains the same
it.SetValue(arc);
@@ -2112,7 +2126,7 @@ namespace hfst {
// go through all states and arcs in replace transducer tr
- for (fst::StateIterator<fst::StdFst> siter(*transducer);
+ for (fst::StateIterator<fst::StdFst> siter(*transducer);
!siter.Done(); siter.Next()) {
StateId tr_state_id = siter.Value();
@@ -2124,28 +2138,28 @@ namespace hfst {
t->AddArc( tr_state_id + start_state,
fst::StdArc( 0,
0,
- // final weight is copied to
+ // final weight is copied to
// the epsilon transition
- transducer->Final(tr_state_id),
+ transducer->Final(tr_state_id),
destination_state
)
- );
+ );
for (fst::ArcIterator<fst::StdFst> aiter(*transducer, tr_state_id);
!aiter.Done(); aiter.Next()) {
const fst::StdArc &tr_arc = aiter.Value();
- // adding arc from state 'tr_state_id+start_state'
+ // adding arc from state 'tr_state_id+start_state'
// to state 'tr_arc.nextstate'
// copy arcs from tr to t
- t->AddArc( tr_state_id + start_state,
- fst::StdArc( tr_arc.ilabel,
- tr_arc.olabel,
- // weight remains the same
- tr_arc.weight,
- tr_arc.nextstate + start_state
- )
+ t->AddArc( tr_state_id + start_state,
+ fst::StdArc( tr_arc.ilabel,
+ tr_arc.olabel,
+ // weight remains the same
+ tr_arc.weight,
+ tr_arc.nextstate + start_state
+ )
);
}
@@ -2175,18 +2189,18 @@ namespace hfst {
fst::StdArc arc = it.Value();
// find arcs that must be replaced
- if ( arc.ilabel == (int)old_number_pair.first &&
- arc.olabel == (int)old_number_pair.second )
+ if ( arc.ilabel == (int)old_number_pair.first &&
+ arc.olabel == (int)old_number_pair.second )
{
StateId destination_state = arc.nextstate;
StateId start_state = t->AddState();
- // change the label of the arc to epsilon and point
+ // change the label of the arc to epsilon and point
// the arc to a new state
arc.ilabel = 0;
arc.olabel = 0;
- arc.nextstate = start_state;
+ arc.nextstate = start_state;
// weight remains the same
it.SetValue(arc);
@@ -2198,7 +2212,7 @@ namespace hfst {
// go through all states and arcs in replace transducer tr
- for (fst::StateIterator<fst::StdFst> siter(*transducer);
+ for (fst::StateIterator<fst::StdFst> siter(*transducer);
!siter.Done(); siter.Next()) {
StateId tr_state_id = siter.Value();
@@ -2210,28 +2224,28 @@ namespace hfst {
t->AddArc( tr_state_id + start_state,
fst::StdArc( 0,
0,
- // final weight is copied to
+ // final weight is copied to
// the epsilon transition
- transducer->Final(tr_state_id),
+ transducer->Final(tr_state_id),
destination_state
)
- );
+ );
for (fst::ArcIterator<fst::StdFst> aiter(*transducer, tr_state_id);
!aiter.Done(); aiter.Next()) {
const fst::StdArc &tr_arc = aiter.Value();
- // adding arc from state 'tr_state_id+start_state'
+ // adding arc from state 'tr_state_id+start_state'
// to state 'tr_arc.nextstate'
// copy arcs from tr to t
- t->AddArc( tr_state_id + start_state,
- fst::StdArc( tr_arc.ilabel,
- tr_arc.olabel,
- // weight remains the same
- tr_arc.weight,
- tr_arc.nextstate + start_state
- )
+ t->AddArc( tr_state_id + start_state,
+ fst::StdArc( tr_arc.ilabel,
+ tr_arc.olabel,
+ // weight remains the same
+ tr_arc.weight,
+ tr_arc.nextstate + start_state
+ )
);
}
@@ -2247,7 +2261,7 @@ namespace hfst {
StdVectorFst * t2)
{
StringSet foo;
- // a copy of t2 is created so that its symbol table check sum
+ // a copy of t2 is created so that its symbol table check sum
// is the same as t1's
// (else OpenFst complains about non-matching check sums... )
StdVectorFst * t2_ = expand_arcs(t2, foo, false);
@@ -2293,8 +2307,8 @@ namespace hfst {
StateId s = t->Start();
- for (StringPairVector::const_iterator it = spv.begin();
- it != spv.end(); it++)
+ for (StringPairVector::const_iterator it = spv.begin();
+ it != spv.end(); it++)
{
unsigned int inumber = st->AddSymbol(it->first.c_str());
unsigned int onumber = st->AddSymbol(it->second.c_str());
@@ -2328,8 +2342,8 @@ namespace hfst {
{
StateId s = t->Start();
- for (NumberPairVector::const_iterator it = npv.begin();
- it != npv.end(); it++)
+ for (NumberPairVector::const_iterator it = npv.begin();
+ it != npv.end(); it++)
{
unsigned int inumber = it->first;
unsigned int onumber = it->second;
@@ -2416,11 +2430,11 @@ namespace hfst {
// Remove weights from t2, is this really needed?
StdVectorFst *t2_ = copy(t2);
- for (fst::StateIterator<StdVectorFst> siter(*t2_);
+ for (fst::StateIterator<StdVectorFst> siter(*t2_);
! siter.Done(); siter.Next())
{
StateId s = siter.Value();
- for (fst::MutableArcIterator<StdVectorFst> aiter(t2_,s);
+ for (fst::MutableArcIterator<StdVectorFst> aiter(t2_,s);
!aiter.Done(); aiter.Next())
{
const StdArc &arc = aiter.Value();
@@ -2429,7 +2443,7 @@ namespace hfst {
new_arc.olabel = arc.olabel;
new_arc.weight = 0;
new_arc.nextstate = arc.nextstate;
- aiter.SetValue(new_arc);
+ aiter.SetValue(new_arc);
}
if (t2_->Final(s) != fst::TropicalWeight::Zero())
{ t2_->SetFinal(s,0); }
@@ -2459,7 +2473,7 @@ namespace hfst {
t1->SetOutputSymbols(NULL);
t2->SetOutputSymbols(NULL);
- StdVectorFst *result = new StdVectorFst(subtract);
+ StdVectorFst *result = new StdVectorFst(subtract);
return result;
}
@@ -2467,7 +2481,7 @@ namespace hfst {
StdVectorFst * TropicalWeightTransducer::set_weight(StdVectorFst * t,float f)
{
StdVectorFst * t_copy = new StdVectorFst(*t);
- for (fst::StateIterator<StdVectorFst> iter(*t);
+ for (fst::StateIterator<StdVectorFst> iter(*t);
! iter.Done(); iter.Next())
{
if (t_copy->Final(iter.Value()) != fst::TropicalWeight::Zero())
@@ -2480,20 +2494,20 @@ namespace hfst {
// ----- TRIE FUNCTIONS BEGINS -----
int TropicalWeightTransducer::has_arc(fst::StdVectorFst &t,
- StdArc::StateId sourcestate,
- StdArc::Label ilabel,
+ StdArc::StateId sourcestate,
+ StdArc::Label ilabel,
StdArc::Label olabel)
{
for (fst::ArcIterator<StdVectorFst> aiter(t,sourcestate);
!aiter.Done();
aiter.Next())
{
- if ((aiter.Value().ilabel == ilabel) &&
+ if ((aiter.Value().ilabel == ilabel) &&
(aiter.Value().olabel == olabel))
{ return aiter.Position(); }
}
- return -1;
+ return -1;
}
fst::StdVectorFst * TropicalWeightTransducer::disjunct_as_tries(fst::StdVectorFst * t1,
@@ -2567,14 +2581,14 @@ namespace hfst {
static bool extract_paths
(StdVectorFst * t, StdArc::StateId s,
- std::map<StateId,unsigned short> all_visitations,
+ std::map<StateId,unsigned short> all_visitations,
std::map<StateId, unsigned short> path_visitations,
float weight_sum,
hfst::ExtractStringsCb& callback, int cycles,
- std::vector<hfst::FdState<int64> >* fd_state_stack,
- bool filter_fd,
+ std::vector<hfst::FdState<int64> >* fd_state_stack,
+ bool filter_fd,
StringPairVector &spv)
- {
+ {
if(cycles >= 0 && path_visitations[s] > cycles)
return true;
all_visitations[s]++;
@@ -2616,7 +2630,7 @@ namespace hfst {
bool added_fd_state = false;
if (fd_state_stack) {
- if(fd_state_stack->back().get_table().get_operation(arc.ilabel)
+ if(fd_state_stack->back().get_table().get_operation(arc.ilabel)
!= NULL) {
fd_state_stack->push_back(fd_state_stack->back());
if(fd_state_stack->back().apply_operation(arc.ilabel))
@@ -2628,17 +2642,17 @@ namespace hfst {
}
}
- /* Handle spv here. Special symbols (flags, epsilons)
+ /* Handle spv here. Special symbols (flags, epsilons)
are always inserted. */
std::string istring("");
std::string ostring("");
- if (!filter_fd ||
+ if (!filter_fd ||
fd_state_stack->back().get_table().get_operation(arc.ilabel)==NULL)
istring = t->InputSymbols()->Find(arc.ilabel);
- if (!filter_fd ||
+ if (!filter_fd ||
fd_state_stack->back().get_table().get_operation(arc.olabel)==NULL)
ostring = t->InputSymbols()->Find(arc.olabel);
@@ -2647,7 +2661,7 @@ namespace hfst {
res = extract_paths
(t, arc.nextstate, all_visitations, path_visitations,
- weight_sum+arc.weight.Value(),
+ weight_sum+arc.weight.Value(),
callback, cycles, fd_state_stack, filter_fd,
spv);
@@ -2672,14 +2686,14 @@ namespace hfst {
map<StateId, unsigned short> all_visitations;
map<StateId, unsigned short> path_visitations;
- std::vector<hfst::FdState<int64> >* fd_state_stack
+ std::vector<hfst::FdState<int64> >* fd_state_stack
= (fd==NULL) ? NULL : new std::vector<hfst::FdState<int64> >
(1, hfst::FdState<int64>(*fd));
StringPairVector spv;
hfst::implementations::extract_paths
(t,t->Start(),all_visitations,path_visitations,
- 0.0f,callback,cycles,fd_state_stack,filter_fd,
+ 0.0f,callback,cycles,fd_state_stack,filter_fd,
spv);
// add epsilon path, if needed
@@ -2688,6 +2702,8 @@ namespace hfst {
HfstTwoLevelPath epsilon_path(t->Final(t->Start()).Value(), empty_spv);
callback(epsilon_path, true /* final*/);
}
+ if (fd_state_stack != NULL)
+ { delete fd_state_stack; }
}
static bool is_minimal_and_empty(StdVectorFst *t)
@@ -2696,7 +2712,7 @@ namespace hfst {
if (start_state < 0)
return true;
- for (fst::ArcIterator<StdVectorFst> aiter(*t,start_state);
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,start_state);
!aiter.Done(); aiter.Next())
{
return false;
@@ -2731,7 +2747,7 @@ namespace hfst {
visited.reserve(num_states);
/* Whether the state is marked as broken, i.e. we cannot proceed from
- that state. These arrays are used for giving more probability for
+ that state. These arrays are used for giving more probability for
shorter paths if \a t is cyclic. */
std::vector<int> broken;
broken.reserve(num_states);
@@ -2747,7 +2763,7 @@ namespace hfst {
vector<fst::StdArc> t_transitions;
- for (fst::ArcIterator<StdVectorFst> aiter(*t,current_state);
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,current_state);
!aiter.Done(); aiter.Next())
{
t_transitions.push_back(aiter.Value());
@@ -2756,7 +2772,7 @@ namespace hfst {
/* If we cannot proceed, return the longest path so far. */
if (t_transitions.empty() || broken[current_state]) {
for (int i=(int)path.second.size()-1; i>=last_index; i--) {
- path.second.pop_back();
+ path.second.pop_back();
}
if (!is_epsilon_path_accepted && path.second.size() == 0)
throw "cannot extract random path";
@@ -2786,17 +2802,17 @@ namespace hfst {
throw "cannot extract random path";
return path;
} // or continue.
- last_index = (int)path.second.size();
- }
+ last_index = (int)path.second.size();
+ }
/* Give more probability for shorter paths. */
if ( broken[ t_target ] == 0 ) {
- if ( visited[ t_target ] == 1 )
+ if ( visited[ t_target ] == 1 )
if ( (rand() % 4) == 0 )
broken[ t_target ] = 1;
}
- if ( visited[ t_target ] == 1 ) {
+ if ( visited[ t_target ] == 1 ) {
if ( (rand() % 4) == 0 )
broken[ t_target ] = 1;
}
@@ -2804,7 +2820,7 @@ namespace hfst {
/* Proceed to the target state. */
current_state = t_target;
break;
- }
+ }
}
if (!is_epsilon_path_accepted && path.second.size() == 0)
throw "cannot extract random path";
@@ -2818,7 +2834,7 @@ namespace hfst {
while (max_times > 0)
{
- try
+ try
{
--max_times;
path = random_path_(t);
@@ -2848,7 +2864,7 @@ namespace hfst {
{
FlagDiacriticTable fdt;
StringSet alpha = get_alphabet(t);
- for (StringSet::const_iterator it = alpha.begin();
+ for (StringSet::const_iterator it = alpha.begin();
it != alpha.end(); it++)
{
fdt.insert_symbol(*it);
@@ -2885,16 +2901,16 @@ namespace hfst {
{
srand((unsigned int)(time(0)));
- while (max_num > 0)
+ while (max_num > 0)
{
/* Try to extract one path at most 5 times. */
HfstTwoLevelPath path;
- try
+ try
{
--max_num;
path = random_path(t, 5);
}
- catch (const char * msg)
+ catch (const char * msg)
{
if(strcmp("cannot extract random path", msg) == 0)
{
@@ -2914,7 +2930,7 @@ namespace hfst {
--i;
path = random_path(t, 5);
}
- catch (const char * msg) {} // keep on trying
+ catch (const char * msg) { (void)msg; } // keep on trying
}
/* Insert the path (another or the same). */
@@ -2929,7 +2945,7 @@ namespace hfst {
{
FdTable<int64>* table = new FdTable<int64>();
const fst::SymbolTable* symbols = t->InputSymbols();
- for(fst::SymbolTableIterator it=fst::SymbolTableIterator(*symbols);
+ for(fst::SymbolTableIterator it=fst::SymbolTableIterator(*symbols);
!it.Done(); it.Next())
{
if(FdOperation::is_diacritic(it.Symbol()))
@@ -2970,8 +2986,8 @@ namespace hfst {
output_stream.put(char(c));
}
- void TropicalWeightOutputStream::write_transducer(StdVectorFst * transducer)
- {
+ void TropicalWeightOutputStream::write_transducer(StdVectorFst * transducer)
+ {
if (!output_stream)
fprintf(stderr, "TropicalWeightOutputStream: ERROR: failbit set (1).\n");
/* When writing a transducer in the backend format,
@@ -2986,13 +3002,13 @@ namespace hfst {
output_st = new fst::SymbolTable(*(transducer->InputSymbols()));
transducer->SetOutputSymbols(output_st);
}
- transducer->Write(output_stream,FstWriteOptions());
+ transducer->Write(output_stream,FstWriteOptions());
if (output_st != NULL)
delete output_st;
}
- void TropicalWeightOutputStream::close(void)
+ void TropicalWeightOutputStream::close(void)
{
if (filename != string())
{ o_stream.close(); }
@@ -3007,7 +3023,7 @@ namespace hfst {
using namespace hfst::implementations;
-int main(int argc, char * argv[])
+int main(int argc, char * argv[])
{
std::cout << "Unit tests for " __FILE__ ":";
TropicalWeightTransducer ofst;
diff --git a/libhfst/src/implementations/TropicalWeightTransducer.h b/libhfst/src/implementations/TropicalWeightTransducer.h
index 894e8c3..2b346af 100644
--- a/libhfst/src/implementations/TropicalWeightTransducer.h
+++ b/libhfst/src/implementations/TropicalWeightTransducer.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _TROPICAL_WEIGHT_TRANSDUCER_H_
@@ -18,32 +18,67 @@
#include "../../../config.h"
#endif
-#ifdef _MSC_VER
-#include "back-ends/openfstwin/src/include/fst/fstlib.h"
-#else
-#include "back-ends/openfst/src/include/fst/fstlib.h"
-#endif // _MSC_VER
+//#ifdef _MSC_VER
+//#include "back-ends/openfstwin/src/include/fst/fstlib.h"
+//#else
+//#include "back-ends/openfst/src/include/fst/fstlib.h"
+//#endif // _MSC_VER
#include "HfstExtractStrings.h"
#include <cstdio>
#include <string>
#include <sstream>
-#include <iostream>
+#include <iosfwd>
+#include <fstream>
+
//#include "HfstAlphabet.h"
+#ifdef OPENFST_VERSION_1_5_4
+ #include "back-ends/openfst/src/include/fst/fst-decl.h"
+#else
+namespace fst
+{
+ template <class W> class TropicalWeightTpl;
+ typedef TropicalWeightTpl<float> TropicalWeight;
+
+ template <class W> class LogWeightTpl;
+ typedef LogWeightTpl<float> LogWeight;
+
+ template <class W> class ArcTpl;
+ typedef ArcTpl<TropicalWeight> StdArc;
+ typedef ArcTpl<LogWeight> LogArc;
+
+ template <class A> class VectorFst;
+ typedef VectorFst<StdArc> StdVectorFst;
+ typedef VectorFst<LogArc> LogFst;
+
+ template <class F> class StateIterator;
+ template <class F> class ArcIterator;
+
+ class SymbolTable;
+}
+#endif
+
+#ifdef _MSC_VER
+typedef __int64 int64;
+#else
+typedef int64_t int64;
+#endif // _MSC_VER
+
/** @file TropicalWeightTransducer.h
\brief Declarations of functions and datatypes that form a bridge between
HFST API and OpenFst's transducers with tropical weights. */
-namespace hfst {
+namespace hfst {
namespace implementations
{
using namespace fst;
;
- typedef StdArc::StateId StateId;
+ typedef unsigned int StateId;
+ //typedef StdArc::StateId StateId;
typedef std::vector<StdArc> StdArcVector;
- struct StdArcLessThan {
+ struct StdArcLessThan {
bool operator() (const StdArc &arc1,const StdArc &arc2) const; };
using std::ostream;
@@ -52,12 +87,12 @@ namespace implementations
void openfst_tropical_set_hopcroft(bool value);
- class TropicalWeightInputStream
+ class TropicalWeightInputStream
{
private:
std::string filename;
- ifstream i_stream;
- istream &input_stream;
+ std::ifstream i_stream;
+ std::istream &input_stream;
void skip_identifier_version_3_0(void);
void skip_hfst_header(void);
public:
@@ -77,19 +112,19 @@ namespace implementations
void stream_unget(char c);
static bool is_fst(FILE * f);
- static bool is_fst(istream &s);
+ static bool is_fst(std::istream &s);
};
- class TropicalWeightOutputStream
+ class TropicalWeightOutputStream
{
private:
std::string filename;
- ofstream o_stream;
- ostream &output_stream;
+ std::ofstream o_stream;
+ std::ostream &output_stream;
bool hfst_format;
//void write_3_0_library_header(std::ostream &out);
public:
- TropicalWeightOutputStream(bool hfst_format=true);
+ TropicalWeightOutputStream(bool hfst_format=true);
TropicalWeightOutputStream
(const std::string &filename, bool hfst_format=false);
void close(void);
@@ -97,24 +132,24 @@ namespace implementations
void write_transducer(StdVectorFst * transducer);
};
- class TropicalWeightTransitionIterator;
+ /*class TropicalWeightTransitionIterator;
typedef StateId TropicalWeightState;
- class TropicalWeightStateIterator
+ class TropicalWeightStateIterator
{
protected:
- StateIterator<StdVectorFst> * iterator;
+ fst::StateIterator<fst::StdVectorFst> * iterator;
public:
TropicalWeightStateIterator(StdVectorFst * t);
~TropicalWeightStateIterator(void);
void next(void);
bool done(void);
TropicalWeightState value(void);
- };
+ };*/
- class TropicalWeightTransition
+ /* class TropicalWeightTransition
{
protected:
StdArc arc;
@@ -132,15 +167,15 @@ namespace implementations
class TropicalWeightTransitionIterator
{
protected:
- ArcIterator<StdVectorFst> * arc_iterator;
- StdVectorFst * t;
+ fst::ArcIterator<fst::StdVectorFst> * arc_iterator;
+ fst::StdVectorFst * t;
public:
TropicalWeightTransitionIterator(StdVectorFst * t, StateId state);
~TropicalWeightTransitionIterator(void);
void next(void);
bool done(void);
TropicalWeightTransition value(void);
- };
+ };*/
class TropicalWeightTransducer
@@ -148,6 +183,7 @@ namespace implementations
public:
static StdVectorFst * create_empty_transducer(void);
static StdVectorFst * create_epsilon_transducer(void);
+ static void delete_transducer(StdVectorFst * t);
// string versions
static StdVectorFst * define_transducer(const std::string &symbol);
@@ -188,7 +224,7 @@ namespace implementations
static StdVectorFst * extract_output_language(StdVectorFst * t);
static void extract_paths
(StdVectorFst * t, hfst::ExtractStringsCb& callback,
- int cycles=-1, FdTable<int64>* fd=NULL, bool filter_fd=false
+ int cycles=-1, FdTable<int64>* fd=NULL, bool filter_fd=false
/*bool include_spv=false*/);
static void extract_random_paths
@@ -283,7 +319,7 @@ namespace implementations
static void get_first_input_symbols
(StdVectorFst *t, StateId s, std::set<StateId> & visited_states, StringSet & symbols);
static StringSet get_first_input_symbols(StdVectorFst *t);
- static unsigned int get_symbol_number(StdVectorFst *t,
+ static unsigned int get_symbol_number(StdVectorFst *t,
const std::string &symbol);
static unsigned int get_biggest_symbol_number(StdVectorFst *t);
static StringVector get_symbol_vector(StdVectorFst *t);
@@ -291,9 +327,9 @@ namespace implementations
static NumberNumberMap create_mapping
(StdVectorFst * t1, StdVectorFst * t2);
static void recode_symbol_numbers
- (StdVectorFst * t, hfst::NumberNumberMap &km);
+ (StdVectorFst * t, hfst::NumberNumberMap &km);
static StdVectorFst * expand_arcs
- (StdVectorFst * t, hfst::StringSet &unknown,
+ (StdVectorFst * t, hfst::StringSet &unknown,
bool unknown_symbols_in_use);
#ifdef FOO
@@ -308,7 +344,7 @@ namespace implementations
// for HFST version 2 transducer handling
static void set_symbol_table
- (StdVectorFst * t,
+ (StdVectorFst * t,
std::vector<std::pair<unsigned short, std::string> > symbol_mappings);
static void set_warning_stream(std::ostream * os);
@@ -317,20 +353,20 @@ namespace implementations
private:
static fst::SymbolTable create_symbol_table(std::string name);
static void initialize_symbol_tables(StdVectorFst *t);
- static void remove_symbol_table(StdVectorFst *t);
+ static void remove_symbol_table(StdVectorFst *t);
static std::ostream * warning_stream;
- /* Maps state numbers in AT&T text format to state ids used by
+ /* Maps state numbers in AT&T text format to state ids used by
OpenFst transducers. */
typedef std::map<int, StateId> StateMap;
- static StateId add_and_map_state(StdVectorFst *t, int state_number,
+ static StateId add_and_map_state(StdVectorFst *t, int state_number,
StateMap &state_map);
static int has_arc(StdVectorFst &t,
- StdArc::StateId sourcestate,
- StdArc::Label ilabel,
- StdArc::Label olabel);
+ /*StdArc::StateId*/ int sourcestate,
+ /*StdArc::Label*/ int ilabel,
+ /*StdArc::Label*/ int olabel);
static void disjunct_as_tries(fst::StdVectorFst &t1,
StateId t1_state,
const fst::StdVectorFst * t2,
@@ -350,7 +386,7 @@ namespace implementations
static float is_final(StdVectorFst *t, StateId s);
static StateId get_initial_state(StdVectorFst *t);
static void represent_empty_transducer_as_having_one_state
- (StdVectorFst *t);
+ (StdVectorFst *t);
};
diff --git a/libhfst/src/implementations/XfsmTransducer.cc b/libhfst/src/implementations/XfsmTransducer.cc
index 5331be0..9875852 100644
--- a/libhfst/src/implementations/XfsmTransducer.cc
+++ b/libhfst/src/implementations/XfsmTransducer.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "XfsmTransducer.h"
@@ -42,9 +42,9 @@ namespace hfst { namespace implementations {
"XfsmInputStream::XfsmInputStream(\"\") not supported");
}
else {
- FILE * input_file = fopen(filename.c_str(),"r");
+ FILE * input_file = hfst::hfst_fopen(filename.c_str(),"r");
if (input_file == NULL)
- {
+ {
HFST_THROW(StreamNotReadableException); }
fclose(input_file);
@@ -108,7 +108,7 @@ namespace hfst { namespace implementations {
filename(std::string(str)), net_list(NULL)
{
if (filename != std::string()) {
- FILE * ofile = fopen(filename.c_str(), "wb");
+ FILE * ofile = hfst::hfst_fopen(filename.c_str(), "wb");
if (ofile == NULL) {
HFST_THROW(StreamNotReadableException);
}
@@ -141,9 +141,9 @@ namespace hfst { namespace implementations {
}
/* Writing is delayed and done when flush() is called. */
- void XfsmOutputStream::write_transducer(NETptr transducer)
+ void XfsmOutputStream::write_transducer(NETptr transducer)
{
- if (net_list == NULL)
+ if (net_list == NULL)
{
net_list = make_nv(0);
}
@@ -247,12 +247,12 @@ namespace hfst { namespace implementations {
return result;
}
- NETptr XfsmTransducer::create_empty_transducer(void)
+ NETptr XfsmTransducer::create_empty_transducer(void)
{
return null_net();
}
- NETptr XfsmTransducer::create_epsilon_transducer(void)
+ NETptr XfsmTransducer::create_epsilon_transducer(void)
{
NETptr result = null_net();
return optional_net(result, DONT_KEEP);
@@ -313,7 +313,7 @@ namespace hfst { namespace implementations {
return read_regex(regex.c_str());
}
- NETptr XfsmTransducer::copy(NETptr t)
+ NETptr XfsmTransducer::copy(NETptr t)
{
return copy_net(t);
}
@@ -333,9 +333,9 @@ namespace hfst { namespace implementations {
NETptr XfsmTransducer::minimize(NETptr t)
{
- if (minimize_even_if_already_minimal_)
+ if (minimize_even_if_already_minimal_)
{
- NET_minimized(t) = 0;
+ NET_minimized(t) = 0;
}
if (minimize_net(t) == 1)
{
@@ -452,57 +452,57 @@ namespace hfst { namespace implementations {
return retval;
}
- NETptr XfsmTransducer::repeat_star(NETptr t)
+ NETptr XfsmTransducer::repeat_star(NETptr t)
{
return repeat_net(t, 0, -1, DONT_KEEP);
}
- NETptr XfsmTransducer::repeat_plus(NETptr t)
+ NETptr XfsmTransducer::repeat_plus(NETptr t)
{
return repeat_net(t, 1, -1, DONT_KEEP);
}
- NETptr XfsmTransducer::repeat_n(NETptr t, unsigned int n)
+ NETptr XfsmTransducer::repeat_n(NETptr t, unsigned int n)
{
return repeat_net(t, n, n, DONT_KEEP);
}
- NETptr XfsmTransducer::repeat_le_n(NETptr t, unsigned int n)
+ NETptr XfsmTransducer::repeat_le_n(NETptr t, unsigned int n)
{
return repeat_net(t, 0, n, DONT_KEEP);
}
- NETptr XfsmTransducer::repeat_n_plus(NETptr t, unsigned int n)
+ NETptr XfsmTransducer::repeat_n_plus(NETptr t, unsigned int n)
{
return repeat_net(t, n, -1, DONT_KEEP);
}
- NETptr XfsmTransducer::repeat_n_to_k(NETptr t, unsigned int n, unsigned int k)
+ NETptr XfsmTransducer::repeat_n_to_k(NETptr t, unsigned int n, unsigned int k)
{
return repeat_net(t, n, k, DONT_KEEP);
}
- NETptr XfsmTransducer::optionalize(NETptr t)
+ NETptr XfsmTransducer::optionalize(NETptr t)
{
return optional_net(t, DONT_KEEP);
}
- NETptr XfsmTransducer::invert(NETptr t)
+ NETptr XfsmTransducer::invert(NETptr t)
{
return invert_net(t, DONT_KEEP);
}
- NETptr XfsmTransducer::reverse(NETptr t)
+ NETptr XfsmTransducer::reverse(NETptr t)
{
return reverse_net(t, DONT_KEEP);
}
- NETptr XfsmTransducer::extract_input_language(NETptr t)
+ NETptr XfsmTransducer::extract_input_language(NETptr t)
{
return upper_side_net(t, DONT_KEEP);
}
- NETptr XfsmTransducer::extract_output_language(NETptr t)
+ NETptr XfsmTransducer::extract_output_language(NETptr t)
{
return lower_side_net(t, DONT_KEEP);
}
@@ -543,7 +543,7 @@ namespace hfst { namespace implementations {
#include <iostream>
using namespace hfst::implementations;
-int main(int argc, char * argv[])
+int main(int argc, char * argv[])
{
std::cout << "Unit tests for " __FILE__ ":";
std::cout << std::endl << "ok" << std::endl;
diff --git a/libhfst/src/implementations/XfsmTransducer.h b/libhfst/src/implementations/XfsmTransducer.h
index 08adc0b..c87683d 100644
--- a/libhfst/src/implementations/XfsmTransducer.h
+++ b/libhfst/src/implementations/XfsmTransducer.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _XFSM_TRANSDUCER_H_
@@ -23,7 +23,7 @@
#include <cstdio>
#include <string>
#include <sstream>
-#include <iostream>
+#include <iosfwd>
#include "../FormatSpecifiers.h"
@@ -41,7 +41,7 @@ namespace hfst {
private:
/* The name of the file where transducers will be read from. */
std::string filename;
- /* A list of transducers that will return transducers one by one when
+ /* A list of transducers that will return transducers one by one when
read_transducer() is called. */
NVptr net_list;
/* The size of net_list. */
@@ -86,7 +86,7 @@ namespace hfst {
/* Does nothing. */
void close();
/* Delayed write, stores a copy of \a transducer in net_list. */
- void write_transducer(NETptr transducer);
+ void write_transducer(NETptr transducer);
};
class XfsmTransducer {
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectFst.cc b/libhfst/src/implementations/compose_intersect/ComposeIntersectFst.cc
index 69accf9..81aacd4 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectFst.cc
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectFst.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "ComposeIntersectFst.h"
@@ -19,8 +19,8 @@ namespace hfst
const HfstState ComposeIntersectFst::START = 0;
- template<> ComposeIntersectFst::CompareTransitions
- ComposeIntersectFst::TransitionSet::comparator =
+ template<> ComposeIntersectFst::CompareTransitions
+ ComposeIntersectFst::TransitionSet::comparator =
ComposeIntersectFst::CompareTransitions();
ComposeIntersectFst::Transition::Transition(const HfstBasicTransition &t):
@@ -36,7 +36,7 @@ namespace hfst
}
ComposeIntersectFst::Transition::Transition
- (HfstState target,size_t ilabel,size_t olabel, float weight):
+ (HfstState target,size_t ilabel,size_t olabel, float weight):
ilabel(ilabel),
olabel(olabel),
weight(weight),
@@ -46,7 +46,7 @@ namespace hfst
bool ComposeIntersectFst::Transition::operator==
(const ComposeIntersectFst::Transition &another) const
{
- return
+ return
ilabel == another.ilabel &&
olabel == another.olabel &&
weight == another.weight &&
@@ -101,24 +101,24 @@ namespace hfst
{ finality_vector.push_back
(std::numeric_limits<float>::infinity()); }
++source_state;
- SymbolTransitionMap &symbol_transition_map =
+ SymbolTransitionMap &symbol_transition_map =
transition_map_vector.back();
bool identity_found = false;
- for (std::vector<HfstBasicTransition>::const_iterator jt =
+ for (std::vector<HfstBasicTransition>::const_iterator jt =
it->begin();
jt != it->end();
++jt)
- {
+ {
if (jt->get_input_symbol() == "@_IDENTITY_SYMBOL_@")
- {
+ {
identity_found = true;
identity_transition_vector.push_back(*jt);
}
else
{ symbol_transition_map
- [input_keys ?
+ [input_keys ?
HfstTropicalTransducerTransitionData::get_number
- (jt->get_input_symbol()) :
+ (jt->get_input_symbol()) :
HfstTropicalTransducerTransitionData::get_number
(jt->get_output_symbol())].
insert(*jt); }
@@ -138,7 +138,7 @@ namespace hfst
ComposeIntersectFst::~ComposeIntersectFst(void)
{}
- float ComposeIntersectFst::get_final_weight(HfstState s) const
+ float ComposeIntersectFst::get_final_weight(HfstState s) const
{
if (s >= transition_map_vector.size())
{ HFST_THROW(StateNotDefined); }
@@ -148,25 +148,25 @@ namespace hfst
size_t ComposeIntersectFst::get_symbol_number(const std::string &symbol)
{ return HfstTropicalTransducerTransitionData::get_number(symbol); }
- const ComposeIntersectFst::TransitionSet
+ const ComposeIntersectFst::TransitionSet
&ComposeIntersectFst::get_transitions
- (HfstState s,size_t symbol)
+ (HfstState s,size_t symbol)
{
if (s >= transition_map_vector.size())
{ HFST_THROW(StateNotDefined); }
- if (transition_map_vector.at(s).find(symbol) ==
+ if (transition_map_vector.at(s).find(symbol) ==
transition_map_vector.at(s).end())
- {
+ {
if (is_known_symbol(symbol) || ! has_identity_transition(s))
{ return transition_map_vector.at(s)[symbol] = TransitionSet(); }
else
- {
- Transition identity_transition =
+ {
+ Transition identity_transition =
get_identity_transition(s);
transition_map_vector.at(s)[symbol] = TransitionSet();
transition_map_vector.at(s)[symbol].insert
(Transition(identity_transition.target,symbol,symbol,
- identity_transition.weight));
+ identity_transition.weight));
return transition_map_vector.at(s)[symbol];
}
}
@@ -176,25 +176,25 @@ namespace hfst
bool ComposeIntersectFst::is_known_symbol(size_t symbol) const
{ return symbol_set.find(symbol) != symbol_set.end(); }
- ComposeIntersectFst::Transition
+ ComposeIntersectFst::Transition
ComposeIntersectFst::get_identity_transition
- (HfstState s)
+ (HfstState s)
{
if (s >= transition_map_vector.size())
{ HFST_THROW(StateNotDefined); }
- return identity_transition_vector.at(s);
+ return identity_transition_vector.at(s);
}
bool ComposeIntersectFst::has_identity_transition(HfstState s)
{
if (s >= transition_map_vector.size())
{ HFST_THROW(StateNotDefined); }
- return identity_transition_vector.at(s).ilabel
+ return identity_transition_vector.at(s).ilabel
== HfstTropicalTransducerTransitionData::get_number
- ("@_IDENTITY_SYMBOL_@");
+ ("@_IDENTITY_SYMBOL_@");
}
- const ComposeIntersectFst::SymbolSet
+ const ComposeIntersectFst::SymbolSet
&ComposeIntersectFst::get_symbols(void) const
{ return symbol_set; }
}
@@ -208,31 +208,31 @@ using namespace hfst::implementations;
using namespace hfst;
std::ostream &ComposeIntersectFst::print(std::ostream &out) const
{
-/* for (size_t i = 0; i < transition_map_vector.size(); ++i)
- {
+/* for (size_t i = 0; i < transition_map_vector.size(); ++i)
+ {
std::cout << "State " << i << ":" << std::endl;
std::cout << " Final weight: " << get_final_weight(i) << std::endl;
- if (identity_transition_vector.at(i).get_input_symbol() ==
+ if (identity_transition_vector.at(i).get_input_symbol() ==
"@_IDENTITY_SYMBOL_@")
- { std::cout << " Identity target:"
- << identity_transition_vector.at(i).get_target_state()
+ { std::cout << " Identity target:"
+ << identity_transition_vector.at(i).get_target_state()
<< std::endl; }
else
{ std::cout << " No identity transition" << std::endl; }
- for (SymbolTransitionMap::const_iterator it =
+ for (SymbolTransitionMap::const_iterator it =
transition_map_vector.at(i).begin();
it != transition_map_vector.at(i).end();
++it)
- {
+ {
std::cout << it->first << " transitions: " << std::endl;
for (TransitionSet::const_iterator jt = it->second.begin();
jt != it->second.end();
++jt)
{
- std::cout << it->first << "\t"
- << jt->get_target_state() << "\t"
- << jt->get_input_symbol() << "\t"
- << jt->get_output_symbol() << "\t"
+ std::cout << it->first << "\t"
+ << jt->get_target_state() << "\t"
+ << jt->get_input_symbol() << "\t"
+ << jt->get_output_symbol() << "\t"
<< jt->get_weight() << std::endl;
}
}
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectFst.h b/libhfst/src/implementations/compose_intersect/ComposeIntersectFst.h
index 8095ae9..42039d4 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectFst.h
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectFst.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
#ifndef COMPOSE_INTERSECT_FST_H
#define COMPOSE_INTERSECT_FST_H
@@ -13,7 +13,7 @@
#include "ComposeIntersectUtilities.h"
#include "../../HfstExceptionDefs.h"
-#include "../HfstTransitionGraph.h"
+#include "../HfstBasicTransducer.h"
HFST_EXCEPTION_CHILD_DECLARATION(StateNotDefined);
@@ -22,8 +22,8 @@ namespace hfst
namespace implementations
{
class ComposeIntersectFst
- {
- public:
+ {
+ public:
struct Transition
{
size_t ilabel;
@@ -38,12 +38,12 @@ namespace hfst
struct CompareTransitions
{
bool operator() (const Transition &transition1,
- const Transition &transition2) const;
+ const Transition &transition2) const;
};
typedef compose_intersect_utilities::SpaceSavingSet
- <Transition,CompareTransitions>
- TransitionSet;
+ <Transition,CompareTransitions>
+ TransitionSet;
typedef std::set<size_t> SymbolSet;
static const HfstState START; // = 0;
ComposeIntersectFst(const HfstBasicTransducer &, bool input_keys);
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.cc b/libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.cc
index 602f103..796c504 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.cc
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "ComposeIntersectLexicon.h"
@@ -25,7 +25,7 @@ namespace hfst
bool ComposeIntersectLexicon::is_flag_diacritic(size_t symbol)
{ return FdOperation::is_diacritic
- (HfstTropicalTransducerTransitionData::get_symbol(symbol)); }
+ (HfstTropicalTransducerTransitionData::get_symbol(hfst::size_t_to_uint(symbol))); }
void ComposeIntersectLexicon::clear_all_info(void)
{
@@ -74,12 +74,12 @@ namespace hfst
return compute_composition_result(rules);
}
- HfstState ComposeIntersectLexicon::get_state(const StatePair &p,
+ HfstState ComposeIntersectLexicon::get_state(const StatePair &p,
bool allow_lexicon_epsilons)
- {
+ {
if (state_pair_map.find(p) == state_pair_map.end())
- {
- return map_state_and_add_to_agenda(p, allow_lexicon_epsilons);
+ {
+ return map_state_and_add_to_agenda(p, allow_lexicon_epsilons);
}
return state_pair_map[p];
@@ -94,13 +94,13 @@ namespace hfst
float rules_weight = rules->get_final_weight(pair_vector[s].second);
if (lexicon_weight != std::numeric_limits<float>::infinity() &&
rules_weight != std::numeric_limits<float>::infinity())
- { result.set_final_weight(s,lexicon_weight+rules_weight); }
+ { result.set_final_weight(hfst::size_t_to_uint(s),lexicon_weight+rules_weight); }
}
}
HfstBasicTransducer &ComposeIntersectLexicon::compute_composition_result
(ComposeIntersectRule * rules)
- {
+ {
while (! agenda.empty())
{
HfstState s = agenda.front();
@@ -110,7 +110,7 @@ namespace hfst
}
set_final_state_weights(rules);
- return result;
+ return result;
}
ComposeIntersectLexicon::StatePair ComposeIntersectLexicon::get_pair
@@ -127,32 +127,32 @@ namespace hfst
{
StatePair p = get_pair(state);
- bool lexicon_eps_transition_found = false;
+ //bool lexicon_eps_transition_found = false;
- for (SymbolTransitionMap::const_iterator it =
+ for (SymbolTransitionMap::const_iterator it =
transition_map_vector[p.first].begin();
it != transition_map_vector[p.first].end();
++it)
- {
+ {
if (it->first == HfstTropicalTransducerTransitionData::get_number
("@_EPSILON_SYMBOL_@"))
- {
+ {
if (allow_lexicon_epsilons)
{
lexicon_skip_symbol_compose(it->second,p.second,state);
- lexicon_eps_transition_found = true;
+ //lexicon_eps_transition_found = true;
}
}
else if (is_flag_diacritic(it->first) &&
(! rules->known_symbol(it->first)))
- {
- lexicon_skip_symbol_compose(it->second,p.second,state);
- lexicon_eps_transition_found = true;
+ {
+ lexicon_skip_symbol_compose(it->second,p.second,state);
+ //lexicon_eps_transition_found = true;
}
else
- {
+ {
compose(it->second,rules->get_transitions
- (p.second,it->first),state);
+ (p.second,it->first),state);
}
}
@@ -169,10 +169,10 @@ namespace hfst
for (TransitionSet::const_iterator it = transitions.begin();
it != transitions.end();
++it)
- {
+ {
add_transition
(origin,it->ilabel,it->olabel,it->weight,
- get_state(StatePair(it->target,rule_state)));
+ get_state(StatePair(it->target,rule_state)));
}
}
@@ -182,10 +182,10 @@ namespace hfst
for (TransitionSet::const_iterator it = transitions.begin();
it != transitions.end();
++it)
- {
+ {
add_transition
(origin,it->ilabel,it->olabel,it->weight,
- get_state(StatePair(lex_state,it->target), false));
+ get_state(StatePair(lex_state,it->target), false));
}
}
@@ -205,7 +205,7 @@ namespace hfst
++jt)
{ add_transition
(origin,it->ilabel,jt->olabel,it->weight + jt->weight,
- get_state(StatePair(it->target,jt->target))); }
+ get_state(StatePair(it->target,jt->target))); }
}
}
@@ -216,8 +216,8 @@ namespace hfst
(origin,
HfstBasicTransition
(target,
- HfstTropicalTransducerTransitionData::get_symbol(input),
- HfstTropicalTransducerTransitionData::get_symbol(output),
+ HfstTropicalTransducerTransitionData::get_symbol(hfst::size_t_to_uint(input)),
+ HfstTropicalTransducerTransitionData::get_symbol(hfst::size_t_to_uint(output)),
weight)); }
}
@@ -272,7 +272,7 @@ int main(int argc, char * argv[])
HfstTransducer right_rule(universal);
right_rule.subtract(inv_right_rule).minimize();
- ComposeIntersectRule * ci_right_rule =
+ ComposeIntersectRule * ci_right_rule =
new ComposeIntersectRule(right_rule);
HfstTransducer left_rule_abstract_center_pair(x);
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.h b/libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.h
index cf24ee0..cac4205 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.h
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef COMPOSE_INTERSECT_LEXICON_H
#define COMPOSE_INTERSECT_LEXICON_H
@@ -49,7 +49,7 @@ namespace hfst
HfstState get_state(const StatePair &, bool allow_lexicon_epsilons=true);
StatePair get_pair(HfstState);
void clear_all_info(void);
- HfstState map_state_and_add_to_agenda(const StatePair &,
+ HfstState map_state_and_add_to_agenda(const StatePair &,
bool allow_lexicon_epsilons);
HfstBasicTransducer &compute_composition_result
(ComposeIntersectRule *);
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectRule.cc b/libhfst/src/implementations/compose_intersect/ComposeIntersectRule.cc
index 1c2e4f4..7cdaee8 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectRule.cc
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectRule.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "ComposeIntersectRule.h"
@@ -13,16 +13,16 @@
namespace hfst
{
namespace implementations
- {
+ {
ComposeIntersectRule::ComposeIntersectRule(const HfstBasicTransducer &t):
ComposeIntersectFst(t,true)
- { symbols = t.get_alphabet(); }
+ { symbols = t.get_alphabet(); }
ComposeIntersectRule::ComposeIntersectRule(void):
ComposeIntersectFst()
- {}
+ {}
bool ComposeIntersectRule::known_symbol(size_t symbol)
- { return
- symbols.count(HfstTropicalTransducerTransitionData::get_symbol(symbol))
+ { return
+ symbols.count(HfstTropicalTransducerTransitionData::get_symbol(hfst::size_t_to_uint(symbol)))
> 0; }
}
}
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectRule.h b/libhfst/src/implementations/compose_intersect/ComposeIntersectRule.h
index ac7ee96..22a24fd 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectRule.h
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectRule.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef COMPOSE_INTERSECT_RULE_H
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.cc b/libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.cc
index 3a156f6..2f99c07 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.cc
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "ComposeIntersectRulePair.h"
@@ -58,17 +58,17 @@ namespace hfst
bool ComposeIntersectRulePair::transitions_computed
(HfstState state,size_t symbol)
- { return state_transition_vector.at(state).find(symbol)
+ { return state_transition_vector.at(state).find(symbol)
!= state_transition_vector.at(state).end(); }
HfstState ComposeIntersectRulePair::get_state(const StatePair &p)
{
if (! has_pair(p))
- {
- pair_state_map[p] = state_pair_vector.size();
+ {
+ pair_state_map[p] = hfst::size_t_to_uint(state_pair_vector.size());
state_pair_vector.push_back(p);
state_transition_vector.push_back(SymbolTransitionMap());
- return state_pair_vector.size() - 1;
+ return hfst::size_t_to_uint(state_pair_vector.size() - 1);
}
return pair_state_map[p];
}
@@ -88,7 +88,7 @@ namespace hfst
const StatePair &state_pair = state_pair_vector[s];
return fst1->get_final_weight(state_pair.first) +
fst2->get_final_weight(state_pair.second);
- }
+ }
void ComposeIntersectRulePair::compute_transition_set
(HfstState state, size_t symbol)
@@ -96,11 +96,11 @@ namespace hfst
StatePair state_pair = state_pair_vector[state];
const ComposeIntersectRule::TransitionSet &fst1_transitions =
fst1->get_transitions(state_pair.first,symbol);
- ComposeIntersectRule::TransitionSet::const_iterator it =
+ ComposeIntersectRule::TransitionSet::const_iterator it =
fst1_transitions.begin();
const ComposeIntersectRule::TransitionSet &fst2_transitions =
fst2->get_transitions(state_pair.second,symbol);
- ComposeIntersectRule::TransitionSet::const_iterator jt =
+ ComposeIntersectRule::TransitionSet::const_iterator jt =
fst2_transitions.begin();
(void)state_transition_vector[state][symbol];
@@ -111,7 +111,7 @@ namespace hfst
{
size_t output = it->olabel;
HfstState target = get_state(StatePair(it->target,jt->target));
- float weight = it->weight + jt->weight;
+ float weight = it->weight + jt->weight;
add_transition(transitions,target,symbol,output,weight);
++it; ++jt;
}
@@ -135,15 +135,15 @@ using hfst::implementations::ComposeIntersectRule;
using hfst::implementations::ComposeIntersectRulePair;
std::ostream &ComposeIntersectRulePair::print(std::ostream &out)
-{
+{
HfstState s = ComposeIntersectRulePair::START;
while (1)
{
- for (ComposeIntersectRule::SymbolSet::const_iterator it =
+ for (ComposeIntersectRule::SymbolSet::const_iterator it =
ComposeIntersectRule::symbol_set.begin();
it != ComposeIntersectRule::symbol_set.end();
++it)
- {
+ {
const TransitionSet &transitions = get_transitions(s,*it);
for (TransitionSet::const_iterator jt = transitions.begin();
jt != transitions.end();
@@ -154,7 +154,7 @@ std::ostream &ComposeIntersectRulePair::print(std::ostream &out)
<< HfstTropicalTransducerTransitionData::get_symbol(jt->ilabel) << "\t"
<< HfstTropicalTransducerTransitionData::get_symbol(jt->olabel) << "\t"
<< jt->weight << std::endl;
- }
+ }
}
out << s << "\t" << get_final_weight(s) << std::endl;
++s;
@@ -169,6 +169,7 @@ std::ostream &operator<<
{ return p.print(out); }
#include <cassert>
+#include <sstream>
int main(int argc, char * argv[])
{
@@ -195,8 +196,12 @@ int main(int argc, char * argv[])
}
catch (const StateNotDefined &e)
{ std::cerr << e() << std::endl; } */ // FIXME get_transitions wants size_t as second argument
- std::cout << "Print:" << std::endl;
- std::cout << compose_intersect_rule_pair << std::endl;
+
+
+ std::string str;
+ std::stringstream sstr(str);
+ sstr << "Print:" << std::endl;
+ sstr << compose_intersect_rule_pair << std::endl;
std::cout << "ok" << std::endl;
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.h b/libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.h
index 7d23b51..4d958a0 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.h
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef COMPOSE_INTERSECT_RULE_PAIR_H
#define COMPOSE_INTERSECT_RULE_PAIR_H
@@ -43,7 +43,7 @@ namespace hfst
protected:
typedef std::pair<HfstState,HfstState> StatePair;
typedef std::vector<StatePair> StatePairVector;
- typedef std::map<StatePair,HfstState> PairStateMap;
+ typedef std::map<StatePair,HfstState> PairStateMap;
typedef std::map<size_t,TransitionSet> SymbolTransitionMap;
typedef std::vector<SymbolTransitionMap> StateTransitionVector;
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.cc b/libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.cc
index 6f93fa5..8b637d1 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.cc
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "ComposeIntersectUtilities.h"
@@ -13,8 +13,8 @@
struct CmpInt
{ bool operator() (int i, int j) const { return i < j; } };
-typedef
-hfst::implementations::compose_intersect_utilities::SpaceSavingSet<int,CmpInt>
+typedef
+hfst::implementations::compose_intersect_utilities::SpaceSavingSet<int,CmpInt>
IntSpaceSavingSet;
template<> CmpInt IntSpaceSavingSet::comparator = CmpInt();
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.h b/libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.h
index 4e7471d..fbf2d79 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.h
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef HEADER_COMPOSE_INTERSECT_UTILITIES_H
@@ -93,10 +93,10 @@ namespace hfst
XVector container_;
const_iterator get_least_upper_bound(const X &x) const
- {
+ {
const_iterator it = container_.begin();
for ( ; it != container_.end(); ++it)
- {
+ {
if (! comparator(*it,x))
{ break; }
}
@@ -104,10 +104,10 @@ namespace hfst
}
iterator get_least_upper_bound(const X &x)
- {
+ {
iterator it = container_.begin();
for ( ; it != container_.end(); ++it)
- {
+ {
if (! comparator(*it,x))
{ break; }
}
diff --git a/libhfst/src/implementations/optimized-lookup/convert.cc b/libhfst/src/implementations/optimized-lookup/convert.cc
index db008aa..d50437b 100644
--- a/libhfst/src/implementations/optimized-lookup/convert.cc
+++ b/libhfst/src/implementations/optimized-lookup/convert.cc
@@ -1,14 +1,30 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "convert.h"
+#ifdef _MSC_VER
+#include "back-ends/openfstwin/src/include/fst/fstlib.h"
+#else
+#include "back-ends/openfst/src/include/fst/fstlib.h"
+#endif // _MSC_VER
+
+#ifndef _MSC_VER
+namespace fst
+{
+ extern template class TropicalWeightTpl<float>;
+ extern template class ArcTpl<TropicalWeight>;
+ extern template class VectorFst<StdArc>;
+ extern template class ArcIterator<StdVectorFst>;
+}
+#endif // _MSC_VER
+
#ifndef MAIN_TEST
namespace hfst_ol {
@@ -86,7 +102,7 @@ void add_transitions_with(SymbolNumber symbol,
// target is simple (ie. should point directly to TA entry)
unsigned int target;
if (state_placeholders[it->target].is_simple()) {
- target = state_placeholders[it->target].first_transition +
+ target = state_placeholders[it->target].first_transition +
TRANSITION_TARGET_TABLE_START - 1;
} else {
target = state_placeholders[it->target].start_index;
@@ -145,7 +161,7 @@ StateIdNumber ConvertIdNumberMap::get_node_id(StateId n) const
{
StateIdsToIdNumbers::const_iterator i = node_to_id.find(n);
if(i == node_to_id.end())
- return NO_ID_NUMBER;
+ return NO_ID_NUMBER;
return i->second;
}
@@ -209,7 +225,7 @@ void ConvertTransducerAlphabet::populate_symbol_table(
if(!FdOperation::is_diacritic(ofst_symbol_table->Find(it->first)))
count_keys.insert(std::pair<unsigned int,int64>(it->second,it->first));
else
- count_keys.insert(std::pair<unsigned int,int64>(0,it->first));
+ count_keys.insert(std::pair<unsigned int,int64>(0,it->first));
}
symbol_table.push_back(ofst_symbol_table->Find((int64)0));
@@ -275,7 +291,7 @@ ConvertTransducerAlphabet::ConvertTransducerAlphabet(TransduceR* t):
OfstSymbolCountMap symbol_count_map;
SymbolSet all_symbol_set;
- get_symbol_info(symbol_count_map, all_symbol_set);
+ get_symbol_info(symbol_count_map, all_symbol_set);
populate_symbol_table(symbol_count_map, all_symbol_set);
set_maps();
@@ -583,7 +599,7 @@ void ConvertFstState::insert_transition_indices(
template<class T>
TransitionTableIndex ConvertFstState::append_transitions(
- TransducerTable<T>& transition_table,
+ TransducerTable<T>& transition_table,
TransitionTableIndex place) const
{
while(place < get_first_transition_index())
@@ -638,7 +654,7 @@ bool ConvertTransitionTableIndices::state_fits(SymbolNumberSet * input_symbols,
// The input symbols start after the finality indicator.
PlaceHolderVector::size_type input_symbol_start = index+1;
- // The node fits, if every one of its input symbols goes on
+ // The node fits, if every one of its input symbols goes on
// an EMPTY or EMPTY_START index.
for(SymbolNumberSet::iterator it=input_symbols->begin();
it!=input_symbols->end(); ++it)
@@ -711,7 +727,7 @@ PlaceHolderVector::size_type ConvertTransitionTableIndices::add_state(
#endif
bool final_state = state->is_final();
- SymbolNumberSet * state_input_symbols =
+ SymbolNumberSet * state_input_symbols =
state->get_input_symbols();
++lower_bound_test_count;
diff --git a/libhfst/src/implementations/optimized-lookup/convert.h b/libhfst/src/implementations/optimized-lookup/convert.h
index fd82edb..c5309a0 100644
--- a/libhfst/src/implementations/optimized-lookup/convert.h
+++ b/libhfst/src/implementations/optimized-lookup/convert.h
@@ -1,22 +1,23 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _HFST_OL_CONVERT_H_
#define _HFST_OL_CONVERT_H_
-#if HAVE_OPENFST
-#include "fst/fstlib.h"
-#endif // HAVE_OPENFST
-
#include "transducer.h"
#include "pmatch.h"
+#ifdef OPENFST_VERSION_1_5_4
+ #include "back-ends/openfst/src/include/fst/fst-decl.h"
+ namespace fst { template <class F> class ArcIterator; }
+#endif
+
namespace hfst_ol {
typedef std::map<hfst_ol::TransitionTableIndex,unsigned int>
@@ -57,19 +58,19 @@ struct StatePlaceholder {
state_number(state),
start_index(UINT_MAX),
first_transition(first),
- final(finality),
- final_weight(final_weight),
type(state == 0 ? nonsimple: empty),
- inputs(0)
+ inputs(0),
+ final(finality),
+ final_weight(final_weight)
{ }
StatePlaceholder ():
state_number(UINT_MAX),
start_index(UINT_MAX),
first_transition(UINT_MAX),
- final(false),
- final_weight(0.0),
type(empty),
- inputs(0)
+ inputs(0),
+ final(false),
+ final_weight(0.0)
{ }
bool is_simple(void) const
@@ -82,7 +83,7 @@ struct StatePlaceholder {
for(std::vector<std::vector<TransitionPlaceholder> >::const_iterator it
= transition_placeholders.begin();
it != transition_placeholders.end(); ++it) {
- count += it->size();
+ count += hfst::size_t_to_uint(it->size());
}
return count;
}
@@ -100,7 +101,7 @@ struct StatePlaceholder {
while (symbol_to_transition_placeholder_v.size() <= input) {
symbol_to_transition_placeholder_v.push_back(UINT_MAX);
}
- symbol_to_transition_placeholder_v[input] = transition_placeholders.size();
+ symbol_to_transition_placeholder_v[input] = hfst::size_t_to_uint(transition_placeholders.size());
transition_placeholders.push_back(std::vector<TransitionPlaceholder>());
++inputs;
if (type != nonsimple) {
@@ -158,7 +159,7 @@ struct StatePlaceholder {
// }
// } else {
if (input_present(0)) { // if there are epsilons
- offset = get_transition_placeholders(0).size();
+ offset = hfst::size_t_to_uint(get_transition_placeholders(0).size());
}
for(std::set<SymbolNumber>::iterator flag_it = flag_symbols.begin();
flag_it != flag_symbols.end(); ++flag_it) {
@@ -167,7 +168,7 @@ struct StatePlaceholder {
// Flags go to 0 (even if there's no epsilon)
return 0;
}
- offset += get_transition_placeholders(*flag_it).size();
+ offset += hfst::size_t_to_uint(get_transition_placeholders(*flag_it).size());
}
}
for(unsigned int i = 1; i < symbol_to_transition_placeholder_v.size(); ++i) {
@@ -179,7 +180,7 @@ struct StatePlaceholder {
if (symbol == i) {
return offset;
}
- offset += get_transition_placeholders(i).size();
+ offset += hfst::size_t_to_uint(get_transition_placeholders(i).size());
}
}
std::string message("error in conversion between optimized lookup "
@@ -211,7 +212,7 @@ struct IndexPlaceholders
while (position >= indices.size()) {
indices.push_back(NO_TABLE_INDEX);
}
- indices[position] = targets.size();
+ indices[position] = hfst::size_t_to_uint(targets.size());
targets.push_back(std::pair<unsigned int, SymbolNumber>(target, sym));
}
@@ -283,7 +284,7 @@ void add_transitions_with(SymbolNumber symbol,
std::set<SymbolNumber> & flag_symbols);
#if HAVE_OPENFST // Covers remainder of file
-typedef fst::StdArc::StateId StateId;
+typedef /*fst::StdArc::StateId*/ int StateId;
typedef fst::StdArc StdArc;
typedef fst::StdVectorFst TransduceR;
typedef fst::ArcIterator<TransduceR> ArcIterator;
@@ -301,7 +302,7 @@ struct transition_label
int64 output_symbol;
};
-struct compare_transition_labels
+struct compare_transition_labels
{
bool operator() ( const transition_label &l1,
const transition_label &l2) const
@@ -333,7 +334,7 @@ private:
void add_node(StateId n, TransduceR *tr);
/*
- Assign every node n in t a unique id number i. Assign the start node
+ Assign every node n in t a unique id number i. Assign the start node
t->root_node() id number 0. Set node_to_id[n] = i and
id_to_node[i] = n.
*/
@@ -402,7 +403,7 @@ private:
TransitionTableIndex table_index; // location in the transition table
public:
- /*
+ /*
Set the symbol numbers and set the indices of the states according
to ConvertIdNumberMap nodes_to_id_numbers.
*/
@@ -461,18 +462,18 @@ public:
struct ConvertTransitionCompare
{
bool operator() (const ConvertTransition * t1,
- const ConvertTransition * t2) const
- {
- return t1->operator<(*t2);
+ const ConvertTransition * t2) const
+ {
+ return t1->operator<(*t2);
}
};
struct ConvertTransitionIndexCompare
{
bool operator() (const ConvertTransitionIndex * i1,
- const ConvertTransitionIndex * i2) const
+ const ConvertTransitionIndex * i2) const
{
- return i1->operator<(*i2);
+ return i1->operator<(*i2);
}
};
@@ -526,9 +527,9 @@ public:
SymbolNumberSet * get_input_symbols(void) const;
SymbolNumber number_of_input_symbols(void) const
- { return transition_indices.size(); }
+ { return hfst::size_t_to_uint(transition_indices.size()); }
SymbolNumber number_of_transitions(void) const
- { return transitions.size(); }
+ { return hfst::size_t_to_uint(transitions.size()); }
bool is_final(void) const {return final;}
bool is_big_state(void) const
{
@@ -545,13 +546,13 @@ public:
void insert_transition_indices(TransducerTable<T>& index_table) const;
template<class T>
- TransitionTableIndex append_transitions(TransducerTable<T>& transition_table,
+ TransitionTableIndex append_transitions(TransducerTable<T>& transition_table,
TransitionTableIndex place) const;
};
typedef std::vector<ConvertFstState*> ConvertFstStateVector;
-class ConvertTransitionTableIndices
+class ConvertTransitionTableIndices
{
private:
PlaceHolderVector indices;
@@ -641,7 +642,7 @@ public:
Transducer* to_transducer() const;
// exposed to this module during the constructor
- static ConvertTransducer* constructing_transducer;
+ static ConvertTransducer* constructing_transducer;
};
#endif // HAVE_OPENFST
diff --git a/libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc b/libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc
index ad91c6e..e759db9 100644
--- a/libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc
+++ b/libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "./transducer.h"
@@ -42,7 +42,7 @@ bool TraversalState::operator<(const TraversalState & rhs) const
return false;
}
}
- return false;
+ return false;
}
void Transducer::find_loop_epsilon_transitions(
@@ -127,7 +127,7 @@ void Transducer::find_loop_index(SymbolNumber input,
{
find_loop_transitions(input,
input_pos,
- tables->get_index_target(i+input) -
+ tables->get_index_target(i+input) -
TRANSITION_TARGET_TABLE_START);
found_transition = true;
}
diff --git a/libhfst/src/implementations/optimized-lookup/ospell.cc b/libhfst/src/implementations/optimized-lookup/ospell.cc
index 8f76f5a..e81bb2f 100644
--- a/libhfst/src/implementations/optimized-lookup/ospell.cc
+++ b/libhfst/src/implementations/optimized-lookup/ospell.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "transducer.h"
diff --git a/libhfst/src/implementations/optimized-lookup/pmatch.cc b/libhfst/src/implementations/optimized-lookup/pmatch.cc
index f0c8ac2..8d65f25 100644
--- a/libhfst/src/implementations/optimized-lookup/pmatch.cc
+++ b/libhfst/src/implementations/optimized-lookup/pmatch.cc
@@ -1,20 +1,24 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "pmatch.h"
#include "hfst.h"
+using hfst::HfstTransducer;
+
namespace hfst_ol {
PmatchAlphabet::PmatchAlphabet(std::istream & inputstream,
- SymbolNumber symbol_count):
+ SymbolNumber symbol_count,
+ PmatchContainer * cont):
TransducerAlphabet(inputstream, symbol_count, false),
- special_symbols(SPECIALSYMBOL_NR_ITEMS, NO_SYMBOL_NUMBER) // SpecialSymbols enum
+ special_symbols(SPECIALSYMBOL_NR_ITEMS, NO_SYMBOL_NUMBER), // SpecialSymbols enum
+ container(cont)
{
symbol2lists = SymbolNumberVector(orig_symbol_count, NO_SYMBOL_NUMBER);
list2symbols = SymbolNumberVector(orig_symbol_count, NO_SYMBOL_NUMBER);
@@ -30,9 +34,11 @@ PmatchAlphabet::PmatchAlphabet(std::istream & inputstream,
}
}
-PmatchAlphabet::PmatchAlphabet(TransducerAlphabet const & a):
+PmatchAlphabet::PmatchAlphabet(TransducerAlphabet const & a,
+ PmatchContainer * cont):
TransducerAlphabet(a),
- special_symbols(SPECIALSYMBOL_NR_ITEMS, NO_SYMBOL_NUMBER) {
+ special_symbols(SPECIALSYMBOL_NR_ITEMS, NO_SYMBOL_NUMBER),
+ container(cont) {
symbol2lists = SymbolNumberVector(orig_symbol_count, NO_SYMBOL_NUMBER);
list2symbols = SymbolNumberVector(orig_symbol_count, NO_SYMBOL_NUMBER);
rtns = RtnVector(orig_symbol_count, NULL);
@@ -59,7 +65,7 @@ void PmatchAlphabet::add_symbol(const std::string & symbol)
printable_vector.push_back(true);
if (exclusionary_lists.size() != 0) {
// if there are exclusionary lists, they should all accept the new symbol
- symbol2lists[symbol_table.size()] = symbol_lists.size();
+ symbol2lists[symbol_table.size()] = hfst::size_t_to_ushort(symbol_lists.size());
symbol_lists.push_back(SymbolNumberVector(exclusionary_lists.begin(),
exclusionary_lists.end()));
#ifndef _MSC_VER
@@ -68,7 +74,7 @@ void PmatchAlphabet::add_symbol(const std::string & symbol)
}
#else
for (SymbolNumberVector::const_iterator exc = exclusionary_lists.begin(); exc != exclusionary_lists.end(); exc++) {
- symbol_list_members[list2symbols[*exc]].push_back(symbol_table.size());
+ symbol_list_members[list2symbols[*exc]].push_back(hfst::size_t_to_uint(symbol_table.size()));
}
#endif
}
@@ -114,6 +120,11 @@ void PmatchAlphabet::add_special_symbol(const std::string & str,
end_tag_map[symbol_number] = str.substr(
sizeof("@PMATCH_ENDTAG_") - 1,
str.size() - (sizeof("@PMATCH_ENDTAG_@") - 1));
+ } else if (is_like_arc(str)) {
+ // Fetch the part between @PMATCH_LIKE_ and @
+ words_like_map[symbol_number] = str.substr(
+ sizeof("@PMATCH_LIKE_") - 1,
+ str.size() - (sizeof("@PMATCH_LIKE_@") - 1));
} else if (is_insertion(str)) {
rtn_names[name_from_insertion(str)] = symbol_number;
} else if (is_guard(str)) {
@@ -164,14 +175,14 @@ void PmatchAlphabet::process_symbol_list(std::string str, SymbolNumber sym)
list_symbols.push_back(str_sym);
if (polarity == true) {
if (symbol2lists[str_sym] == NO_SYMBOL_NUMBER) {
- symbol2lists[str_sym] = symbol_lists.size();
+ symbol2lists[str_sym] = hfst::size_t_to_ushort(symbol_lists.size());
symbol_lists.push_back(SymbolNumberVector(1, sym));
} else {
symbol_lists[symbol2lists[str_sym]].push_back(sym);
}
}
}
- list2symbols[sym] = symbol_list_members.size();
+ list2symbols[sym] = hfst::size_t_to_ushort(symbol_list_members.size());
if (polarity == false) {
SymbolNumberVector excl_symbols;
exclusionary_lists.push_back(sym);
@@ -180,7 +191,7 @@ void PmatchAlphabet::process_symbol_list(std::string str, SymbolNumber sym)
find(list_symbols.begin(), list_symbols.end(), candidate_for_list) == list_symbols.end()) {
excl_symbols.push_back(candidate_for_list);
if (symbol2lists[candidate_for_list] == NO_SYMBOL_NUMBER) {
- symbol2lists[candidate_for_list] = symbol_lists.size();
+ symbol2lists[candidate_for_list] = hfst::size_t_to_ushort(symbol_lists.size());
symbol_lists.push_back(SymbolNumberVector(1, sym));
} else {
symbol_lists[symbol2lists[sym]].push_back(sym);
@@ -221,58 +232,13 @@ void PmatchAlphabet::count(SymbolNumber sym)
}
}
-PmatchContainer::PmatchContainer(std::istream & inputstream):
- verbose(false),
- locate_mode(false),
- profile_mode(false),
- single_codepoint_tokenization(false),
- recursion_depth_left(PMATCH_MAX_RECURSION_DEPTH),
- entry_stack()
+void PmatchContainer::collect_first_symbols(void)
{
- std::string transducer_name;
- transducer_name = parse_name_from_hfst3_header(inputstream);
- // the first transducer should be called eg. "TOP", this could be tested
- // for once more established
-
- TransducerHeader header(inputstream);
- alphabet = PmatchAlphabet(inputstream, header.symbol_count());
- orig_symbol_count = symbol_count = alphabet.get_orig_symbol_count();
- alphabet.extract_tags = locate_mode;
- line_number = 0;
- encoder = new Encoder(alphabet.get_symbol_table(), orig_symbol_count);
- toplevel = new hfst_ol::PmatchTransducer(
- inputstream,
- header.index_table_size(),
- header.target_table_size(),
- alphabet,
- this);
- while (inputstream.good()) {
- try {
- transducer_name = parse_name_from_hfst3_header(inputstream);
- } catch (TransducerHeaderException & e) {
- break;
- }
- header = TransducerHeader(inputstream);
- TransducerAlphabet dummy = TransducerAlphabet(
- inputstream, header.symbol_count());
- hfst_ol::PmatchTransducer * rtn =
- new hfst_ol::PmatchTransducer(inputstream,
- header.index_table_size(),
- header.target_table_size(),
- alphabet,
- this);
- if (!alphabet.has_rtn(transducer_name)) {
- alphabet.add_rtn(rtn, transducer_name);
- } else {
- delete rtn;
- }
- }
-
- toplevel->collect_possible_first_symbols();
-
- // Finally fetch the first symbols from any
+ // Fetch the first symbols from any
// first-position rtn arcs in TOP. If they are potential epsilon loops,
// clear out the set.
+ toplevel->collect_possible_first_symbols();
+
SymbolNumber max_input_sym = 0;
std::set<SymbolNumber> & possible_firsts = toplevel->possible_first_symbols;
for (std::set<SymbolNumber>::iterator it = possible_firsts.begin();
@@ -326,21 +292,84 @@ PmatchContainer::PmatchContainer(std::istream & inputstream):
}
}
}
-
+}
+
+PmatchContainer::PmatchContainer(std::istream & inputstream):
+ entry_stack(),
+ verbose(false),
+ locate_mode(false),
+ line_number(0),
+ profile_mode(false),
+ single_codepoint_tokenization(false)
+{
+ set_properties();
+ reset_recursion();
+ std::string transducer_name;
+ std::map<std::string, std::string> properties = parse_hfst3_header(inputstream);
+ if (properties.count("name") == 0) {
+ std::cerr << "pmatch: warning: TOP not defined in archive, using first as TOP\n";
+ transducer_name = "TOP";
+ } else {
+ transducer_name = properties["name"];
+ if (transducer_name != "TOP") {
+ std::cerr << "pmatch: warning: TOP not defined in archive, using first as TOP\n";
+ }
+ }
+ if (properties.count("type") == 0) {
+ std::cerr << "pmatch: warning: type information missing from archive\n";
+ } else {
+ if (properties["type"] != "HFST_OLW") {
+ std::cerr << "pmatch: warning: archive type isn't weighted optimized-lookup according to header\n";
+ }
+ }
+ set_properties(properties);
+ TransducerHeader header(inputstream);
+ alphabet = PmatchAlphabet(inputstream, header.symbol_count(), this);
+ orig_symbol_count = symbol_count = alphabet.get_orig_symbol_count();
+ encoder = new Encoder(alphabet.get_symbol_table(), orig_symbol_count);
+ toplevel = new hfst_ol::PmatchTransducer(
+ inputstream,
+ header.index_table_size(),
+ header.target_table_size(),
+ alphabet,
+ this);
+ while (inputstream.good()) {
+ try {
+ properties = parse_hfst3_header(inputstream);
+ transducer_name = properties["name"];
+ } catch (TransducerHeaderException & e) {
+ (void)e; break;
+ }
+ header = TransducerHeader(inputstream);
+ TransducerAlphabet dummy = TransducerAlphabet(
+ inputstream, header.symbol_count());
+ hfst_ol::PmatchTransducer * rtn =
+ new hfst_ol::PmatchTransducer(inputstream,
+ header.index_table_size(),
+ header.target_table_size(),
+ alphabet,
+ this);
+ if (!alphabet.has_rtn(transducer_name)) {
+ alphabet.add_rtn(rtn, transducer_name);
+ } else {
+ delete rtn;
+ }
+ }
+ collect_first_symbols();
}
PmatchContainer::PmatchContainer(Transducer * t):
+ entry_stack(),
verbose(false),
locate_mode(false),
profile_mode(false),
- single_codepoint_tokenization(false),
- recursion_depth_left(PMATCH_MAX_RECURSION_DEPTH),
- entry_stack()
+ single_codepoint_tokenization(false)
{
- TransducerHeader header = t->get_header();
- alphabet = PmatchAlphabet(t->get_alphabet());
+ set_properties();
+ reset_recursion();
+ //TransducerHeader header = t->get_header();
+ alphabet = PmatchAlphabet(t->get_alphabet(), this);
orig_symbol_count = symbol_count = alphabet.get_orig_symbol_count();
- alphabet.extract_tags = locate_mode;
line_number = 0;
encoder = new Encoder(alphabet.get_symbol_table(), orig_symbol_count);
TransducerTable<TransitionW> transitions = t->copy_transitionw_table();
@@ -350,64 +379,156 @@ PmatchContainer::PmatchContainer(Transducer * t):
indices.get_vector(),
alphabet,
this);
- toplevel->collect_possible_first_symbols();
+ collect_first_symbols();
+}
- // Finally fetch the first symbols from any
- // first-position rtn arcs in TOP. If they are potential epsilon loops,
- // clear out the set.
- SymbolNumber max_input_sym = 0;
- std::set<SymbolNumber> & possible_firsts = toplevel->possible_first_symbols;
- for (std::set<SymbolNumber>::iterator it = possible_firsts.begin();
- it != possible_firsts.end(); ++it) {
- if (*it > max_input_sym) { max_input_sym = *it; }
- if (alphabet.has_rtn(*it)) {
- if (alphabet.get_rtn(*it) == toplevel) {
- possible_firsts.clear();
- break;
- }
- alphabet.get_rtn(*it)->collect_possible_first_symbols();
- std::set<SymbolNumber> rtn_firsts =
- alphabet.get_rtn(*it)->possible_first_symbols;
- for (RtnNameMap::const_iterator it = alphabet.rtn_names.begin();
- it != alphabet.rtn_names.end(); ++it) {
- if (rtn_firsts.count(it->second) == 1) {
- // For now we are very conservative:
- // if we can go through two levels of rtns
- // without any input, we just assume the full
- // input set is possible
- possible_firsts.clear();
+// This constructor handles all the awkward optimized-lookup specific
+// harmonization, for which the conversion needs to be done anyway,
+// so there's no advantage to passing it transducers in optimized-lookup
+// format.
+PmatchContainer::PmatchContainer(std::vector<HfstTransducer> transducers):
+ entry_stack(),
+ verbose(false),
+ locate_mode(false),
+ line_number(0),
+ profile_mode(false),
+ single_codepoint_tokenization(false)
+{
+ set_properties();
+ reset_recursion();
+ if (transducers.size() == 0) {
+ return;
+ }
+ std::map<std::string, std::string> properties = transducers[0].get_properties();
+ set_properties(properties);
+ if (transducers.size() == 1) {
+ HfstTransducer * top = NULL;
+ if (transducers[0].get_type() != hfst::HFST_OLW_TYPE) {
+ top = new HfstTransducer(transducers[0]);
+ top->convert(hfst::HFST_OLW_TYPE);
+ } else {
+ top = &(transducers[0]);
+ }
+ Transducer * backend = hfst::implementations::ConversionFunctions::
+ hfst_transducer_to_hfst_ol(top);
+ TransducerHeader header(backend->get_header());
+ alphabet = PmatchAlphabet(backend->get_alphabet(), this);
+ orig_symbol_count = symbol_count = alphabet.get_orig_symbol_count();
+ encoder = new Encoder(alphabet.get_symbol_table(), orig_symbol_count);
+ TransducerTable<TransitionW> transitions = backend->copy_transitionw_table();
+ TransducerTable<TransitionWIndex> indices = backend->copy_windex_table();
+ toplevel = new hfst_ol::PmatchTransducer(
+ transitions.get_vector(),
+ indices.get_vector(),
+ alphabet,
+ this);
+ if (transducers[0].get_type() != hfst::HFST_OLW_TYPE) {
+ // clean up if we needed a temp transducer
+ delete top;
+ }
+ } else {
+ // This is the difficult case where we have to make sure multiple
+ // optimized-lookup transducers are harmonized with each other.
+
+ HfstTransducer * top = NULL;
+ std::vector<HfstTransducer *> temporaries(transducers.size(), NULL);
+ // A dummy transducer with an alphabet with all the symbols
+ HfstTransducer harmonizer(hfst::TROPICAL_OPENFST_TYPE);
+ // First we need to collect a unified alphabet from all the transducers.
+ hfst::StringSet symbols_seen;
+ // We collect all the symbols and also copy and convert any non-olw bits
+ for (size_t i = 0; i < transducers.size(); ++i) {
+ hfst::StringSet string_set = transducers[i].get_alphabet();
+ for (hfst::StringSet::const_iterator sym = string_set.begin();
+ sym != string_set.end(); ++sym) {
+ if (symbols_seen.count(*sym) == 0) {
+ harmonizer.disjunct(HfstTransducer(*sym, harmonizer.get_type()));
+ symbols_seen.insert(*sym);
}
}
- if (rtn_firsts.empty() || possible_firsts.empty()) {
- possible_firsts.clear();
- break;
+ if (transducers[i].get_name() == "TOP") {
+ if (transducers[i].get_type() == hfst::HFST_OLW_TYPE) {
+ top = &(transducers[i]);
+ } else {
+ top = new HfstTransducer(transducers[i]);
+ top->convert(hfst::HFST_OLW_TYPE);
+ }
} else {
- for (std::set<SymbolNumber>::
- const_iterator rtn_it = rtn_firsts.begin();
- rtn_it != rtn_firsts.end(); ++rtn_it) {
- if (*rtn_it > max_input_sym) { max_input_sym = *rtn_it ;}
- possible_firsts.insert(*rtn_it);
+ if (transducers[i].get_type() == hfst::HFST_OLW_TYPE) {
+ temporaries[i] = &(transducers[i]);
+ } else {
+ temporaries[i] = new HfstTransducer(transducers[i]);
+ (temporaries[i])->convert(hfst::HFST_OLW_TYPE);
}
}
}
- }
- for (RtnNameMap::const_iterator it = alphabet.rtn_names.begin();
- it != alphabet.rtn_names.end(); ++it) {
- possible_firsts.erase(it->second);
- }
- if (!possible_firsts.empty() &&
- alphabet.get_special(boundary) != NO_SYMBOL_NUMBER) {
- possible_firsts.insert(alphabet.get_special(boundary));
- }
- if (!possible_firsts.empty()) {
- for (int i = 0; i <= max_input_sym; ++i) {
- if (possible_firsts.count(i) == 1) {
- possible_first_symbols.push_back(1);
+ if (top == NULL) {
+ std::cerr << "pmatch: warning: TOP not defined in archive, using first as TOP\n";
+ top = temporaries[0];
+ }
+ // Then we convert the harmonizer...
+ harmonizer.convert(hfst::HFST_OLW_TYPE);
+ // Use these for naughty intermediate steps to make sure
+ // everything has the same alphabet
+ hfst::HfstBasicTransducer * intermediate_tmp;
+ hfst_ol::Transducer * harmonized_tmp;
+
+ // We take care of TOP first
+ intermediate_tmp = hfst::implementations::ConversionFunctions::
+ hfst_transducer_to_hfst_basic_transducer(*top);
+ harmonized_tmp = hfst::implementations::ConversionFunctions::
+ hfst_basic_transducer_to_hfst_ol(intermediate_tmp,
+ true, // weighted
+ "", // no special options
+ &harmonizer); // harmonize with this
+ //TransducerHeader header = harmonized_tmp->get_header();
+ // this will be the alphabet of the entire container
+ alphabet = PmatchAlphabet(harmonized_tmp->get_alphabet(), this);
+ orig_symbol_count = symbol_count = alphabet.get_orig_symbol_count();
+ encoder = new Encoder(alphabet.get_symbol_table(), orig_symbol_count);
+ TransducerTable<TransitionW> transitions = harmonized_tmp->copy_transitionw_table();
+ TransducerTable<TransitionWIndex> indices = harmonized_tmp->copy_windex_table();
+ toplevel = new hfst_ol::PmatchTransducer(
+ transitions.get_vector(),
+ indices.get_vector(),
+ alphabet,
+ this);
+ // Then we do the same for the other transducers except without
+ // alphabets or encoders because those should be identical
+ for (size_t i = 0; i < temporaries.size(); ++i) {
+ if (temporaries[i] != NULL) {
+ // there's a NULL where TOP should be
+ intermediate_tmp = hfst::implementations::ConversionFunctions::
+ hfst_transducer_to_hfst_basic_transducer(*(temporaries[i]));
+ harmonized_tmp = hfst::implementations::ConversionFunctions::
+ hfst_basic_transducer_to_hfst_ol(intermediate_tmp,
+ true, // weighted
+ "", // no special options
+ &harmonizer); // harmonize with this
+ TransducerTable<TransitionW> transitions = harmonized_tmp->copy_transitionw_table();
+ TransducerTable<TransitionWIndex> indices = harmonized_tmp->copy_windex_table();
+ PmatchTransducer * rtn = new hfst_ol::PmatchTransducer(
+ transitions.get_vector(),
+ indices.get_vector(),
+ alphabet,
+ this);
+ alphabet.add_rtn(rtn, temporaries[i]->get_name());
+ }
+ }
+ // clean up the temporaries
+ for (size_t i = 0; i < transducers.size(); ++i) {
+ if (transducers[i].get_name() == "TOP") {
+ if (transducers[i].get_type() != hfst::HFST_OLW_TYPE) {
+ delete top;
+ }
} else {
- possible_first_symbols.push_back(0);
+ if (transducers[i].get_type() != hfst::HFST_OLW_TYPE) {
+ delete temporaries[i];
+ }
}
}
}
+ collect_first_symbols();
}
void PmatchContainer::add_rtn(Transducer * rtn, const std::string & name)
@@ -442,6 +563,17 @@ bool PmatchAlphabet::is_end_tag(const SymbolNumber symbol) const
return end_tag_map.count(symbol) == 1;
}
+bool PmatchAlphabet::is_like_arc(const std::string & symbol)
+{
+ return symbol.find("@PMATCH_LIKE_") == 0 &&
+ symbol.rfind("@") == symbol.size() - 1;
+}
+
+bool PmatchAlphabet::is_like_arc(const SymbolNumber symbol) const
+{
+ return words_like_map.count(symbol) == 1;
+}
+
bool PmatchAlphabet::is_insertion(const std::string & symbol)
{
return symbol.find("@I.") == 0 && symbol.rfind("@") == symbol.size() - 1;
@@ -544,8 +676,9 @@ PmatchAlphabet::~PmatchAlphabet(void)
}
-std::string PmatchContainer::parse_name_from_hfst3_header(std::istream & f)
+std::map<std::string, std::string> PmatchContainer::parse_hfst3_header(std::istream & f)
{
+ std::map<std::string, std::string> properties;
const char* header1 = "HFST";
unsigned int header_loc = 0; // how much of the header has been found
int c;
@@ -568,38 +701,18 @@ std::string PmatchContainer::parse_name_from_hfst3_header(std::istream & f)
if (headervalue[remaining_header_len - 1] != '\0') {
HFST_THROW(TransducerHeaderException);
}
- char * type = new char [remaining_header_len];
- bool type_defined = false;
- char * name = new char [remaining_header_len];
- bool name_defined = false;
int i = 0;
while (i < remaining_header_len) {
- if (!type_defined && strstr(headervalue + i, "type")) {
- strcpy(type, headervalue + i + strlen("type") + 1);
- type_defined = true;
- } else if (!name_defined && strstr(headervalue + i, "name")) {
- strcpy(name, headervalue + i + strlen("name") + 1);
- name_defined = true;
- }
- while (i < remaining_header_len &&
- headervalue[i] != '\0') {
- ++i;
- }
- ++i;
+ int length = hfst::size_t_to_int(strlen(headervalue + i));
+ std::string property(headervalue + i, headervalue + i + length);
+ i += length + 1;
+ length = hfst::size_t_to_int(strlen(headervalue + i));
+ std::string value(headervalue + i, headervalue + i + length);
+ properties[property] = value;
+ i += length + 1;
}
delete[] headervalue;
- if (strcmp(type, "HFST_OL") == 0) {
- std::cerr << "\nThis version of pmatch uses weighted rulesets only, please recompile your rules\n\n";
- HFST_THROW_MESSAGE(TransducerHeaderException,
- "This version of pmatch uses weighted rulesets only, please recompile your rules\n");
- }
- if (strcmp(type, "HFST_OLW") != 0) {
- HFST_THROW(TransducerHeaderException);
- }
- std::string retval = std::string(name);
- delete [] type;
- delete [] name;
- return retval;
+ return properties;
} else // nope. put back what we've taken
{
f.unget(); // first the non-matching character
@@ -681,7 +794,7 @@ void PmatchContainer::process(const std::string & input_str)
if (locate_mode) {
if (!nonmatching_locations.empty()) {
LocationVector ls;
- Location nonmatching = alphabet.locatefy(printable_input_pos - nonmatching_locations.size(),
+ Location nonmatching = alphabet.locatefy(printable_input_pos - hfst::size_t_to_uint(nonmatching_locations.size()),
WeightedDoubleTape(nonmatching_locations, 0.0));
nonmatching.output = "@_NONMATCHING_@";
ls.push_back(nonmatching);
@@ -713,7 +826,7 @@ void PmatchContainer::process(const std::string & input_str)
}
if (locate_mode && !nonmatching_locations.empty()) {
LocationVector ls;
- Location nonmatching = alphabet.locatefy(printable_input_pos - nonmatching_locations.size(),
+ Location nonmatching = alphabet.locatefy(printable_input_pos - hfst::size_t_to_uint(nonmatching_locations.size()),
WeightedDoubleTape(nonmatching_locations, 0.0));
nonmatching.output = "@_NONMATCHING_@";
ls.push_back(nonmatching);
@@ -735,7 +848,7 @@ std::string PmatchContainer::match(const std::string & input,
return stringify_output();
}
-LocationVectorVector PmatchContainer::locate(std::string & input,
+LocationVectorVector PmatchContainer::locate(const std::string & input,
double time_cutoff)
{
max_time = time_cutoff;
@@ -790,6 +903,29 @@ std::string PmatchContainer::get_profiling_info(void)
return retval.str();
}
+std::string PmatchContainer::get_pattern_count_info(void)
+{
+ size_t total = 0;
+ std::string retval = "Pattern\t\t# of matches\n------------------------\n";
+ for (std::map<std::string, size_t>::iterator it = pattern_counts.begin();
+ it != pattern_counts.end(); ++it) {
+ retval.append(it->first);
+ retval.append("\t\t");
+ std::ostringstream converter;
+ converter << it->second;
+ retval.append(converter.str());
+ retval.append("\n");
+ total += it->second;
+ }
+ retval.append("------------------------\n");
+ std::ostringstream converter;
+ converter << total;
+ retval.append("Total:\t\t");
+ retval.append(converter.str());
+ retval.append("\n");
+ return retval;
+}
+
void PmatchContainer::copy_to_output(const DoubleTape & best_result)
{
for (DoubleTape::const_iterator it = best_result.begin();
@@ -821,12 +957,19 @@ std::string PmatchAlphabet::stringify(const DoubleTape & str)
it != str.end(); ++it) {
SymbolNumber output = it->output;
if (output == special_symbols[entry]) {
- start_tag_pos.push(retval.size());
+ start_tag_pos.push(hfst::size_t_to_uint(retval.size()));
} else if (output == special_symbols[exit]) {
if (start_tag_pos.size() != 0) {
start_tag_pos.pop();
}
} else if (is_end_tag(output)) {
+ if (container->count_patterns) {
+ if ((container->pattern_counts).count(start_tag(output)) == 0) {
+ (container->pattern_counts)[start_tag(output)] = 1;
+ } else {
+ (container->pattern_counts)[start_tag(output)] += 1;
+ }
+ }
unsigned int pos;
if (start_tag_pos.size() == 0) {
std::cerr << "Warning: end tag without start tag\n";
@@ -834,10 +977,15 @@ std::string PmatchAlphabet::stringify(const DoubleTape & str)
} else {
pos = start_tag_pos.top();
}
- retval.insert(pos, start_tag(output));
- retval.append(end_tag(output));
+ if (container->delete_patterns) {
+ size_t how_much_to_delete = retval.size() - pos;
+ retval.replace(pos, how_much_to_delete, start_tag(output));
+ } else if (container->mark_patterns) {
+ retval.insert(pos, start_tag(output));
+ retval.append(end_tag(output));
+ }
} else {
- if ((!extract_tags || start_tag_pos.size() != 0)
+ if ((!(container->extract_patterns) || start_tag_pos.size() != 0)
&& is_printable(output)) {
retval.append(string_from_symbol(output));
}
@@ -862,21 +1010,32 @@ Location PmatchAlphabet::locatefy(unsigned int input_offset,
SymbolNumber input = it->input;
SymbolNumber output = it->output;
if (is_end_tag(output)) {
+ if (container->count_patterns) {
+ if ((container->pattern_counts).count(start_tag(output)) == 0) {
+ (container->pattern_counts)[start_tag(output)] = 1;
+ } else {
+ (container->pattern_counts)[start_tag(output)] += 1;
+ }
+ }
retval.tag = start_tag(output);
continue;
}
if (is_printable(output)) {
- retval.output.append(string_from_symbol(output));
+ std::string s = string_from_symbol(output);
+ retval.output.append(s);
+ retval.output_symbol_strings.push_back(s);
}
if (is_printable(input)) {
- retval.input.append(string_from_symbol(input));
+ std::string s = string_from_symbol(input);
+ retval.input.append(s);
+ retval.input_symbol_strings.push_back(s);
++input_offset;
}
if (is_input_mark(output)) {
retval.output_parts.push_back(output_mark);
retval.input_parts.push_back(input_mark);
- output_mark = retval.output.size();
- input_mark = retval.input.size();
+ output_mark = retval.output_symbol_strings.size();
+ input_mark = retval.input_symbol_strings.size();
}
}
if (output_mark > 0) {
@@ -913,11 +1072,12 @@ PmatchTransducer::PmatchTransducer(std::istream & is,
container(cont),
locations(NULL)
{
- orig_symbol_count = alphabet.get_symbol_table().size();
+ orig_symbol_count = hfst::size_t_to_uint(alphabet.get_symbol_table().size());
// initialize the stack for local variables
LocalVariables locals_front;
locals_front.flag_state = alphabet.get_fd_table();
locals_front.tape_step = 1;
+ locals_front.max_context_length_remaining = 254;
locals_front.context = none;
locals_front.context_placeholder = 0;
locals_front.default_symbol_trap = false;
@@ -967,17 +1127,18 @@ PmatchTransducer::PmatchTransducer(std::vector<TransitionW> transition_vector,
std::vector<TransitionWIndex> index_vector,
PmatchAlphabet & alpha,
PmatchContainer * cont):
+ transition_table(transition_vector),
+ index_table(index_vector),
alphabet(alpha),
container(cont),
- locations(NULL),
- transition_table(transition_vector),
- index_table(index_vector)
+ locations(NULL)
{
- orig_symbol_count = alphabet.get_symbol_table().size();
+ orig_symbol_count = hfst::size_t_to_uint(alphabet.get_symbol_table().size());
// initialize the stack for local variables
LocalVariables locals_front;
locals_front.flag_state = alphabet.get_fd_table();
locals_front.tape_step = 1;
+ locals_front.max_context_length_remaining = 254;
locals_front.context = none;
locals_front.context_placeholder = 0;
locals_front.default_symbol_trap = false;
@@ -1002,10 +1163,11 @@ void PmatchTransducer::collect_possible_first_symbols(void)
std::set<SymbolNumber> special_symbols(special_symbol_v.begin(),
special_symbol_v.end());
SymbolTable table = alphabet.get_symbol_table();
- SymbolNumber symbol_count = table.size();
+ SymbolNumber symbol_count = hfst::size_t_to_uint(table.size());
for (SymbolNumber i = 1; i < symbol_count; ++i) {
if (!alphabet.is_like_epsilon(i) &&
!alphabet.is_end_tag(i) &&
+ !alphabet.is_like_arc(i) &&
special_symbols.count(i) == 0 &&
i != alphabet.get_unknown_symbol() &&
i != alphabet.get_identity_symbol() &&
@@ -1085,7 +1247,7 @@ void PmatchTransducer::collect_first_epsilon_index(TransitionTableIndex i,
if (index_table[i].get_input_symbol() == 0) {
collect_first_epsilon(
index_table[i].get_target() - TRANSITION_TARGET_TABLE_START,
- input_symbols, seen_indices);
+ input_symbols, seen_indices);
}
}
@@ -1182,7 +1344,77 @@ void PmatchTransducer::collect_first(TransitionTableIndex i,
}
}
+void PmatchContainer::set_properties(void)
+{
+ count_patterns = false;
+ delete_patterns = false;
+ extract_patterns = false;
+ locate_mode = false;
+ mark_patterns = true;
+ max_context_length = 254;
+ max_recursion = 5000;
+ need_separators = true;
+}
+void PmatchContainer::set_properties(std::map<std::string, std::string> & properties)
+{
+ for (std::map<std::string, std::string>::iterator it = properties.begin();
+ it != properties.end(); ++it) {
+ if (it->first == "count-patterns") {
+ if (it->second == "on") {
+ count_patterns = true;
+ } else if (it->second == "off") {
+ count_patterns = false;
+ }
+ } else if (it->first == "delete-patterns") {
+ if (it->second == "on") {
+ delete_patterns = true;
+ } else if (it->second == "off") {
+ delete_patterns = false;
+ }
+ } else if (it->first == "extract-patterns") {
+ if (it->second == "on") {
+ extract_patterns = true;
+ } else if (it->second == "off") {
+ extract_patterns = false;
+ }
+ } else if (it->first == "locate-patterns") {
+ if (it->second == "on") {
+ locate_mode = true;
+ } else if (it->second == "off") {
+ locate_mode = false;
+ }
+ } else if (it->first == "mark-patterns") {
+ if (it->second == "on") {
+ mark_patterns = true;
+ } else if (it->second == "off") {
+ mark_patterns = false;
+ }
+ } else if (it->first == "max-context-length") {
+ std::stringstream converter(it->second);
+ converter >> max_context_length;
+ if (max_context_length == 0) {
+ if (it->second != "0") {
+ max_context_length = 254;
+ }
+ }
+ } else if (it->first == "max-recursion") {
+ std::stringstream converter(it->second);
+ converter >> max_recursion;
+ if (max_recursion == 0) {
+ if (it->second != "0") {
+ max_recursion = 5000;
+ }
+ }
+ } else if (it->first == "need-separators") {
+ if (it->second == "on") {
+ need_separators = true;
+ } else if (it->second == "off") {
+ need_separators = false;
+ }
+ }
+ }
+}
void PmatchContainer::initialize_input(const char * input_s)
{
@@ -1192,7 +1424,7 @@ void PmatchContainer::initialize_input(const char * input_s)
SymbolNumber k = NO_SYMBOL_NUMBER;
SymbolNumber boundary_sym = alphabet.get_special(boundary);
char * single_codepoint_scratch;
- char * single_codepoint_scratch_orig;
+ char * single_codepoint_scratch_orig = NULL;
if (single_codepoint_tokenization) {
single_codepoint_scratch = new char[5];
single_codepoint_scratch_orig = single_codepoint_scratch;
@@ -1331,7 +1563,7 @@ void PmatchTransducer::note_analysis(unsigned int input_pos,
std::cerr << "\n\tline " << container->line_number << ": conflicting equally weighted matches found, keeping:\n\t"
<< alphabet.stringify(rtn_stack.top().best_result) << std::endl
<< "\tdiscarding:\n\t"
- << alphabet.stringify(discarded) << std::endl << std::endl;
+ << alphabet.stringify(discarded) << std::endl << std::endl;
}
}
@@ -1382,6 +1614,8 @@ void PmatchTransducer::take_epsilons(unsigned int input_pos,
container->entry_stack.push(input_pos);
} else if (output == alphabet.get_special(exit)) {
container->entry_stack.pop();
+ } else if (alphabet.is_like_arc(output)) { // a Like() arc
+ match_like_arc(input_pos, tape_pos);
}
get_analyses(input_pos, tape_pos + 1, target);
@@ -1482,6 +1716,11 @@ void PmatchTransducer::take_rtn(SymbolNumber input,
local_stack.top().running_weight = original_weight;
}
+void PmatchTransducer::match_like_arc(unsigned int input_pos,
+ unsigned int tape_pos)
+{
+}
+
void PmatchTransducer::take_flag(SymbolNumber input,
unsigned int input_pos,
unsigned int tape_pos,
@@ -1532,7 +1771,7 @@ void PmatchTransducer::take_transitions(SymbolNumber input,
local_stack.top().running_weight +=
transition_table[i].get_weight();
if (this_input == alphabet.get_special(Pmatch_passthrough)) {
- get_analyses(input_pos, tape_pos, target); // FIXME
+ get_analyses(input_pos, tape_pos, target); // awkward
} else {
container->tape.write(tape_pos, this_input, this_output);
get_analyses(input_pos + 1, tape_pos + 1, target);
@@ -1540,7 +1779,11 @@ void PmatchTransducer::take_transitions(SymbolNumber input,
local_stack.top().running_weight = tmp;
} else {
// Checking context so don't touch output
- get_analyses(input_pos + local_stack.top().tape_step, tape_pos, target);
+ if (local_stack.top().max_context_length_remaining > 0) {
+ local_stack.top().max_context_length_remaining -= 1;
+ get_analyses(input_pos + local_stack.top().tape_step, tape_pos, target);
+ local_stack.top().max_context_length_remaining += 1;
+ }
}
local_stack.top().default_symbol_trap = false;
} else {
@@ -1631,21 +1874,29 @@ bool PmatchTransducer::try_entering_context(SymbolNumber symbol)
local_stack.push(local_stack.top());
local_stack.top().context = LC;
local_stack.top().tape_step = -1;
+ local_stack.top().max_context_length_remaining =
+ container->max_context_length;
return true;
} else if (symbol == alphabet.get_special(RC_entry)) {
local_stack.push(local_stack.top());
local_stack.top().context = RC;
local_stack.top().tape_step = 1;
+ local_stack.top().max_context_length_remaining =
+ container->max_context_length;
return true;
} else if (symbol == alphabet.get_special(NLC_entry)) {
local_stack.push(local_stack.top());
local_stack.top().context = NLC;
local_stack.top().tape_step = -1;
+ local_stack.top().max_context_length_remaining =
+ container->max_context_length;
return true;
} else if (symbol == alphabet.get_special(NRC_entry)) {
local_stack.push(local_stack.top());
local_stack.top().context = NRC;
local_stack.top().tape_step = 1;
+ local_stack.top().max_context_length_remaining =
+ container->max_context_length;
return true;
} else {
return false;
diff --git a/libhfst/src/implementations/optimized-lookup/pmatch.h b/libhfst/src/implementations/optimized-lookup/pmatch.h
index e60d052..626812e 100644
--- a/libhfst/src/implementations/optimized-lookup/pmatch.h
+++ b/libhfst/src/implementations/optimized-lookup/pmatch.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _HFST_OL_TRANSDUCER_PMATCH_H_
#define _HFST_OL_TRANSDUCER_PMATCH_H_
@@ -14,6 +14,7 @@
#include <sstream>
#include <algorithm>
#include <ctime>
+#include "HfstTransducer.h"
#include "transducer.h"
namespace hfst_ol {
@@ -21,10 +22,8 @@ namespace hfst_ol {
class PmatchTransducer;
class PmatchContainer;
struct Location;
- class WeightedDoubleTape;
+ struct WeightedDoubleTape;
- const unsigned int PMATCH_MAX_RECURSION_DEPTH = 5000;
-
typedef std::vector<PmatchTransducer *> RtnVector;
typedef std::map<std::string, SymbolNumber> RtnNameMap;
typedef std::vector<Location> LocationVector;
@@ -47,37 +46,6 @@ namespace hfst_ol {
Pmatch_input_mark,
SPECIALSYMBOL_NR_ITEMS};
- struct SymbolPair
- {
- SymbolNumber input;
- SymbolNumber output;
- SymbolPair(void): input(0), output(0) {}
- SymbolPair(SymbolNumber i, SymbolNumber o): input(i), output(o) {}
- };
-
- class DoubleTape: public std::vector<SymbolPair>
- {
- public:
-
- void write(unsigned int pos, SymbolNumber in, SymbolNumber out)
- {
- while (pos >= this->size()) {
- this->push_back(SymbolPair());
- }
- this->operator[](pos) = SymbolPair(in, out);
- }
-
- DoubleTape extract_slice(unsigned int start, unsigned int stop)
- {
- DoubleTape retval;
- while(start < stop) {
- retval.push_back(this->at(start));
- ++start;
- }
- return retval;
- }
-
- };
class PositionStack: public std::vector<unsigned int>
{
@@ -89,18 +57,12 @@ namespace hfst_ol {
unsigned int top(void) { return back(); }
};
- class WeightedDoubleTape: public DoubleTape
- {
- public:
- Weight weight;
- WeightedDoubleTape(DoubleTape dt, Weight w): DoubleTape(dt), weight(w) {}
- };
-
class PmatchAlphabet: public TransducerAlphabet {
protected:
RtnVector rtns;
SymbolNumberVector special_symbols;
std::map<SymbolNumber, std::string> end_tag_map;
+ std::map<SymbolNumber, std::string> words_like_map;
RtnNameMap rtn_names;
// For each symbol, either NO_SYMBOL for "no corresponding list" or an index into symbol_lists
SymbolNumberVector symbol2lists;
@@ -112,24 +74,26 @@ namespace hfst_ol {
SymbolNumberVector exclusionary_lists;
std::vector<SymbolNumberVector> symbol_lists;
std::vector<SymbolNumberVector> symbol_list_members;
- std::vector<unsigned long int> counters;
+ std::vector<unsigned long> counters;
SymbolNumberVector guards;
std::vector<bool> printable_vector;
bool is_end_tag(const SymbolNumber symbol) const;
bool is_input_mark(const SymbolNumber symbol) const;
+ bool is_like_arc(const SymbolNumber symbol) const;
bool is_guard(const SymbolNumber symbol) const;
bool is_counter(const SymbolNumber symbol) const;
std::string end_tag(const SymbolNumber symbol);
std::string start_tag(const SymbolNumber symbol);
- bool extract_tags;
+ PmatchContainer * container;
public:
- PmatchAlphabet(std::istream& is, SymbolNumber symbol_count);
- PmatchAlphabet(TransducerAlphabet const & a);
+ PmatchAlphabet(std::istream& is, SymbolNumber symbol_count, PmatchContainer * cont);
+ PmatchAlphabet(TransducerAlphabet const & a, PmatchContainer * cont);
PmatchAlphabet(void);
~PmatchAlphabet(void);
virtual void add_symbol(const std::string & symbol);
static bool is_end_tag(const std::string & symbol);
+ static bool is_like_arc(const std::string & symbol);
static bool is_insertion(const std::string & symbol);
static bool is_guard(const std::string & symbol);
static bool is_list(const std::string & symbol);
@@ -174,7 +138,18 @@ namespace hfst_ol {
LocationVectorVector locations;
std::vector<char> possible_first_symbols;
bool verbose;
+
+ bool count_patterns;
+ bool delete_patterns;
+ bool extract_patterns;
bool locate_mode;
+ bool mark_patterns;
+ size_t max_context_length;
+ size_t max_recursion;
+ bool need_separators;
+
+ unsigned long line_number;
+ std::map<std::string, size_t> pattern_counts;
bool profile_mode;
bool single_codepoint_tokenization;
unsigned int recursion_depth_left;
@@ -187,15 +162,19 @@ namespace hfst_ol {
// A flag to set for when time has been overstepped
bool limit_reached;
+ void collect_first_symbols(void);
+
public:
PmatchContainer(std::istream & is);
PmatchContainer(Transducer * toplevel);
+ PmatchContainer(std::vector<hfst::HfstTransducer> transducers);
PmatchContainer(void);
~PmatchContainer(void);
- unsigned long line_number;
+ void set_properties(void);
+ void set_properties(std::map<std::string, std::string> & properties);
void initialize_input(const char * input);
bool has_unsatisfied_rtns(void) const;
std::string get_unsatisfied_rtn_name(void) const;
@@ -203,9 +182,10 @@ namespace hfst_ol {
void process(const std::string & input);
std::string match(const std::string & input,
double time_cutoff = 0.0);
- LocationVectorVector locate(std::string & input,
+ LocationVectorVector locate(const std::string & input,
double time_cutoff = 0.0);
std::string get_profiling_info(void);
+ std::string get_pattern_count_info(void);
bool has_queued_input(unsigned int input_pos);
bool not_possible_first_symbol(SymbolNumber sym)
{
@@ -219,12 +199,24 @@ namespace hfst_ol {
void copy_to_output(SymbolNumber input, SymbolNumber output);
std::string stringify_output(void);
// LocationVector locatefy_output(void);
- static std::string parse_name_from_hfst3_header(std::istream & f);
+ static std::map<std::string, std::string> parse_hfst3_header(std::istream & f);
void set_verbose(bool b) { verbose = b; }
- void set_extract_tags_mode(bool b)
- { alphabet.extract_tags = b; }
+ void set_locate_mode(bool b) { locate_mode = b; }
+ void set_extract_patterns(bool b)
+ { extract_patterns = b; }
void set_single_codepoint_tokenization(bool b)
{ single_codepoint_tokenization = b; }
+ void set_count_patterns(bool b)
+ { count_patterns = b; }
+ void set_delete_patterns(bool b)
+ { delete_patterns = b; }
+ void set_mark_patterns(bool b)
+ { mark_patterns = b; }
+ void set_max_recursion(size_t max)
+ { max_recursion = max; }
+ void set_max_context(size_t max)
+ { max_context_length = max; }
+ bool is_in_locate_mode(void) { return locate_mode; }
void set_profile(bool b) { profile_mode = b; }
bool try_recurse(void)
{
@@ -236,9 +228,10 @@ namespace hfst_ol {
}
}
void unrecurse(void) { ++recursion_depth_left; }
- void reset_recursion(void) { recursion_depth_left = PMATCH_MAX_RECURSION_DEPTH; }
+ void reset_recursion(void) { recursion_depth_left = (unsigned int)max_recursion; }
friend class PmatchTransducer;
+ friend class PmatchAlphabet;
};
struct Location
@@ -249,8 +242,10 @@ namespace hfst_ol {
std::string output;
std::string tag;
Weight weight;
- std::vector<size_t> input_parts;
- std::vector<size_t> output_parts;
+ std::vector<size_t> input_parts; // indices in input_symbol_strings
+ std::vector<size_t> output_parts; // indices in output_symbol_strings
+ std::vector<std::string> input_symbol_strings;
+ std::vector<std::string> output_symbol_strings;
bool operator<(Location rhs) const
{ return this->weight < rhs.weight; }
@@ -275,6 +270,7 @@ namespace hfst_ol {
{
hfst::FdState<SymbolNumber> flag_state;
char tape_step;
+ size_t max_context_length_remaining;
unsigned int context_placeholder;
ContextChecking context;
bool default_symbol_trap;
@@ -336,6 +332,8 @@ namespace hfst_ol {
}
}
+ void match_like_arc(unsigned int input_pos, unsigned int tape_pos);
+
// The mutually recursive lookup-handling functions
void take_epsilons(unsigned int input_pos,
diff --git a/libhfst/src/implementations/optimized-lookup/transducer.cc b/libhfst/src/implementations/optimized-lookup/transducer.cc
index fa54370..f87db43 100644
--- a/libhfst/src/implementations/optimized-lookup/transducer.cc
+++ b/libhfst/src/implementations/optimized-lookup/transducer.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include "./transducer.h"
@@ -49,7 +49,7 @@ TransducerAlphabet::TransducerAlphabet(std::istream& is,
}
symbol_table.push_back(str.c_str());
}
- orig_symbol_count = symbol_table.size();
+ orig_symbol_count = hfst::size_t_to_uint(symbol_table.size());
}
void TransducerAlphabet::add_symbol(char * symbol)
@@ -80,7 +80,7 @@ TransducerAlphabet::TransducerAlphabet(const SymbolTable& st):
identity_symbol = i;
}
}
- orig_symbol_count = symbol_table.size();
+ orig_symbol_count = hfst::size_t_to_uint(symbol_table.size());
}
SymbolNumber TransducerAlphabet::symbol_from_string(
@@ -228,8 +228,8 @@ void Encoder::read_input_symbol(const char * s, const int s_num)
ascii_symbols[(unsigned char)(*s)] = s_num;
}
// If there's an ascii tokenized symbol shadowing this, remove it
- if (strlen(s) > 1 &&
- should_ascii_tokenize((unsigned char)(*s)) &&
+ if (strlen(s) > 1 &&
+ should_ascii_tokenize((unsigned char)(*s)) &&
ascii_symbols[(unsigned char)(*s)] != NO_SYMBOL_NUMBER) {
ascii_symbols[(unsigned char)(*s)] = NO_SYMBOL_NUMBER;
}
@@ -269,7 +269,7 @@ bool Transducer::initialize_input(const char * input)
new_symbol[bytes_to_tokenize] = '\0';
(*input_str_ptr) += bytes_to_tokenize;
alphabet->add_symbol(new_symbol);
- k = alphabet->get_symbol_table().size() - 1;
+ k = hfst::size_t_to_uint(alphabet->get_symbol_table().size() - 1);
encoder->read_input_symbol(new_symbol, k);
delete [] new_symbol;
}
@@ -286,10 +286,11 @@ void Transducer::include_symbol_in_alphabet(const std::string & sym)
if (key != NO_SYMBOL_NUMBER) {
return;
}
- key = alphabet->get_symbol_table().size();
+ key = hfst::size_t_to_uint(alphabet->get_symbol_table().size());
alphabet->add_symbol(sym);
char * cstr_for_encoder = new char[sym.size() + 1];
- std::strcpy(cstr_for_encoder, sym.c_str());
+ //std::strcpy(cstr_for_encoder, sym.c_str());
+ strcpy(cstr_for_encoder, sym.c_str());
encoder->read_input_symbol(cstr_for_encoder, key);
delete[] cstr_for_encoder;
}
@@ -310,6 +311,12 @@ HfstOneLevelPaths * Transducer::lookup_fd(const std::string & s, ssize_t limit,
return lookup_fd(s.c_str(), limit, time_cutoff);
}
+HfstTwoLevelPaths * Transducer::lookup_fd_pairs(const std::string & s, ssize_t limit,
+ double time_cutoff)
+{
+ return lookup_fd_pairs(s.c_str(), limit, time_cutoff);
+}
+
bool Transducer::is_lookup_infinitely_ambiguous(const std::string & s)
{
if (!initialize_input(s.c_str())) {
@@ -346,6 +353,39 @@ HfstOneLevelPaths * Transducer::lookup_fd(const char * s, ssize_t limit,
start_clock = clock();
}
HfstOneLevelPaths * results = new HfstOneLevelPaths;
+ if (!initialize_input(s)) {
+ return results;
+ }
+ lookup_paths = new HfstTwoLevelPaths;
+ traversal_states.clear();
+ //current_weight += s.second;
+ get_analyses(0, 0, 0);
+ //current_weight -= s.second;
+ for (HfstTwoLevelPaths::iterator it = lookup_paths->begin();
+ it != lookup_paths->end(); ++it) {
+ HfstOneLevelPath output_path;
+ output_path.first = it->first;
+ for (StringPairVector::const_iterator v_it = (it->second).begin();
+ v_it != (it->second).end(); ++v_it) {
+ output_path.second.push_back(v_it->second);
+ }
+ results->insert(output_path);
+ }
+ delete lookup_paths;
+ lookup_paths = NULL;
+ return results;
+}
+
+HfstTwoLevelPaths * Transducer::lookup_fd_pairs(const char * s, ssize_t limit,
+ double time_cutoff)
+{
+ max_lookups = limit;
+ max_time = 0.0;
+ if (time_cutoff > 0.0) {
+ max_time = time_cutoff;
+ start_clock = clock();
+ }
+ HfstTwoLevelPaths * results = new HfstTwoLevelPaths;
lookup_paths = results;
if (!initialize_input(s)) {
lookup_paths = NULL;
@@ -359,7 +399,6 @@ HfstOneLevelPaths * Transducer::lookup_fd(const char * s, ssize_t limit,
return results;
}
-
void Transducer::try_epsilon_transitions(unsigned int input_pos,
unsigned int output_pos,
TransitionTableIndex i)
@@ -372,7 +411,7 @@ void Transducer::try_epsilon_transitions(unsigned int input_pos,
Weight weight = tables->get_weight(i);
if (input == 0) // epsilon
{
- output_tape.write(output_pos, output);
+ output_tape.write(output_pos, input, output);
current_weight += weight;
get_analyses(input_pos, output_pos + 1, target);
found_transition = true;
@@ -392,7 +431,7 @@ void Transducer::try_epsilon_transitions(unsigned int input_pos,
}
traversal_states.insert(flag_reachable);
- output_tape.write(output_pos, output);
+ output_tape.write(output_pos, input, output);
current_weight += weight;
get_analyses(input_pos, output_pos + 1, target);
found_transition = true;
@@ -415,7 +454,7 @@ void Transducer::try_epsilon_indices(unsigned int input_pos,
{
try_epsilon_transitions(input_pos,
output_pos,
- tables->get_index_target(i) -
+ tables->get_index_target(i) -
TRANSITION_TARGET_TABLE_START);
found_transition = true;
}
@@ -441,7 +480,7 @@ void Transducer::find_transitions(SymbolNumber input,
// back in the input tape to find the symbol we want to write
output = input_tape[input_pos - 1];
}
- output_tape.write(output_pos, output);
+ output_tape.write(output_pos, input, output);
current_weight += tables->get_weight(i);
get_analyses(input_pos,
output_pos + 1,
@@ -467,7 +506,7 @@ void Transducer::find_index(SymbolNumber input,
find_transitions(input,
input_pos,
output_pos,
- tables->get_index_target(i+input) -
+ tables->get_index_target(i+input) -
TRANSITION_TARGET_TABLE_START);
found_transition = true;
}
@@ -487,7 +526,7 @@ void Transducer::get_analyses(unsigned int input_pos,
// ": maximum recursion depth exceeded, discarding results\n";
return;
}
- if (max_lookups >= 0 && lookup_paths->size() >= max_lookups) {
+ if (max_lookups >= 0 && (ssize_t)lookup_paths->size() >= max_lookups) {
// Back out because we have enough results already
return;
}
@@ -503,8 +542,8 @@ void Transducer::get_analyses(unsigned int input_pos,
i -= TRANSITION_TARGET_TABLE_START;
// First we check for finality and collect the result
if (input_tape[input_pos] == NO_SYMBOL_NUMBER) {
- if (max_lookups < 0 || lookup_paths->size() < max_lookups) {
- output_tape.write(output_pos, NO_SYMBOL_NUMBER);
+ if (max_lookups < 0 || (ssize_t)lookup_paths->size() < max_lookups) {
+ output_tape.write(output_pos, NO_SYMBOL_NUMBER, NO_SYMBOL_NUMBER);
if (tables->get_transition_finality(i)) {
current_weight += tables->get_weight(i);
note_analysis();
@@ -552,8 +591,8 @@ void Transducer::get_analyses(unsigned int input_pos,
else
{
if (input_tape[input_pos] == NO_SYMBOL_NUMBER) {
- if (max_lookups < 0 || lookup_paths->size() < max_lookups) {
- output_tape.write(output_pos, NO_SYMBOL_NUMBER);
+ if (max_lookups < 0 || (ssize_t)lookup_paths->size() < max_lookups) {
+ output_tape.write(output_pos, NO_SYMBOL_NUMBER, NO_SYMBOL_NUMBER);
if (tables->get_index_finality(i)) {
current_weight += tables->get_final_weight(i);
note_analysis();
@@ -594,23 +633,22 @@ void Transducer::get_analyses(unsigned int input_pos,
input_pos, output_pos, i+1);
}
}
- output_tape.write(output_pos, NO_SYMBOL_NUMBER);
+ output_tape.write(output_pos, NO_SYMBOL_NUMBER, NO_SYMBOL_NUMBER);
++recursion_depth_left;
}
void Transducer::note_analysis(void)
{
- HfstOneLevelPath result;
- for (SymbolNumberVector::const_iterator it = output_tape.begin();
- *it != NO_SYMBOL_NUMBER; ++it) {
- result.second.push_back(alphabet->string_from_symbol(*it));
+ HfstTwoLevelPath result;
+ for (DoubleTape::const_iterator it = output_tape.begin();
+ it->output != NO_SYMBOL_NUMBER; ++it) {
+ result.second.push_back(StringPair(alphabet->string_from_symbol(it->input),
+ alphabet->string_from_symbol(it->output)));
}
result.first = current_weight;
lookup_paths->insert(result);
}
-
-
Transducer::Transducer():
header(NULL), alphabet(NULL), tables(NULL),
current_weight(0.0), lookup_paths(NULL), encoder(NULL),
@@ -759,9 +797,9 @@ void Transducer::write(std::ostream& os) const
header->write(os);
alphabet->write(os);
for(size_t i=0;i<header->index_table_size();i++)
- tables->get_index(i).write(os, header->probe_flag(Weighted));
+ tables->get_index(hfst::size_t_to_uint(i)).write(os, header->probe_flag(Weighted));
for(size_t i=0;i<header->target_table_size();i++)
- tables->get_transition(i).write(os, header->probe_flag(Weighted));
+ tables->get_transition(hfst::size_t_to_uint(i)).write(os, header->probe_flag(Weighted));
}
Transducer * Transducer::copy(Transducer * t, bool weighted)
diff --git a/libhfst/src/implementations/optimized-lookup/transducer.h b/libhfst/src/implementations/optimized-lookup/transducer.h
index 4f2c7d4..b247a6d 100644
--- a/libhfst/src/implementations/optimized-lookup/transducer.h
+++ b/libhfst/src/implementations/optimized-lookup/transducer.h
@@ -1,11 +1,11 @@
// -*- mode: c++; -*-
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#ifndef _HFST_OL_TRANSDUCER_TRANSDUCER_H_
@@ -34,6 +34,7 @@
#include "../../HfstExceptionDefs.h"
#include "../../HfstFlagDiacritics.h"
#include "../../HfstSymbolDefs.h"
+#include "../../HfstDataTypes.h"
#ifdef _MSC_VER
#include <BaseTsd.h>
@@ -46,9 +47,12 @@ namespace hfst_ol {
using hfst::FdOperation;
using hfst::FdState;
using hfst::FdTable;
-
-// using namespace hfst;
-
+using hfst::HfstTwoLevelPath;
+using hfst::HfstOneLevelPath;
+using hfst::HfstTwoLevelPaths;
+using hfst::HfstOneLevelPaths;
+using hfst::StringVector;
+using hfst::StringPairVector;
typedef unsigned short SymbolNumber;
typedef unsigned int TransitionTableIndex;
@@ -62,9 +66,7 @@ typedef std::set<TransitionTableIndex> TransitionTableIndexSet;
typedef std::vector<std::string> SymbolTable;
// for lookup
-typedef std::pair<Weight, std::vector<std::string> > HfstOneLevelPath;
-typedef std::set<HfstOneLevelPath> HfstOneLevelPaths;
-typedef std::vector<std::string> StringVector;
+typedef std::pair<std::string, std::string> StringPair;
// for ospell
typedef std::vector<short> FlagDiacriticState;
@@ -277,6 +279,7 @@ public:
void set_flag(HeaderFlag flag, bool value)
{
+ (void)value;
switch (flag) {
case Weighted:
weighted = true;
@@ -652,7 +655,7 @@ public:
const T& operator[](TransitionTableIndex i) const
{
- return (i < TRANSITION_TARGET_TABLE_START) ?
+ return (i < TRANSITION_TARGET_TABLE_START) ?
table[i] : table[i-TRANSITION_TARGET_TABLE_START];
}
@@ -670,7 +673,7 @@ public:
}
}
- unsigned int size() const {return table.size();}
+ unsigned int size() const {return hfst::size_t_to_uint(table.size());}
};
class TransducerTablesInterface
@@ -769,8 +772,8 @@ private:
public:
OlLetterTrie():
- letters(UCHAR_MAX, static_cast<OlLetterTrie*>(NULL)),
- symbols(UCHAR_MAX,NO_SYMBOL_NUMBER)
+ letters(UCHAR_MAX+1, static_cast<OlLetterTrie*>(NULL)),
+ symbols(UCHAR_MAX+1,NO_SYMBOL_NUMBER)
{}
~OlLetterTrie() {
@@ -811,8 +814,43 @@ public:
friend class PmatchContainer;
};
+struct SymbolPair
+{
+ SymbolNumber input;
+ SymbolNumber output;
+ SymbolPair(void): input(0), output(0) {}
+ SymbolPair(SymbolNumber i, SymbolNumber o): input(i), output(o) {}
+};
+
// A vector that can be written to at any position, so that it
// adds new elements if the desired element isn't already present.
+struct DoubleTape: public std::vector<SymbolPair>
+{
+ void write(unsigned int pos, SymbolNumber in, SymbolNumber out)
+ {
+ while (pos >= this->size()) {
+ this->push_back(SymbolPair());
+ }
+ this->operator[](pos) = SymbolPair(in, out);
+ }
+
+ DoubleTape extract_slice(unsigned int start, unsigned int stop)
+ {
+ DoubleTape retval;
+ while(start < stop) {
+ retval.push_back(this->at(start));
+ ++start;
+ }
+ return retval;
+ }
+};
+
+struct WeightedDoubleTape: public DoubleTape
+{
+ Weight weight;
+ WeightedDoubleTape(DoubleTape dt, Weight w): DoubleTape(dt), weight(w) {}
+};
+
class Tape: public SymbolNumberVector
{
public:
@@ -841,10 +879,10 @@ protected:
// for lookup
Weight current_weight;
- HfstOneLevelPaths * lookup_paths;
+ HfstTwoLevelPaths * lookup_paths;
Encoder * encoder;
Tape input_tape;
- Tape output_tape;
+ DoubleTape output_tape;
hfst::FdState<SymbolNumber> flag_state;
// This is to keep track of whether we're going to take a default transition
bool found_transition;
@@ -971,6 +1009,10 @@ public:
double time_cutoff = 0.0);
HfstOneLevelPaths * lookup_fd(const char * s, ssize_t limit = -1,
double time_cutoff = 0.0);
+ HfstTwoLevelPaths * lookup_fd_pairs(const std::string & s, ssize_t limit = -1,
+ double time_cutoff = 0.0);
+ HfstTwoLevelPaths * lookup_fd_pairs(const char * s, ssize_t limit = -1,
+ double time_cutoff = 0.0);
void note_analysis(void);
// Methods for supporting ospell
@@ -1210,7 +1252,7 @@ public:
unsigned int len(void)
{
- return s.size();
+ return (unsigned int)s.size();
}
SymbolNumber operator[](unsigned int i)
diff --git a/tools/src/hfst-twolc/src/HfstTwolcDefs.h b/libhfst/src/parsers/HfstTwolcDefs.h
similarity index 94%
rename from tools/src/hfst-twolc/src/HfstTwolcDefs.h
rename to libhfst/src/parsers/HfstTwolcDefs.h
index ff9e59d..07bd7e3 100644
--- a/tools/src/hfst-twolc/src/HfstTwolcDefs.h
+++ b/libhfst/src/parsers/HfstTwolcDefs.h
@@ -26,7 +26,7 @@
#include <map>
#include <set>
#include <deque>
-#include <iostream>
+#include <iosfwd>
//! @brief Convenient subclass of std::map.
template <class K,class V> class HandyMap : public std::map<K,V>
@@ -58,7 +58,7 @@ template <class C> class HandyDeque : public std::deque<C>
{
public:
C get_front_and_pop(void)
- {
+ {
C temp = std::deque<C>::front();
std::deque<C>::pop_front();
return temp;
@@ -68,8 +68,8 @@ template <class C> class HandyDeque : public std::deque<C>
template <class C> std::ostream &operator<<
(std::ostream &out, const HandyDeque<C> &d)
{
- for (typename HandyDeque<C>::const_iterator it = d.begin();
- it != d.end();
+ for (typename HandyDeque<C>::const_iterator it = d.begin();
+ it != d.end();
++it)
{ out << *it << " "; }
return out;
diff --git a/libhfst/src/parsers/LexcCompiler.cc b/libhfst/src/parsers/LexcCompiler.cc
index e3c777c..2d714b8 100644
--- a/libhfst/src/parsers/LexcCompiler.cc
+++ b/libhfst/src/parsers/LexcCompiler.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
//! @file LexcCompiler.cc
@@ -42,7 +42,6 @@ using std::set_difference;
#include "xre_utils.h"
#include "HfstSymbolDefs.h"
-//#include "../../../tools/src/HfstStrings2FstTokenizer.h"
#ifdef WINDOWS
#include "hfst-string-conversions.h"
@@ -59,11 +58,6 @@ using hfst::xre::XreCompiler;
using hfst::StringVector;
//using same alg as strings to fst
-static char *epsilonname=NULL; // FIX: use this
-static bool has_spaces=false;
-static bool disjunct_strings=false;
-static bool pairstrings=false;
-static char *multichar_symbol_filename=NULL;
static StringVector multichar_symbols;
@@ -84,24 +78,24 @@ LexcCompiler* lexc_ = 0;
LexcCompiler::LexcCompiler() :
quiet_(false),
verbose_(false),
+ align_strings_(false),
+ with_flags_(false),
+ minimize_flags_(false),
+ rename_flags_(false),
treat_warnings_as_errors_(false),
+ allow_multiple_sublexicon_definitions_(false),
+ error_(&std::cerr),
format_(TROPICAL_OPENFST_TYPE),
xre_(TROPICAL_OPENFST_TYPE),
initialLexiconName_("Root"),
totalEntries_(0),
currentEntries_(0),
- parseErrors_(false),
- with_flags_(false),
- align_strings_(false),
- minimize_flags_(false),
- rename_flags_(false),
- allow_multiple_sublexicon_definitions_(false),
- error_(&std::cerr)
#ifdef WINDOWS
- , output_to_console_(false),
+ output_to_console_(false),
winoss_(std::ostringstream()),
- redirected_stream_(NULL)
+ redirected_stream_(NULL),
#endif
+ parseErrors_(false)
{
xre_.set_expand_definitions(true);
xre_.set_error_stream(this->error_);
@@ -111,24 +105,24 @@ LexcCompiler::LexcCompiler() :
LexcCompiler::LexcCompiler(ImplementationType impl) :
quiet_(false),
verbose_(false),
+ align_strings_(false),
+ with_flags_(false),
+ minimize_flags_(false),
+ rename_flags_(false),
treat_warnings_as_errors_(false),
+ allow_multiple_sublexicon_definitions_(false),
+ error_(&std::cerr),
format_(impl),
xre_(impl),
initialLexiconName_("Root"),
totalEntries_(0),
currentEntries_(0),
- parseErrors_(false),
- with_flags_(false),
- align_strings_(false),
- minimize_flags_(false),
- rename_flags_(false),
- allow_multiple_sublexicon_definitions_(false),
- error_(&std::cerr)
#ifdef WINDOWS
- , output_to_console_(false),
+ output_to_console_(false),
winoss_(std::ostringstream()),
- redirected_stream_(NULL)
+ redirected_stream_(NULL),
#endif
+ parseErrors_(false)
{
tokenizer_.add_multichar_symbol("@_EPSILON_SYMBOL_@");
tokenizer_.add_multichar_symbol("@0@");
@@ -145,24 +139,24 @@ LexcCompiler::LexcCompiler(ImplementationType impl) :
LexcCompiler::LexcCompiler(ImplementationType impl, bool withFlags, bool alignStrings) :
quiet_(false),
verbose_(false),
+ align_strings_(alignStrings),
+ with_flags_(withFlags),
+ minimize_flags_(false),
+ rename_flags_(false),
treat_warnings_as_errors_(false),
+ allow_multiple_sublexicon_definitions_(false),
+ error_(&std::cerr),
format_(impl),
xre_(impl),
initialLexiconName_("Root"),
totalEntries_(0),
currentEntries_(0),
- parseErrors_(false),
- with_flags_(withFlags),
- align_strings_(alignStrings),
- minimize_flags_(false),
- rename_flags_(false),
- allow_multiple_sublexicon_definitions_(false),
- error_(&std::cerr)
#ifdef WINDOWS
- , output_to_console_(false),
+ output_to_console_(false),
winoss_(std::ostringstream()),
- redirected_stream_(NULL)
+ redirected_stream_(NULL),
#endif
+ parseErrors_(false)
{
tokenizer_.add_multichar_symbol("@_EPSILON_SYMBOL_@");
tokenizer_.add_multichar_symbol("@0@");
@@ -176,6 +170,29 @@ LexcCompiler::LexcCompiler(ImplementationType impl, bool withFlags, bool alignSt
xre_.set_verbosity(!quiet_);
}
+ void LexcCompiler::reset()
+ {
+ tokenizer_ = hfst::HfstTokenizer();
+ tokenizer_.add_multichar_symbol("@_EPSILON_SYMBOL_@");
+ tokenizer_.add_multichar_symbol("@0@");
+ tokenizer_.add_multichar_symbol("@ZERO@");
+ tokenizer_.add_multichar_symbol("@@ANOTHER_EPSILON@@");
+ initialLexiconName_ = "Root";
+ totalEntries_ = 0;
+ currentEntries_ = 0;
+ parseErrors_ = false;
+ lexiconNames_.clear();
+ noFlags_.clear();
+ continuations_.clear();
+ currentLexiconName_ = ""; // ?
+ string hash("#");
+ lexiconNames_.insert(hash);
+ stringsTrie_ =hfst::implementations::HfstBasicTransducer(); // ?
+ stringTries_.clear();
+ stringVectors_.clear();
+ regexps_.clear();
+ }
+
std::ostream * LexcCompiler::get_stream(std::ostream * oss)
{
@@ -239,6 +256,7 @@ LexcCompiler& LexcCompiler::parse(FILE* infile)
hlexclex_destroy();
hlexcin = infile;
hlexcparse();
+ xre_.remove_defined_multichar_symbols();
if (hlexcnerrs > 0)
{
parseErrors_ = true;
@@ -251,17 +269,17 @@ LexcCompiler& LexcCompiler::parse(const char* filename)
lexc_ = this;
hlexclex_destroy();
hfst::lexc::set_infile_name(filename);
- hlexcin = fopen(filename, "r");
+ hlexcin = hfst::hfst_fopen(filename, "r");
if (hlexcin == NULL)
{
std::ostream * err = get_stream(error_);
*err << "could not open " << filename << " for reading" << std::endl;
- flush(err);
- //fprintf(stderr, "could not open %s for reading\n", filename);
+ flush(err);
parseErrors_ = true;
return *this;
}
hlexcparse();
+ xre_.remove_defined_multichar_symbols();
if (hlexcnerrs > 0)
{
parseErrors_ = true;
@@ -288,8 +306,6 @@ unsigned int LexcCompiler::getVerbosity()
LexcCompiler&
LexcCompiler::setVerbosity(unsigned int verbose)
{
- //quiet_ = !verbose;
- //verbose_ = verbose;
if (verbose == 0)
{
quiet_ = true;
@@ -378,8 +394,12 @@ LexcCompiler::addNoFlag(const string& lexname)
LexcCompiler&
LexcCompiler::addAlphabet(const string& alpha)
{
- //printf("alpha: %s \n", alpha.c_str());
tokenizer_.add_multichar_symbol(alpha);
+ if (!quiet_ && verbose_)
+ {
+ // warn about undefined multichars
+ xre_.add_defined_multichar_symbol(alpha);
+ }
return *this;
}
@@ -388,7 +408,6 @@ LexcCompiler&
LexcCompiler::addStringEntry(const string& data,
const string& continuation, double weight)
{
- //printf("data: %s \n", data.c_str());
string str = replace_zero(data);
currentEntries_++;
@@ -431,7 +450,7 @@ LexcCompiler::addStringEntry(const string& data,
}
tokenizer_.add_multichar_symbol(joinerEnc);
StringPairVector newVector(tokenizer_.tokenize(joinerEnc + str + encodedCont));
- stringsTrie_.disjunct(newVector, weight);
+ stringsTrie_.disjunct(newVector, hfst::double_to_float(weight));
return *this;
}
@@ -449,11 +468,9 @@ static void warn_about_one_sided_flags(const std::pair<std::string, std::string>
{
if (treat_one_sided_flags_as_errors_)
{
- if (true /*!quiet_one_sided_flags_*/) // error messages are always printed
- {
- *errorstr_ << std::endl << "*** ERROR: one-sided flag diacritic: " << symbol_pair.first << ":" << symbol_pair.second << " [--Werror]" << std::endl;
- lexc_->flush(errorstr_);
- }
+ // error messages are always printed
+ *errorstr_ << std::endl << "*** ERROR: one-sided flag diacritic: " << symbol_pair.first << ":" << symbol_pair.second << " [--Werror]" << std::endl;
+ lexc_->flush(errorstr_);
throw "one-sided flag";
}
if (!quiet_one_sided_flags_)
@@ -467,11 +484,9 @@ static void warn_about_one_sided_flags(const std::pair<std::string, std::string>
{
if (treat_one_sided_flags_as_errors_)
{
- if (true /*!quiet_one_sided_flags_*/) // error messages are always printed
- {
- *hfst::lexc::errorstr_ << std::endl << "*** ERROR: one-sided flag diacritic: " << symbol_pair.first << ":" << symbol_pair.second << " [--Werror]" << std::endl;
- lexc_->flush(errorstr_);
- }
+ // error messages are always printed
+ *hfst::lexc::errorstr_ << std::endl << "*** ERROR: one-sided flag diacritic: " << symbol_pair.first << ":" << symbol_pair.second << " [--Werror]" << std::endl;
+ lexc_->flush(errorstr_);
throw "one-sided flag";
}
if (!quiet_one_sided_flags_)
@@ -488,8 +503,6 @@ LexcCompiler::addStringPairEntry(const string& upper, const string& lower,
string upper_string = replace_zero(upper);
string lower_string = replace_zero(lower);
- //printf("upper: %s lower: %s: continuation: %s \n", upper.c_str(), lower.c_str(), continuation.c_str());
- //identityWoJoin.write_in_att_format(stdout, 1);
currentEntries_++;
totalEntries_++;
continuations_.insert(continuation);
@@ -583,24 +596,14 @@ LexcCompiler::addStringPairEntry(const string& upper, const string& lower,
StringPairVector lowerV;
lowerV = tokenizer_.tokenize(lower_string);
- int upperSize = upperV.size();
- int lowerSize = lowerV.size();
-
- //cout << "u: " << upperSize << "\n";
- //cout << "L: " << lowerSize << "\n";
-
-
-
- // StringVector upV;
- // StringVector loV;
-
+ int upperSize = hfst::size_t_to_int(upperV.size());
+ int lowerSize = hfst::size_t_to_int(lowerV.size());
if ( upperSize > lowerSize)
{
std::string epsilons = "";
for(int i=1; i <= upperSize-lowerSize ; i++)
{
- //cout << "another epsilon \n";
epsilons = epsilons + string("@@ANOTHER_EPSILON@@");
}
@@ -614,7 +617,6 @@ LexcCompiler::addStringPairEntry(const string& upper, const string& lower,
std::string epsilons = "";
for(int i=1; i <= lowerSize-upperSize ; i++)
{
- //cout << "another epsilon \n";
epsilons = epsilons + string("@@ANOTHER_EPSILON@@");
}
@@ -630,7 +632,7 @@ LexcCompiler::addStringPairEntry(const string& upper, const string& lower,
}
}
- stringsTrie_.disjunct(newVector, weight);
+ stringsTrie_.disjunct(newVector, hfst::double_to_float(weight));
return *this;
}
@@ -660,27 +662,11 @@ LexcCompiler::addXreEntry(const string& regexp, const string& continuation,
encodedCont = joinerEncode(encodedCont);
}
tokenizer_.add_multichar_symbol(encodedCont);
- char* xre_encoded = hfst::xre::add_percents(encodedCont.c_str());
+ //char* xre_encoded = hfst::xre::add_percents(encodedCont.c_str());
-
- // cout << "XRE COMPILE: " << "\n";
-
-
-
- //HfstTransducer* newPaths = xre_.compile(regexp + " " + string(xre_encoded));
HfstTransducer* newPaths = xre_.compile(regexp);
-// if (weight != 0)
-// {
-// newPaths->set_final_weights(weight);
-// }
- newPaths->minimize();
-
-
- //printf("newPaths: \n");
- //newPaths->write_in_att_format(stdout, 1);
-
- //std::map<std::string,hfst::HfstTransducer*> regexps_;
+ newPaths->optimize();
// encode key
// keep regexps with different continuations separate
@@ -688,7 +674,6 @@ LexcCompiler::addXreEntry(const string& regexp, const string& continuation,
regExpresionEncode(regex_key);
tokenizer_.add_multichar_symbol(regex_key);
- // cout << "lexicon " << regex_key << "\n";
// FIXME: add all implicit chars to multichar symbols
if (regexps_.find(regex_key) == regexps_.end())
@@ -696,7 +681,7 @@ LexcCompiler::addXreEntry(const string& regexp, const string& continuation,
regexps_.insert(pair<string,HfstTransducer*>(regex_key,
new HfstTransducer(format_)));
}
- regexps_[regex_key]->disjunct(*newPaths).minimize();
+ regexps_[regex_key]->disjunct(*newPaths).optimize();
if (!quiet_)
{
if ((currentEntries_ % 10000) == 0)
@@ -704,7 +689,6 @@ LexcCompiler::addXreEntry(const string& regexp, const string& continuation,
std::ostream * err = get_stream(error_);
*err << currentEntries_ << "...";
flush(err);
- //fprintf(stderr, SIZE_T_SPECIFIER "...", currentEntries_);
}
}
@@ -730,7 +714,7 @@ LexcCompiler::addXreEntry(const string& regexp, const string& continuation,
}
tokenizer_.add_multichar_symbol(joinerEnc);
StringPairVector newVector(tokenizer_.tokenize(joinerEnc + regex_key + encodedCont));
- stringsTrie_.disjunct(newVector, weight);
+ stringsTrie_.disjunct(newVector, hfst::double_to_float(weight));
@@ -747,9 +731,6 @@ LexcCompiler::addXreDefinition(const string& definition_name, const string& xre)
std::ostream * err = get_stream(error_);
*err << "Defined '" << definition_name << "': ? Kb., ? states, ? arcs, ? paths." << std::endl;
flush(err);
- //fprintf(stderr,
- // "Defined '%s': ? Kb., ? states, ? arcs, ? paths.\n",
- // definition_name.c_str());
}
return *this;
}
@@ -790,20 +771,19 @@ LexcCompiler::setCurrentLexiconName(const string& lexiconName)
}
else if ((firstLexicon) && (lexiconName != "Root"))
{
- if (!quiet_) *err << "first lexicon is not named Root" << std::endl; // fprintf(stderr, "first lexicon is not named Root\n");
+ if (!quiet_) *err << "first lexicon is not named Root" << std::endl;
setInitialLexiconName(lexiconName);
}
else if ((!firstLexicon) && (lexiconName == "Root"))
{
- if (!quiet_) *err << "Root is not first the first lexicon" << std::endl; // fprintf(stderr, "Root is not first the first lexicon\n");
+ if (!quiet_) *err << "Root is not first the first lexicon" << std::endl;
setInitialLexiconName(lexiconName);
}
if (!firstLexicon && !quiet_)
{
*err << currentEntries_ << " ";
- // fprintf(stderr, SIZE_T_SPECIFIER " ", currentEntries_);
}
- if (!quiet_) *err << lexiconName << "..."; // fprintf(stderr, "%s...", lexiconName.c_str());
+ if (!quiet_) *err << lexiconName << "...";
firstLexicon = false;
flush(err);
@@ -820,8 +800,6 @@ LexcCompiler::setInitialLexiconName(const string& lexiconName)
// for connectedness calculation:
continuations_.insert(lexiconName);
-// string joiner_initial_name;
- // tokenizer_.add_multichar_symbol(joinerEncode(joiner_initial_name));
return *this;
}
@@ -840,7 +818,7 @@ LexcCompiler::compileLexical()
printConnectedness(warnings_generated);
if (warnings_generated && treat_warnings_as_errors_)
{
- if (!quiet_) *err << "*** ERROR: could not parse lexc file: treating warnings as errors [--Werror] ***" << std::endl; //fprintf(stderr, "*** ERROR: could not parse lexc file: treating warnings as errors [--Werror] ***\n");
+ if (!quiet_) *err << "*** ERROR: could not parse lexc file: treating warnings as errors [--Werror] ***" << std::endl;
flush(err);
return 0;
}
@@ -848,10 +826,10 @@ LexcCompiler::compileLexical()
HfstTransducer lexicons(stringsTrie_, format_);
- lexicons.minimize();
+ lexicons.optimize();
// repeat star to overgenerate
- lexicons.repeat_star().minimize();
+ lexicons.repeat_star().optimize();
HfstSymbolSubstitutions smallSubstitutions;
smallSubstitutions.insert(StringPair("@0@", "@_EPSILON_SYMBOL_@"));
@@ -868,31 +846,12 @@ LexcCompiler::compileLexical()
if ( !with_flags_ )
{
string joinerinitialLexiconName_ = initialLexiconName_;
- //std::cout << "initialLexiconName_ " << initialLexiconName_ << "\n";
+
HfstTransducer start(joinerEncode(joinerinitialLexiconName_), tokenizer_, format_);
string endString = "#";
joinerEncode(endString);
HfstTransducer end(endString, tokenizer_, format_);
- lexicons = start.concatenate(lexicons).concatenate(end).minimize();
-
- //printf("lexicons: \n");
- //lexicons.write_in_att_format(stdout, 1);
-
- /*
- // if flags
-
- // for every lex_joiner in noFlags, find only strings where it occurs twice in a row
- // and then replace those joiners with epsilons
- for (set<string>::const_iterator s = noFlags_.begin();
- s != noFlags_.end();
- ++s)
- {
- ...
- string joinerEnc = *s;
- joinerEncode(joinerEnc);
- HfstTransducer joiner(joinerEnc, joinerEnc, format_);
- }
- */
+ lexicons = start.concatenate(lexicons).concatenate(end).optimize();
for (set<string>::const_iterator s = lexiconNames_.begin();
s != lexiconNames_.end();
@@ -902,20 +861,16 @@ LexcCompiler::compileLexical()
{
*err << "Morphotaxing... " << *s << " ";
flush(err);
- // fprintf(stderr, "Morphotaxing... %s ", s->c_str());
}
string joinerEnc = *s;
joinerEncode(joinerEnc);
// joiners trie version (later compose)
- // tokenizer_.add_multichar_symbol(joinerEnc);
StringPairVector newVector(tokenizer_.tokenize(joinerEnc + joinerEnc));
joinersTrie_.disjunct(newVector, 0);
- //printf ("\ninsert...\n");
allJoinersToEpsilon.insert(StringPair(joinerEnc, "@_EPSILON_SYMBOL_@"));
- //printf ("\ndone.\n");
}
string rootJoiner = initialLexiconName_;
@@ -950,33 +905,7 @@ LexcCompiler::compileLexical()
lexicons = startP.
concatenate(lexicons).
concatenate(endR).
- minimize();
-
- //printf("lexicons: \n");
- //lexicons.write_in_att_format(stdout, 1);
-
-
- // for every lex_joiner in noFlags, find only strings where it occurs twice in a row
- // and then replace those joiners with epsilons
- /*
- for (set<string>::const_iterator s = noFlags_.begin();
- s != noFlags_.end();
- ++s)
- {
- string joinerEnc = *s;
- joinerEncode(joinerEnc);
- //HfstTransducer joiner(joinerEnc, joinerEnc, format_);
- allJoinersToEpsilon.insert(StringPair(joinerEnc, "@_EPSILON_SYMBOL_@"));
-
- // joiners trie version (later compose)
- // tokenizer_.add_multichar_symbol(joinerEnc);
- StringPairVector newVector(tokenizer_.tokenize(joinerEnc + joinerEnc));
- joinersTrie_.disjunct(newVector, 0);
-
-
- }
- */
- ///
+ optimize();
for (set<string>::const_iterator s = lexiconNames_.begin();
s != lexiconNames_.end();
@@ -986,18 +915,14 @@ LexcCompiler::compileLexical()
{
*err << "Morphotaxing... " << *s << " ";
flush(err);
- //fprintf(stderr, "Morphotaxing... %s ", s->c_str());
}
string flagPstring = *s;
string flagRstring = *s;
- //joinerEncode(joinerEnc);
- //HfstTransducer joiner(joinerEnc, joinerEnc, format_);
flagJoinerEncode(flagPstring, false);
flagJoinerEncode(flagRstring, true);
// joiners trie version (later compose)
- // tokenizer_.add_multichar_symbol(joinerEnc);
StringPairVector newVector(tokenizer_.tokenize(flagPstring + flagRstring));
joinersTrie_.disjunct(newVector, 0);
}
@@ -1012,7 +937,7 @@ LexcCompiler::compileLexical()
it != fsm.end(); it++ )
{
// Go through all transitions
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
+ for (hfst::implementations::HfstBasicTransitions::const_iterator tr_it
= it->begin(); tr_it != it->end(); tr_it++)
{
String alph2 = tr_it->get_output_symbol();
@@ -1049,30 +974,25 @@ LexcCompiler::compileLexical()
joinersAll.repeat_star();
- joinersAll.minimize();
+ joinersAll.optimize();
- if (debug)
+ if (debug)
{
*err << "lexicons before compose: " << std::endl;
*err << lexicons;
- //fprintf(stderr, "lexicons before compose: \n");
- //lexicons.write_in_att_format(stderr, 1);
*err << "joinersAll: " << std::endl;
*err << joinersAll;
*err << std::endl;
- //fprintf(stderr, "joinersAll: \n");
- //joinersAll.write_in_att_format(stderr, 1);
- //fprintf(stderr, "\n");
flush(err);
}
- lexicons.compose(joinersAll).minimize();
+ lexicons.compose(joinersAll).optimize();
- if (debug)
+ if (debug)
{
*err << "lexicons after composition: " << std::endl;
- *err << lexicons << std::endl;
+ *err << lexicons << std::endl;
flush(err);
}
@@ -1083,57 +1003,50 @@ LexcCompiler::compileLexical()
{
*err << std::endl << "Changing flags..." << std::endl;
flush(err);
- // fprintf(stderr, "\nChanging flags...\n");
}
HfstSymbolSubstitutions fakeFlagsToRealFlags;
// Change fake flags to real flags
lexicons.prune_alphabet();
- //printf("alphabet lexicons: \n");
+
StringSet transducerAlphabet = lexicons.get_alphabet();
for (StringSet::const_iterator s = transducerAlphabet.begin();
s != transducerAlphabet.end();
++s)
{
- if (debug)
- {
+ if (debug)
+ {
*err << "handling alpha: '" << *s << "'..." << std::endl;
flush(err);
- //fprintf(stderr, "handling alpha: '%s'...\n", s->c_str());
}
String alph = *s;
if ( alph[0] == '$' && *alph.rbegin() == '$' && alph.size() > 2)
{
replace(alph.begin(), alph.end(), '$', '@');
- //std::cout << alph << '\n';
+
fakeFlagsToRealFlags.insert(StringPair(*s, alph));
- if (debug)
+ if (debug)
{
- *err << "debug: inserting fakeFlagsToRealFlags replacement: " << *s << " -> " << alph << std::endl;
+ *err << "debug: inserting fakeFlagsToRealFlags replacement: " << *s << " -> " << alph << std::endl;
flush(err);
}
- //lexicons.substitute(*s, alph).minimize();
}
}
allSubstitutions.insert(fakeFlagsToRealFlags.begin(), fakeFlagsToRealFlags.end());
- //lexicons.substitute(fakeFlagsToRealFlags).minimize();
- //lexicons.prune_alphabet();
}
else
{
allSubstitutions.insert(allJoinersToEpsilon.begin(), allJoinersToEpsilon.end());
- //lexicons.substitute(allJoinersToEpsilon).minimize();
- //lexicons.prune_alphabet();
}
- lexicons.substitute(allSubstitutions).minimize();
+ lexicons.substitute(allSubstitutions).optimize();
lexicons.prune_alphabet();
- if (debug)
+ if (debug)
{
*err << "lexicons after substitution: " << std::endl;
- *err << lexicons << std::endl;
+ *err << lexicons << std::endl;
flush(err);
}
@@ -1142,7 +1055,6 @@ LexcCompiler::compileLexical()
{
*err << std::endl << "Inserting regular expressions..." << std::endl;
flush(err);
- //fprintf(stderr, "\nInserting regular expressions...\n");
}
@@ -1158,28 +1070,17 @@ LexcCompiler::compileLexical()
// TODO: do this only for strings that look like $.....$
replace(alph.begin(), alph.end(), '$', '@');
- //std::cout << alph << '\n';
fakeRegexprToReal.insert(StringPair(it->first, alph));
- if (debug)
+ if (debug)
{
- *err << "debug: inserting fakeRegexprToReal replacement: " << it->first << " -> " << alph << std::endl;
+ *err << "debug: inserting fakeRegexprToReal replacement: " << it->first << " -> " << alph << std::endl;
flush(err);
}
-
- // lexicons.substitute(it->first, alph).minimize();
}
- //lexicons.substitute(StringPair(it->first, it->first), *it->second).minimize();
- //lexicons.substitute(StringPair(alph, alph), *it->second, true).minimize();
}
- lexicons.substitute(fakeRegexprToReal).minimize();
+ lexicons.substitute(fakeRegexprToReal).optimize();
lexicons.prune_alphabet();
-/*
- printf("lexicons before substitute: \n");
- lexicons.write_in_att_format(stdout, 1);
- printf("--\n");
-*/
- //SubstMap regMarkToTr;
std::map<String, HfstBasicTransducer> regMarkToTr;
@@ -1196,13 +1097,12 @@ LexcCompiler::compileLexical()
}
HfstBasicTransducer btr(*(it->second));
regMarkToTr[alph] = btr;
- if (debug)
- {
+ if (debug)
+ {
*err << "debug: regMarkToTr[" << alph << "] = " << std::endl;
btr.write_in_att_format(*err);
flush(err);
}
- //lexicons.substitute(StringPair(alph, alph), *it->second, true).minimize();
}
HfstBasicTransducer lexicons_basic(lexicons);
@@ -1210,7 +1110,7 @@ LexcCompiler::compileLexical()
lexicons_basic.prune_alphabet();
- if (debug)
+ if (debug)
{
*err << "lexicons_basic after regexp substitution: " << std::endl;
lexicons_basic.write_in_att_format(*err);
@@ -1228,9 +1128,9 @@ LexcCompiler::compileLexical()
HfstTransducer* rv = new HfstTransducer(lexicons_basic, format_);
- // Preserve only first flag of consecutive P and R lexname flag series,
- // e.g. change P.LEXNAME.1 R.LEXNAME.1 P.LEXNAME.2 R.LEXNAME.2 into P.LEXNAME.1
- if (with_flags_) //&& minimize_flags_)
+ // Preserve only first flag of consecutive P and R lexname flag series,
+ // e.g. change P.LEXNAME.1 R.LEXNAME.1 P.LEXNAME.2 R.LEXNAME.2 into P.LEXNAME.1
+ if (with_flags_)
{
StringSet transducerAlphabet = rv->get_alphabet();
StringSet flagD;
@@ -1240,19 +1140,19 @@ LexcCompiler::compileLexical()
{
String alph = *s;
String alph10 = alph.substr(0,10);
- if ( alph10 == "@P.LEXNAME" || alph10 == "@R.LEXNAME" )
+ if ( alph10 == "@P.LEXNAME" || alph10 == "@R.LEXNAME" )
{
flagD.insert(alph);
}
}
- // Construct a rule for consecutive flag removal:
+ // Construct a rule for consecutive flag removal:
// [FLAG1 | FLAG2 ... FLAGN] -> 0 || [FLAG1 | FLAG2 ... FLAGN] _
// and also an inverted rule
std::string flag_remover_regexp("[ ");
bool first_flag = true;
- for (StringSet::const_iterator it
+ for (StringSet::const_iterator it
= flagD.begin(); it != flagD.end(); ++it)
{
if (!first_flag)
@@ -1266,31 +1166,31 @@ LexcCompiler::compileLexical()
std::string context_regexp(flag_remover_regexp);
flag_remover_regexp.append(" -> 0 || ").append(context_regexp).append(" _ ");
- if (debug)
- {
- fprintf(stderr, "flag_remover_regexp: %s\n", flag_remover_regexp.c_str());
+ if (debug)
+ {
+ fprintf(stderr, "flag_remover_regexp: %s\n", flag_remover_regexp.c_str());
}
hfst::xre::XreCompiler xre_comp(format_);
HfstTransducer * flag_filter = xre_comp.compile(flag_remover_regexp);
- flag_filter->minimize();
+ flag_filter->optimize();
HfstTransducer * inverted_flag_filter = new HfstTransducer(*flag_filter);
- inverted_flag_filter->invert().minimize();
+ inverted_flag_filter->invert().optimize();
// [ [FLAG1 | FLAG2 ... FLAGN] -> 0 || [FLAG1 | FLAG2 ... FLAGN] _ ].inv
// .o.
// RESULT
- // .o.
+ // .o.
// [FLAG1 | FLAG2 ... FLAGN] -> 0 || [FLAG1 | FLAG2 ... FLAGN] _
HfstTransducer filtered_lexicons(*inverted_flag_filter);
filtered_lexicons.compose(*rv, true);
- filtered_lexicons.compose(*flag_filter, true).minimize();
+ filtered_lexicons.compose(*flag_filter, true).optimize();
rv->assign(filtered_lexicons);
}
- rv->minimize();
+ rv->optimize();
if(!quiet_) *err << endl;
@@ -1303,7 +1203,7 @@ LexcCompiler::printConnectedness(bool & warnings_generated)
{
std::ostream * err = get_stream(error_);
- if (/* !quiet_ && */ (lexiconNames_ != continuations_))
+ if (lexiconNames_ != continuations_)
{
vector<string> lexMinusCont = vector<string>(lexiconNames_.size());
vector<string> contMinusLex = vector<string>(continuations_.size());
@@ -1324,9 +1224,6 @@ LexcCompiler::printConnectedness(bool & warnings_generated)
{
*err << "Warning: Sublexicon is mentioned but not defined. (" << *s << ") " << std::endl;
flush(err);
- //fprintf(stderr,
- // "Warning: Sublexicon is mentioned but not defined."
- // " (%s) \n", s->c_str());
}
warnings_generated = true;
}
@@ -1337,15 +1234,12 @@ LexcCompiler::printConnectedness(bool & warnings_generated)
if (!quiet_)
{
*err << "Warning: Sublexicons defined but not used" << std::endl;
- //fprintf(stderr, "Warning: Sublexicons defined but not used\n");
for (vector<string>::iterator s = lexMinusCont.begin();
s != lexMinusContEnd; ++s)
{
*err << *s << " ";
- //fprintf(stderr, "%s ", s->c_str());
}
*err << std::endl;
- //fprintf(stderr, "\n");
flush(err);
}
}
@@ -1398,10 +1292,10 @@ main(int argc, char** argv)
lexcFoma.setVerbosity(1);
lexcFoma.setVerbosity(2);
#endif
- FILE* existence_check = fopen("LexcCompiler_test.lexc", "r");
+ FILE* existence_check = hfst::hfst_fopen("LexcCompiler_test.lexc", "r");
if (existence_check == NULL)
{
- existence_check = fopen("LexcCompiler_test.lexc", "w");
+ existence_check = hfst::hfst_fopen("LexcCompiler_test.lexc", "w");
assert(existence_check != NULL);
fprintf(existence_check, "Definitions\n" \
"\tdef1 = a\tb c;\n" \
@@ -1412,10 +1306,10 @@ main(int argc, char** argv)
"LEXICON Root\ncat # ;\ndog Plural ;\n");
fclose(existence_check);
}
- existence_check = fopen("LexcCompiler_test2.lexc", "r");
+ existence_check = hfst::hfst_fopen("LexcCompiler_test2.lexc", "r");
if (existence_check == NULL)
{
- existence_check = fopen("LexcCompiler_test2.lexc", "w");
+ existence_check = hfst::hfst_fopen("LexcCompiler_test2.lexc", "w");
assert(existence_check != NULL);
fprintf(existence_check, "LEXICON Plural\ns # ;\n");
fclose(existence_check);
@@ -1423,7 +1317,7 @@ main(int argc, char** argv)
std::cout << std::endl << "parsing: ";
#if HAVE_SFST
std::cout << "sfst parse(FILE)...";
- FILE* sfstFile = fopen("LexcCompiler_test.lexc", "r");
+ FILE* sfstFile = hfst::hfst_fopen("LexcCompiler_test.lexc", "r");
lexcSfst.parse(sfstFile);
fclose(sfstFile);
std::cout << "parse(filename)...";
@@ -1431,7 +1325,7 @@ main(int argc, char** argv)
#endif
#if HAVE_OPENFST
std::cout << "ofst parse(FILE)...";
- FILE* ofstFile = fopen("LexcCompiler_test.lexc", "r");
+ FILE* ofstFile = hfst::hfst_fopen("LexcCompiler_test.lexc", "r");
lexcOfst.parse(ofstFile);
fclose(ofstFile);
std::cout << "parse(filename)...";
@@ -1439,7 +1333,7 @@ main(int argc, char** argv)
#endif
#if HAVE_FOMA
std::cout << "foma parse(FILE)...";
- FILE* fomaFile = fopen("LexcCompiler_test.lexc", "r");
+ FILE* fomaFile = hfst::hfst_fopen("LexcCompiler_test.lexc", "r");
lexcFoma.parse(fomaFile);
fclose(fomaFile);
std::cout << "parse(filename)...";
diff --git a/libhfst/src/parsers/LexcCompiler.h b/libhfst/src/parsers/LexcCompiler.h
index 56cd3b0..8f8f3d8 100644
--- a/libhfst/src/parsers/LexcCompiler.h
+++ b/libhfst/src/parsers/LexcCompiler.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
//! @file LexcCompiler.h
@@ -32,7 +32,7 @@
namespace hfst { class HfstTransducer; }
#include "XreCompiler.h"
#include "../HfstTokenizer.h"
-#include "../implementations/HfstTransitionGraph.h"
+#include "../implementations/HfstBasicTransducer.h"
namespace hfst {
//! @brief Namespace for Xerox LexC related specific functions and classes.
@@ -54,6 +54,8 @@ class LexcCompiler
// as indicator as the trasnducer should be build with or without flags
LexcCompiler(hfst::ImplementationType impl, bool withFlags, bool alignStrings);
+ void reset();
+
//! @brief compile lexc description from @c infile into current compiler
LexcCompiler& parse(FILE* infile);
@@ -125,7 +127,7 @@ class LexcCompiler
//! @brief add entry defined by regular expression @a xre, pointing to
//! @a continuation weighing @a weight to current lexicon.
LexcCompiler& addXreEntry(const std::string& xre,
- const std::string& continuation,
+ const std::string& continuation,
const double weight);
//! @brief add macro definition named @a name matching regular expression
diff --git a/libhfst/src/parsers/Makefile.am b/libhfst/src/parsers/Makefile.am
index a465815..b6418d1 100644
--- a/libhfst/src/parsers/Makefile.am
+++ b/libhfst/src/parsers/Makefile.am
@@ -1,13 +1,13 @@
## Process this file with automake to produce Makefile.in
-# Copyright (c) 2016 University of Helsinki
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 3 of the License, or (at your option) any later version.
-# See the file COPYING included with this distribution for more
-# information.
+# Copyright (c) 2016 University of Helsinki
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+# See the file COPYING included with this distribution for more
+# information.
noinst_LTLIBRARIES=libhfstparsers.la
@@ -23,12 +23,80 @@ LEXC_SRCS=lexc-lexer.ll lexc-parser.yy lexc-utils.cc LexcCompiler.cc
LEXC_HDRS=LexcCompiler.h lexc-utils.h
LEXC_BUILT=lexc-parser.cc lexc-lexer.cc
+SFST_SRCS=sfst-scanner.ll sfst-compiler.yy SfstCompiler.cc SfstUtf8.cc SfstBasic.cc SfstAlphabet.cc
+SFST_HDRS=SfstCompiler.h SfstUtf8.h SfstBasic.h SfstAlphabet.h
+SFST_BUILT=sfst-compiler.cc sfst-scanner.cc
+
# TODO: hfst-string-conversions.cc
XFST_SRCS=xfst-lexer.ll xfst-parser.yy xfst-utils.cc XfstCompiler.cc xfst_help_message.cc
XFST_HDRS=XfstCompiler.h xfst-utils.h xfst_help_message.h
XFST_BUILT=xfst-parser.cc xfst-lexer.cc
-BUILT_SOURCES=$(XRE_BUILT) $(PMATCH_BUILT) $(LEXC_BUILT) $(XFST_BUILT)
+IO_SRC=$(srcdir)/io_src
+STRING_SRC=$(srcdir)/string_src
+VARIABLE_SRC=$(srcdir)/variable_src
+RULE_SRC=$(srcdir)/rule_src
+ALPHABET_SRC=$(srcdir)/alphabet_src
+CL_SRC=$(srcdir)/commandline_src
+
+TWOLC_SRCS=htwolcpre1-parser.yy htwolcpre1-lexer.ll htwolcpre2-parser.yy htwolcpre2-lexer.ll htwolcpre3-parser.yy htwolcpre3-lexer.ll \
+TwolcCompiler.cc \
+$(CL_SRC)/CommandLine.cc \
+$(ALPHABET_SRC)/Alphabet.cc \
+$(IO_SRC)/InputReader.cc \
+$(RULE_SRC)/ConflictResolvingLeftArrowRule.cc \
+$(RULE_SRC)/ConflictResolvingRightArrowRule.cc \
+$(RULE_SRC)/LeftArrowRule.cc \
+$(RULE_SRC)/LeftArrowRuleContainer.cc \
+$(RULE_SRC)/LeftRestrictionArrowRule.cc \
+$(RULE_SRC)/OtherSymbolTransducer.cc \
+$(RULE_SRC)/RightArrowRule.cc \
+$(RULE_SRC)/RightArrowRuleContainer.cc \
+$(RULE_SRC)/Rule.cc \
+$(RULE_SRC)/RuleContainer.cc \
+$(RULE_SRC)/TwolCGrammar.cc \
+$(STRING_SRC)/string_manipulation.cc \
+$(VARIABLE_SRC)/RuleSymbolVector.cc \
+$(VARIABLE_SRC)/RuleVariables.cc \
+$(VARIABLE_SRC)/RuleVariablesConstIterator.cc \
+$(VARIABLE_SRC)/VariableValues.cc
+
+# remove: $(IO_SRC)/input_defs.h
+TWOLC_HDRS=TwolcCompiler.h \
+$(ALPHABET_SRC)/Alphabet.h \
+$(CL_SRC)/CommandLine.h \
+grammar_defs.h \
+HfstTwolcDefs.h \
+$(IO_SRC)/InputReader.h \
+$(RULE_SRC)/ConflictResolvingLeftArrowRule.h \
+$(RULE_SRC)/ConflictResolvingRightArrowRule.h \
+$(RULE_SRC)/LeftArrowRuleContainer.h \
+$(RULE_SRC)/LeftArrowRule.h \
+$(RULE_SRC)/LeftRestrictionArrowRule.h \
+$(RULE_SRC)/OtherSymbolTransducer.h \
+$(RULE_SRC)/RightArrowRuleContainer.h \
+$(RULE_SRC)/RightArrowRule.h \
+$(RULE_SRC)/RuleContainer.h \
+$(RULE_SRC)/Rule.h \
+$(RULE_SRC)/TwolCGrammar.h \
+$(STRING_SRC)/string_manipulation.h \
+$(VARIABLE_SRC)/ConstContainerIterator.h \
+$(VARIABLE_SRC)/MatchedConstContainerIterator.h \
+$(VARIABLE_SRC)/MixedConstContainerIterator.h \
+$(VARIABLE_SRC)/RuleSymbolVector.h \
+$(VARIABLE_SRC)/RuleVariablesConstIterator.h \
+$(VARIABLE_SRC)/RuleVariables.h \
+$(VARIABLE_SRC)/VariableBlockContainer.h \
+$(VARIABLE_SRC)/VariableBlock.h \
+$(VARIABLE_SRC)/VariableContainerBase.h \
+$(VARIABLE_SRC)/VariableContainer.h \
+$(VARIABLE_SRC)/VariableDefs.h \
+$(VARIABLE_SRC)/VariableValueIterator.h \
+$(VARIABLE_SRC)/VariableValues.h
+
+TWOLC_BUILT=htwolcpre1-parser.cc htwolcpre1-lexer.cc htwolcpre2-parser.cc htwolcpre2-lexer.cc htwolcpre3-parser.cc htwolcpre3-lexer.cc
+
+BUILT_SOURCES=$(XRE_BUILT) $(PMATCH_BUILT) $(LEXC_BUILT) $(XFST_BUILT) $(SFST_BUILT) $(TWOLC_BUILT)
if YACC_USE_PARSER_H_EXTENSION
HEADER=h
@@ -36,40 +104,47 @@ else
HEADER=hh
endif
-CLEANFILES=$(XRE_BUILT) $(PMATCH_BUILT) $(LEXC_BUILT) $(XFST_BUILT) \
- xre_parse.$(HEADER) pmatch_parse.$(HEADER) lexc-parser.$(HEADER) xfst-parser.$(HEADER)
+CLEANFILES=$(XRE_BUILT) $(PMATCH_BUILT) $(LEXC_BUILT) $(XFST_BUILT) $(SFST_BUILT) $(TWOLC_BUILT) \
+ xre_parse.$(HEADER) pmatch_parse.$(HEADER) lexc-parser.$(HEADER) xfst-parser.$(HEADER) sfst-compiler.$(HEADER) \
+ htwolcpre1-parser.$(HEADER) htwolcpre2-parser.$(HEADER) htwolcpre3-parser.$(HEADER)
AM_YFLAGS=-d
AM_LFLAGS=--outfile=$(LEX_OUTPUT_ROOT).c
-#CLEANFILES+=xre_parse.hh pmatch_parse.hh lexc-parser.hh xfst-parser.hh
-
xre_lex.ll: xre_parse.$(HEADER)
pmatch_lex.ll: pmatch_parse.$(HEADER)
lexc-lexer.ll: lexc-parser.$(HEADER)
xfst-lexer.ll: xfst-parser.$(HEADER)
+sfst-scanner.ll: sfst-compiler.$(HEADER)
+htwolc1pre-lexer.ll: htwolcpre1-parser.$(HEADER)
+htwolc2pre-lexer.ll: htwolcpre2-parser.$(HEADER)
+htwolc3pre-lexer.ll: htwolcpre3-parser.$(HEADER)
-libhfstparsers_la_SOURCES=$(SFST_SRCS) $(XRE_SRCS) $(PMATCH_SRCS) $(LEXC_SRCS) $(XFST_SRCS)
+libhfstparsers_la_SOURCES=$(XRE_SRCS) $(PMATCH_SRCS) $(LEXC_SRCS) $(XFST_SRCS) $(SFST_SRCS) $(TWOLC_SRCS)
AM_CPPFLAGS=-I${top_srcdir}/libhfst/src/parsers -I${top_srcdir}/libhfst/src \
- -I${top_srcdir}/back-ends/foma -I${top_srcdir}/back-ends \
-Wno-deprecated
if WANT_MINGW
- AM_CPPFLAGS+= -I${top_srcdir}/back-ends/openfstwin/src/include -I${top_srcdir}/back-ends/dlfcn -DWINDOWS
-else
- AM_CPPFLAGS+= -I${top_srcdir}/back-ends/openfst/src/include
+ AM_CPPFLAGS+= -I${top_srcdir}/back-ends/dlfcn -DWINDOWS
endif
# headers of parsers
hfstincludedir = $(includedir)/hfst
extincludedir = $(hfstincludedir)/parsers
-extinclude_HEADERS = $(XRE_HDRS) $(PMATCH_HDRS) $(LEXC_HDRS) $(XFST_HDRS)
+extinclude_HEADERS = $(XRE_HDRS) $(PMATCH_HDRS) $(LEXC_HDRS) $(XFST_HDRS) $(SFST_HDRS) $(TWOLC_HDRS)
LIBHFST_PARSER_TSTS=XreCompiler LexcCompiler # PmatchCompiler
-check_PROGRAMS=$(LIBHFST_PARSER_TSTS)
+HFST_TWOLC_TESTS=string_manipulation InputReader Alphabet \
+ OtherSymbolTransducer Rule LeftArrowRule RightArrowRule \
+ RuleContainer LeftArrowRuleContainer RightArrowRuleContainer \
+ ConflictResolvingLeftArrowRule \
+ ConflictResolvingRightArrowRule TwolCGrammar \
+ LeftRestrictionArrowRule
+
+check_PROGRAMS=$(LIBHFST_PARSER_TSTS) $(HFST_TWOLC_TESTS)
XreCompiler_SOURCES=XreCompiler.cc xre_utils.cc xre_parse.yy xre_lex.ll
XreCompiler_CXXFLAGS=-DUNIT_TEST
@@ -81,7 +156,76 @@ LexcCompiler_SOURCES=LexcCompiler.cc lexc-utils.cc lexc-parser.yy lexc-lexer.ll
LexcCompiler_CXXFLAGS=-DUNIT_TEST -DDEBUG_MAIN
LexcCompiler_LDADD=../libhfst.la
-TESTS=$(LIBHFST_PARSER_TSTS)
+string_manipulation_SOURCES=$(STRING_SRC)/string_manipulation.cc
+string_manipulation_CXXFLAGS=-DSTRING_MANIPULATION_TEST -Wno-deprecated
+string_manipulation_LDADD=../libhfst.la
+
+InputReader_SOURCES=$(IO_SRC)/InputReader.cc
+InputReader_CXXFLAGS=-DINPUT_READER_TEST -Wno-deprecated
+InputReader_LDADD=../libhfst.la
+
+Alphabet_SOURCES=$(ALPHABET_SRC)/Alphabet.cc
+Alphabet_CXXFLAGS=-DTEST_ALPHABET -Wno-deprecated
+Alphabet_LDADD=../libhfst.la
+
+OtherSymbolTransducer_SOURCES=$(RULE_SRC)/OtherSymbolTransducer.cc
+OtherSymbolTransducer_CXXFLAGS=-DTEST_OTHER_SYMBOL_TRANSDUCER -Wno-deprecated
+OtherSymbolTransducer_LDADD=../libhfst.la
+
+Rule_SOURCES=$(RULE_SRC)/Rule.cc
+Rule_CXXFLAGS=-DTEST_RULE -Wno-deprecated
+Rule_LDADD=../libhfst.la
+
+LeftArrowRule_SOURCES=$(RULE_SRC)/LeftArrowRule.cc
+LeftArrowRule_CXXFLAGS=-DTEST_LEFT_ARROW_RULE -Wno-deprecated
+LeftArrowRule_LDADD=../libhfst.la
+
+LeftRestrictionArrowRule_SOURCES=$(RULE_SRC)/LeftRestrictionArrowRule.cc
+LeftRestrictionArrowRule_CXXFLAGS=-DTEST_LEFT_RESTRICTION_ARROW_RULE -Wno-deprecated
+LeftRestrictionArrowRule_LDADD=../libhfst.la
+
+RightArrowRule_SOURCES=$(RULE_SRC)/RightArrowRule.cc
+RightArrowRule_CXXFLAGS=-DTEST_RIGHT_ARROW_RULE -Wno-deprecated
+RightArrowRule_LDADD=../libhfst.la
+
+RuleContainer_SOURCES=$(RULE_SRC)/RuleContainer.cc
+RuleContainer_CXXFLAGS=-DTEST_RULE_CONTAINER -Wno-deprecated
+RuleContainer_LDADD=../libhfst.la
+
+LeftArrowRuleContainer_SOURCES=$(RULE_SRC)/LeftArrowRuleContainer.cc
+LeftArrowRuleContainer_CXXFLAGS=-DTEST_LEFT_ARROW_RULE_CONTAINER -Wno-deprecated
+LeftArrowRuleContainer_LDADD=../libhfst.la
+
+RightArrowRuleContainer_SOURCES=$(RULE_SRC)/RightArrowRuleContainer.cc
+RightArrowRuleContainer_CXXFLAGS=-DTEST_RIGHT_ARROW_RULE_CONTAINER -Wno-deprecated
+RightArrowRuleContainer_LDADD=../libhfst.la
+
+ConflictResolvingLeftArrowRule_SOURCES=$(RULE_SRC)/ConflictResolvingLeftArrowRule.cc
+ConflictResolvingLeftArrowRule_CXXFLAGS=-DTEST_CONFLICT_RESOLVING_LEFT_ARROW_RULE -Wno-deprecated
+ConflictResolvingLeftArrowRule_LDADD=../libhfst.la
+
+ConflictResolvingRightArrowRule_SOURCES=$(RULE_SRC)/ConflictResolvingRightArrowRule.cc
+ConflictResolvingRightArrowRule_CXXFLAGS=-DTEST_CONFLICT_RESOLVING_RIGHT_ARROW_RULE -Wno-deprecated
+ConflictResolvingRightArrowRule_LDADD=../libhfst.la
+
+TwolCGrammar_SOURCES=$(RULE_SRC)/TwolCGrammar.cc
+TwolCGrammar_CXXFLAGS=-DTEST_TWOL_C_GRAMMAR -Wno-deprecated
+TwolCGrammar_LDADD=../libhfst.la
+
+TESTS=$(LIBHFST_PARSER_TSTS) $(HFST_TWOLC_TESTS)
+
+EXTRA_DIST=xre_parse.$(HEADER) pmatch_parse.$(HEADER) lexc-parser.$(HEADER) xfst-parser.$(HEADER) sfst-compiler.$(HEADER) \
+ htwolcpre1-parser.$(HEADER) htwolcpre2-parser.$(HEADER) htwolcpre3-parser.$(HEADER) $(BUILT_SOURCES)
+
+
+# some systems might require all these
+# BUILT_SOURCES=htwolcpre1-parser.cc htwolcpre2-parser.cc htwolcpre3-parser.cc
+
+#htwolcpre1-parser.$(HEADER): htwolcpre1-parser.yy
+#htwolcpre2-parser.$(HEADER): htwolcpre2-parser.yy
+#htwolcpre3-parser.$(HEADER): htwolcpre3-parser.yy
-EXTRA_DIST=xre_parse.$(HEADER) pmatch_parse.$(HEADER) lexc-parser.$(HEADER) xfst-parser.$(HEADER) $(BUILT_SOURCES)
+#htwolcpre1-lexer.ll: htwolcpre1-parser.$(HEADER)
+#htwolcpre2-lexer.ll: htwolcpre2-parser.$(HEADER)
+#htwolcpre3-lexer.ll: htwolcpre3-parser.$(HEADER)
diff --git a/libhfst/src/parsers/PmatchCompiler.cc b/libhfst/src/parsers/PmatchCompiler.cc
index 14a8d27..045eba0 100644
--- a/libhfst/src/parsers/PmatchCompiler.cc
+++ b/libhfst/src/parsers/PmatchCompiler.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
//! @file PmatchCompiler.cc
@@ -45,7 +45,12 @@ std::map<std::string, HfstTransducer*>
PmatchCompiler::compile(const std::string& pmatch)
{
return hfst::pmatch::compile(pmatch, definitions_, format_,
- verbose, flatten);
+ verbose, flatten, includedir);
+}
+
+void PmatchCompiler::set_include_path(std::string path)
+{
+ includedir = path;
}
}}
@@ -165,7 +170,7 @@ main(int, char**)
// std::cout << "(a a | b c)...";
// HfstTransducer* ofstAaOrBc = ofstPmatch.compile("a a | b c");
// assert(ofstAaOrBc != 0);
-// assert(ofstAaOrBc->compare(HfstTransducer(basicAaOrBc,
+// assert(ofstAaOrBc->compare(HfstTransducer(basicAaOrBc,
// TROPICAL_OPENFST_TYPE)));
// delete ofstAaOrBc;
// #endif
diff --git a/libhfst/src/parsers/PmatchCompiler.h b/libhfst/src/parsers/PmatchCompiler.h
index 8272c85..df5824d 100644
--- a/libhfst/src/parsers/PmatchCompiler.h
+++ b/libhfst/src/parsers/PmatchCompiler.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
//! @file PmatchCompiler.h
@@ -30,8 +30,8 @@
#include <string>
#include "../HfstDataTypes.h"
-namespace hfst {
-//! @brief hfst::pmatch namespace is used for all functions related to Xerox
+namespace hfst {
+//! @brief hfst::pmatch namespace is used for all functions related to Xerox
//! Regular Expresisions (PMATCH) parsing.
namespace pmatch {
//! @brief A compiler holding information needed to compile PMATCHs.
@@ -40,6 +40,7 @@ class PmatchCompiler
private:
bool flatten;
bool verbose;
+ std::string includedir;
public:
//! @brief Construct compiler for unknown format transducers.
PmatchCompiler();
@@ -58,6 +59,7 @@ private:
//! May return a pointer to @e empty transducer on non-fatal error.
//! A null pointer is returned on fatal error, if abort is not called.
std::map<std::string, HfstTransducer*> compile(const std::string& pmatch);
+ void set_include_path(std::string path);
private:
std::map<std::string,hfst::HfstTransducer*> definitions_;
diff --git a/tools/src/HfstAlphabet.cc b/libhfst/src/parsers/SfstAlphabet.cc
similarity index 79%
rename from tools/src/HfstAlphabet.cc
rename to libhfst/src/parsers/SfstAlphabet.cc
index faffc15..1708f81 100644
--- a/tools/src/HfstAlphabet.cc
+++ b/libhfst/src/parsers/SfstAlphabet.cc
@@ -1,6 +1,6 @@
-#include "HfstAlphabet.h"
-#include "HfstBasic.h"
-#include "HfstUtf8.h"
+#include "SfstAlphabet.h"
+#include "SfstBasic.h"
+#include "SfstUtf8.h"
#include "HfstSymbolDefs.h"
#include <cstdlib>
@@ -9,26 +9,26 @@
namespace hfst {
namespace implementations {
- HfstAlphabet::HfstAlphabet() {
+ SfstAlphabet::SfstAlphabet() {
add(hfst::internal_epsilon.c_str(),0);
add(hfst::internal_unknown.c_str(),1);
add(hfst::internal_identity.c_str(),2);
}
- HfstAlphabet::HfstAlphabet(const HfstAlphabet &alpha)
+ SfstAlphabet::SfstAlphabet(const SfstAlphabet &alpha)
{
for (CharMap::const_iterator it = alpha.cm.begin(); it != alpha.cm.end(); it++) {
- cm[it->first] = basic::fst_strdup(it->second);
+ cm[it->first] = sfst_basic::fst_strdup(it->second);
}
for (SymbolMap::const_iterator it = alpha.sm.begin(); it != alpha.sm.end(); it++) {
- sm[basic::fst_strdup(it->first)] = it->second;
+ sm[sfst_basic::fst_strdup(it->first)] = it->second;
}
for (NumberPairSet::const_iterator it = alpha.pairs.begin(); it != alpha.pairs.end(); it++) {
pairs.insert(NumberPair(it->first, it->second));
}
}
- HfstAlphabet::~HfstAlphabet() {
+ SfstAlphabet::~SfstAlphabet() {
char **s=new char*[cm.size()];
pairs.clear();
sm.clear();
@@ -43,42 +43,42 @@ namespace hfst {
delete[] s;
}
- HfstAlphabet::const_iterator HfstAlphabet::begin() const { return pairs.begin(); }
- HfstAlphabet::const_iterator HfstAlphabet::end() const { return pairs.end(); };
- size_t HfstAlphabet::size() const { return pairs.size(); };
+ SfstAlphabet::const_iterator SfstAlphabet::begin() const { return pairs.begin(); }
+ SfstAlphabet::const_iterator SfstAlphabet::end() const { return pairs.end(); };
+ size_t SfstAlphabet::size() const { return pairs.size(); };
- //bool HfstAlphabet::contains_special_symbols(StringPair sp);
+ //bool SfstAlphabet::contains_special_symbols(StringPair sp);
- void HfstAlphabet::print_pairs(FILE *file) {
+ void SfstAlphabet::print_pairs(FILE *file) {
for (NumberPairSet::const_iterator it = pairs.begin(); it != pairs.end(); it++)
fprintf(file, "%s:%s\n", code2symbol(it->first), code2symbol(it->second));
}
- void HfstAlphabet::print() {
+ void SfstAlphabet::print() {
printf("alphabet..\n");
for( CharMap::iterator it=cm.begin(); it!=cm.end(); it++ )
printf("%i\t%s\n",it->first,it->second);
printf("..alphabet\n");
}
- void HfstAlphabet::insert(NumberPair sp) {
+ void SfstAlphabet::insert(NumberPair sp) {
/* check special symbols */ pairs.insert(sp); };
- void HfstAlphabet::clear_pairs() { pairs.clear(); };
- HfstAlphabet::CharMap HfstAlphabet::get_char_map() { return cm; };
+ void SfstAlphabet::clear_pairs() { pairs.clear(); };
+ SfstAlphabet::CharMap SfstAlphabet::get_char_map() { return cm; };
- void HfstAlphabet::add( const char *symbol, unsigned int c ) {
- char *s = basic::fst_strdup(symbol);
+ void SfstAlphabet::add( const char *symbol, unsigned int c ) {
+ char *s = sfst_basic::fst_strdup(symbol);
cm[c] = s;
sm[s] = c;
}
- int HfstAlphabet::symbol2code( const char * s ) const {
+ int SfstAlphabet::symbol2code( const char * s ) const {
SymbolMap::const_iterator p = sm.find(s);
if (p != sm.end()) return p->second;
- return EOF;
+ return EOF;
}
- const char *HfstAlphabet::code2symbol( unsigned int c ) const {
+ const char *SfstAlphabet::code2symbol( unsigned int c ) const {
CharMap::const_iterator p=cm.find(c);
if (p == cm.end())
return NULL;
@@ -86,7 +86,7 @@ namespace hfst {
return p->second;
}
- unsigned int HfstAlphabet::add_symbol(const char * symbol) {
+ unsigned int SfstAlphabet::add_symbol(const char * symbol) {
if (sm.find(symbol) != sm.end())
return sm[symbol];
@@ -100,7 +100,7 @@ namespace hfst {
throw "Error: too many symbols in transducer definition";
}
- void HfstAlphabet::add_symbol( const char *symbol, unsigned int c )
+ void SfstAlphabet::add_symbol( const char *symbol, unsigned int c )
{
// check whether the symbol was previously defined
@@ -134,7 +134,7 @@ namespace hfst {
}
}
- void HfstAlphabet::complement( std::vector<unsigned int> &sym ) {
+ void SfstAlphabet::complement( std::vector<unsigned int> &sym ) {
std::vector<unsigned int> result;
for( CharMap::const_iterator it=cm.begin(); it!=cm.end(); it++ ) {
unsigned int c = it->first;
@@ -161,7 +161,7 @@ namespace hfst {
/* */
/*******************************************************************/
- int HfstAlphabet::next_mcsym( char* &string, bool insert )
+ int SfstAlphabet::next_mcsym( char* &string, bool insert )
{
char *start=string;
@@ -203,7 +203,7 @@ namespace hfst {
return EOF;
}
- int HfstAlphabet::next_code( char* &string, bool extended, bool insert )
+ int SfstAlphabet::next_code( char* &string, bool extended, bool insert )
{
if (*string == 0)
@@ -218,8 +218,8 @@ namespace hfst {
//if (utf8) {
{
- unsigned int c = hfst_utf8::utf8toint( &string );
- return (int)add_symbol(hfst_utf8::int2utf8(c));
+ unsigned int c = sfst_utf8::utf8toint( &string );
+ return (int)add_symbol(sfst_utf8::int2utf8(c));
}
//}
/*else {
@@ -231,7 +231,7 @@ namespace hfst {
}*/
}
- std::pair<unsigned int, unsigned int> HfstAlphabet::next_label(char * &string, bool extended)
+ std::pair<unsigned int, unsigned int> SfstAlphabet::next_label(char * &string, bool extended)
{
// read first character
int c = next_code( string, extended );
@@ -258,7 +258,7 @@ namespace hfst {
std::pair<unsigned int, unsigned int> retval(lc, (unsigned int)c);
if (retval.first == 0 && retval.second == 0)
return next_label(string, extended); // ignore epsilon transitions
- return retval;
+ return retval;
}
}
@@ -271,12 +271,12 @@ main(int argc, char** argv)
std::cout << "Unit tests for " __FILE__ ":";
std::cout << std::endl << "constructors: ";
std::cout << " default()...";
- HfstAlphabet defaultAlpha;
+ SfstAlphabet defaultAlpha;
std::cout << " (copy)...";
- HfstAlphabet copyAlpha(defaultAlpha);
+ SfstAlphabet copyAlpha(defaultAlpha);
std::cout << std::endl << "destructor: ";
- delete new HfstAlphabet();
- delete new HfstAlphabet(defaultAlpha);
+ delete new SfstAlphabet();
+ delete new SfstAlphabet(defaultAlpha);
std::cout << std::endl << "rest skipped...";
std::cout << "ok" << std::endl;
return EXIT_SUCCESS;
diff --git a/tools/src/HfstAlphabet.h b/libhfst/src/parsers/SfstAlphabet.h
similarity index 91%
rename from tools/src/HfstAlphabet.h
rename to libhfst/src/parsers/SfstAlphabet.h
index 1187234..5b60a8e 100644
--- a/tools/src/HfstAlphabet.h
+++ b/libhfst/src/parsers/SfstAlphabet.h
@@ -1,5 +1,5 @@
-#ifndef _HFST_ALPHABET_H_
-#define _HFST_ALPHABET_H_
+#ifndef _SFST_ALPHABET_H_
+#define _SFST_ALPHABET_H_
#if HAVE_CONFIG_H
# include <config.h>
@@ -21,14 +21,14 @@
#include <string.h>
#include <stdio.h>
-/* @file HfstAlphabet.h
- \brief Declaration of class HfstAlphabet. */
+/* @file SfstAlphabet.h
+ \brief Declaration of class SfstAlphabet. */
namespace hfst {
namespace implementations {
/* copied from SFST's alphabet.h|C */
- class HfstAlphabet {
+ class SfstAlphabet {
public:
typedef std::pair<unsigned int,unsigned int> NumberPair;
@@ -64,9 +64,9 @@ namespace hfst {
NumberPairSet pairs;
public:
- HfstAlphabet();
- HfstAlphabet(const HfstAlphabet &alpha);
- ~HfstAlphabet();
+ SfstAlphabet();
+ SfstAlphabet(const SfstAlphabet &alpha);
+ ~SfstAlphabet();
typedef NumberPairSet::const_iterator const_iterator;
const_iterator begin() const;
diff --git a/tools/src/HfstBasic.cc b/libhfst/src/parsers/SfstBasic.cc
similarity index 94%
rename from tools/src/HfstBasic.cc
rename to libhfst/src/parsers/SfstBasic.cc
index 466dc29..d7147f8 100644
--- a/tools/src/HfstBasic.cc
+++ b/libhfst/src/parsers/SfstBasic.cc
@@ -1,10 +1,10 @@
#include <stdlib.h>
#include <string.h>
-#include "HfstBasic.h"
+#include "SfstBasic.h"
+#include "../HfstExceptionDefs.h"
-namespace hfst {
- namespace basic {
+ namespace sfst_basic {
bool Switch_Bytes=false;
@@ -21,7 +21,7 @@ namespace hfst {
char* pStringCopy = (char*)malloc(strlen(pString) + 1);
if (pStringCopy == NULL) {
fprintf(stderr, "\nError: out of memory (malloc failed)\naborted.\n");
- exit(1);
+ throw HfstException();
}
strcpy(pStringCopy, pString);
return pStringCopy;
@@ -73,4 +73,4 @@ namespace hfst {
}
}
-}
+
diff --git a/tools/src/HfstBasic.h b/libhfst/src/parsers/SfstBasic.h
similarity index 71%
rename from tools/src/HfstBasic.h
rename to libhfst/src/parsers/SfstBasic.h
index 4f5ae54..0f1b796 100644
--- a/tools/src/HfstBasic.h
+++ b/libhfst/src/parsers/SfstBasic.h
@@ -1,13 +1,12 @@
-#ifndef _HFST_BASIC_H_
-#define _HFST_BASIC_H_
+#ifndef _SFST_BASIC_H_
+#define _SFST_BASIC_H_
#include <stdio.h>
-/* @file HfstBasic.h
+/* @file SfstBasic.h
\brief Basic functions for string handling. */
-namespace hfst {
-namespace basic {
+namespace sfst_basic {
extern bool Switch_Bytes;
@@ -16,5 +15,4 @@ namespace basic {
size_t read_num( void *p, size_t size, FILE *file );
}
-}
#endif
diff --git a/tools/src/HfstCompiler.cc b/libhfst/src/parsers/SfstCompiler.cc
similarity index 78%
rename from tools/src/HfstCompiler.cc
rename to libhfst/src/parsers/SfstCompiler.cc
index 82d1724..06affd7 100644
--- a/tools/src/HfstCompiler.cc
+++ b/libhfst/src/parsers/SfstCompiler.cc
@@ -13,8 +13,8 @@
/*
This file contains functions that are needed by the SFST programming
- language parser defined in the file 'hfst-compiler.yy'. The parser is
- used by the command line program 'hfst-calculate'.
+ language parser defined in the file 'sfst-compiler.yy'. The parser is
+ used by the command line program 'hfst-calculate'.
This file is based on SFST's file 'interface.C'. Some functions are
copied as such and some are less or more modified so that they work
@@ -22,13 +22,13 @@
*/
-#include "HfstCompiler.h"
-#include "HfstUtf8.h"
-#include "HfstBasic.h"
+#include "SfstCompiler.h"
+#include "SfstUtf8.h"
+#include "SfstBasic.h"
#include "HfstTransducer.h"
#include "HfstInputStream.h"
#include "HfstOutputStream.h"
-#include "implementations/HfstTransitionGraph.h"
+#include "implementations/HfstBasicTransducer.h"
using hfst::implementations::HfstTransitionGraph;
using hfst::implementations::HfstBasicTransducer;
@@ -36,28 +36,80 @@ using hfst::implementations::HfstState;
using std::cerr;
+extern FILE *sfstin;
+int sfstparse( void );
+
namespace hfst
{
+ SfstCompiler * sfst_compiler = NULL;
+
+ SfstCompiler::SfstCompiler( ImplementationType type, bool verbose /*=false*/ )
+ {
+ result_ = NULL;
+ Verbose = verbose;
+ Alphabet_Defined = false;
+ compiler_type = type;
+ filename = "";
+ foldername = "";
+ switch_ = 0;
+ sfst_compiler = this;
+ };
+
typedef std::pair<unsigned int, unsigned int> NumberPair;
typedef std::set<NumberPair> NumberPairSet;
typedef std::vector<NumberPair> NumberPairVector;
+
+ void SfstCompiler::set_result(HfstTransducer * tr)
+ {
+ result_ = tr;
+ }
- HfstTransducer * HfstCompiler::make_transducer
+ HfstTransducer * SfstCompiler::get_result()
+ {
+ return result_;
+ }
+
+ void SfstCompiler::set_input(FILE * input)
+ {
+ sfstin = input;
+ }
+
+ void SfstCompiler::set_filename(const std::string & name)
+ {
+ filename = name;
+ }
+
+ void SfstCompiler::set_foldername(const std::string & name)
+ {
+ foldername = name;
+ }
+
+ void SfstCompiler::set_switch(int value)
+ {
+ switch_ = value;
+ }
+
+ void SfstCompiler::parse()
+ {
+ sfstparse();
+ }
+
+ HfstTransducer * SfstCompiler::make_transducer
(Range *r1, Range *r2, ImplementationType type)
{
StringPairSet sps;
if (r1 == NULL || r2 == NULL) {
if (!Alphabet_Defined) {
- fprintf(stderr,
+ fprintf(stderr,
"ERROR: The wildcard symbol '.'"
" requires the definition of an alphabet");
- exit(1);
+ throw HfstException();
}
// one of the ranges was '.'
- for(HfstAlphabet::const_iterator it=TheAlphabet.begin();
+ for(SfstAlphabet::const_iterator it=TheAlphabet.begin();
it!=TheAlphabet.end(); it++) {
if ((r1 == NULL || in_range(it->first, r1)) &&
(r2 == NULL || in_range(it->second, r2))) {
@@ -66,7 +118,7 @@ namespace hfst
std::string(TheAlphabet.code2symbol(it->first)),
std::string(TheAlphabet.code2symbol(it->second)) ) );
}
- }
+ }
}
else {
for (;;) {
@@ -86,7 +138,7 @@ namespace hfst
return new HfstTransducer(sps, type);
}
- HfstTransducer * HfstCompiler::new_transducer( Range *r1, Range *r2, ImplementationType type )
+ HfstTransducer * SfstCompiler::new_transducer( Range *r1, Range *r2, ImplementationType type )
{
HfstTransducer * t = make_transducer(r1, r2, type);
if (r1 != r2)
@@ -95,36 +147,36 @@ namespace hfst
return t;
}
- Character HfstCompiler::character_code( unsigned int uc ) {
+ Character SfstCompiler::character_code( unsigned int uc ) {
// UTF-8 is always used
- return symbol_code(basic::fst_strdup(hfst_utf8::int2utf8(uc)));
+ return symbol_code(sfst_basic::fst_strdup(sfst_utf8::int2utf8(uc)));
}
- void HfstCompiler::free_values( Range *r ) {
+ void SfstCompiler::free_values( Range *r ) {
if (r) {
free_values(r->next);
delete r;
}
}
- void HfstCompiler::free_values( Ranges *r ) {
+ void SfstCompiler::free_values( Ranges *r ) {
if (r) {
free_values(r->next);
delete r;
}
}
- void HfstCompiler::error( const char *message ) {
+ void SfstCompiler::error( const char *message ) {
std::cerr << "\nError: " << message << "\naborted.\n";
- exit(1);
+ throw HfstException();
}
- void HfstCompiler::error2( const char *message, char *input ) {
+ void SfstCompiler::error2( const char *message, char *input ) {
std::cerr << "\nError: " << message << ": " << input << "\naborted.\n";
- exit(1);
+ throw HfstException();
}
- Character HfstCompiler::symbol_code( char *symbol )
+ Character SfstCompiler::symbol_code( char *symbol )
{ // In SFST programming language epsilon is denoted as "<>"
// but in HFST as "@_EPSILON_SYMBOL_@". That is why it must be
// treated separately here.
@@ -137,11 +189,11 @@ namespace hfst
return (Character)c;
}
- unsigned int HfstCompiler::utf8toint( char *s ) {
- return hfst_utf8::utf8toint(s);
+ unsigned int SfstCompiler::utf8toint( char *s ) {
+ return sfst_utf8::utf8toint(s);
}
- bool HfstCompiler::in_range( unsigned int c, Range *r ) {
+ bool SfstCompiler::in_range( unsigned int c, Range *r ) {
while (r) {
if (r->character == c)
return true;
@@ -150,33 +202,33 @@ namespace hfst
return false;
}
- Range * HfstCompiler::add_value( Character c, Range *r) {
+ Range * SfstCompiler::add_value( Character c, Range *r) {
Range *result=new Range;
result->character = c;
result->next = r;
return result;
}
- Range * HfstCompiler::add_values( unsigned int c1, unsigned int c2, Range *r) {
+ Range * SfstCompiler::add_values( unsigned int c1, unsigned int c2, Range *r) {
for( unsigned int c=c2; c>=c1; c-- )
r = add_value(character_code(c), r);
return r;
}
- Range * HfstCompiler::append_values( Range *r2, Range *r ) {
+ Range * SfstCompiler::append_values( Range *r2, Range *r ) {
if (r2 == NULL)
return r;
return add_value(r2->character, append_values(r2->next, r));
}
- Ranges * HfstCompiler::add_range( Range *r, Ranges *l ) {
+ Ranges * SfstCompiler::add_range( Range *r, Ranges *l ) {
Ranges *result = new Ranges;
result->range = r;
result->next = l;
return result;
}
- Range * HfstCompiler::complement_range( Range *r ) {
+ Range * SfstCompiler::complement_range( Range *r ) {
std::vector<Character> sym;
for( Range *p=r; p; p=p->next)
sym.push_back( p->character );
@@ -198,7 +250,7 @@ namespace hfst
return result;
}
- bool HfstCompiler::def_var( char *name, HfstTransducer *t ) {
+ bool SfstCompiler::def_var( char *name, HfstTransducer *t ) {
// delete the old value of the variable
VarMap::iterator it=VM.find(name);
if (it != VM.end()) {
@@ -217,30 +269,30 @@ namespace hfst
return false;
}
- bool HfstCompiler::def_rvar( char *name, HfstTransducer *t ) {
+ bool SfstCompiler::def_rvar( char *name, HfstTransducer *t ) {
if (t->is_cyclic())
error2("cyclic transducer assigned to", name);
return def_var( name, t );
}
- HfstTransducer * HfstCompiler::var_value( char *name ) {
+ HfstTransducer * SfstCompiler::var_value( char *name ) {
VarMap::iterator it=VM.find(name);
if (it == VM.end()) {
printf("undefined variable %s\n", name);
- exit(1);
+ throw HfstException();
}
free(name);
return new HfstTransducer(*(it->second));
}
- HfstTransducer * HfstCompiler::rvar_value( char *name, ImplementationType type ) {
+ HfstTransducer * SfstCompiler::rvar_value( char *name, ImplementationType type ) {
if (RS.find(name) == RS.end())
- RS.insert(basic::fst_strdup(name));
+ RS.insert(sfst_basic::fst_strdup(name));
Range *r=add_value(symbol_code(name), NULL);
- return new_transducer(r,r,type);
+ return new_transducer(r,r,type);
}
- bool HfstCompiler::def_svar( char *name, Range *r ) {
+ bool SfstCompiler::def_svar( char *name, Range *r ) {
// delete the old value of the variable
SVarMap::iterator it=SVM.find(name);
if (it != SVM.end()) {
@@ -254,13 +306,13 @@ namespace hfst
return r == NULL;
}
- Range *HfstCompiler::copy_values( const Range *r ) {
+ Range *SfstCompiler::copy_values( const Range *r ) {
if (r == NULL)
return NULL;
return add_value( r->character, copy_values(r->next));
}
- Range *HfstCompiler::svar_value( char *name ) {
+ Range *SfstCompiler::svar_value( char *name ) {
SVarMap::iterator it=SVM.find(name);
if (it == SVM.end())
error2("undefined variable", name);
@@ -268,47 +320,47 @@ namespace hfst
return copy_values(it->second);
}
- Range *HfstCompiler::rsvar_value( char *name ) {
+ Range *SfstCompiler::rsvar_value( char *name ) {
if (RSS.find(name) == RSS.end())
- RSS.insert(basic::fst_strdup(name));
+ RSS.insert(sfst_basic::fst_strdup(name));
return add_value(symbol_code(name), NULL);
}
// HERE...
- HfstTransducer * HfstCompiler::insert_freely(HfstTransducer *t, Character input, Character output) {
+ HfstTransducer * SfstCompiler::insert_freely(HfstTransducer *t, Character input, Character output) {
t->insert_freely(hfst::StringPair(TheAlphabet.code2symbol(input), TheAlphabet.code2symbol(output)));
return t;
}
- HfstTransducer * HfstCompiler::substitute(HfstTransducer *t, Character old_char, Character new_char) {
+ HfstTransducer * SfstCompiler::substitute(HfstTransducer *t, Character old_char, Character new_char) {
t->substitute(std::string(TheAlphabet.code2symbol(old_char)), std::string(TheAlphabet.code2symbol(new_char)));
return t;
}
- HfstTransducer * HfstCompiler::substitute(HfstTransducer *t, Character old_char_in, Character old_char_out,
+ HfstTransducer * SfstCompiler::substitute(HfstTransducer *t, Character old_char_in, Character old_char_out,
Character new_char_in, Character new_char_out) {
- t->substitute( hfst::StringPair(TheAlphabet.code2symbol(old_char_in),
+ t->substitute( hfst::StringPair(TheAlphabet.code2symbol(old_char_in),
TheAlphabet.code2symbol(old_char_out)),
- hfst::StringPair(TheAlphabet.code2symbol(new_char_in),
+ hfst::StringPair(TheAlphabet.code2symbol(new_char_in),
TheAlphabet.code2symbol(new_char_out)) );
return t;
}
- HfstTransducer * HfstCompiler::substitute(HfstTransducer *t, Character old_char_in, Character old_char_out,
+ HfstTransducer * SfstCompiler::substitute(HfstTransducer *t, Character old_char_in, Character old_char_out,
HfstTransducer *tr) {
t->substitute( hfst::StringPair(TheAlphabet.code2symbol(old_char_in),
TheAlphabet.code2symbol(old_char_out)), *tr );
return t;
}
- Contexts *HfstCompiler::make_context( HfstTransducer *l, HfstTransducer *r )
+ Contexts *SfstCompiler::make_context( HfstTransducer *l, HfstTransducer *r )
{
if (l != NULL && r != NULL) {
if (l->get_type() != r->get_type()) {
- fprintf(stderr, "ERROR: in hfst-compiler.yy:"
+ fprintf(stderr, "ERROR: in sfst-compiler.yy:"
" context transducers do not have the same type.\n");
- exit(1);
+ throw HfstException();
}
}
@@ -331,23 +383,23 @@ namespace hfst
return c;
}
- Contexts *HfstCompiler::add_context( Contexts *nc, Contexts *c )
+ Contexts *SfstCompiler::add_context( Contexts *nc, Contexts *c )
{
- if (nc->left->get_type() != c->left->get_type() ||
+ if (nc->left->get_type() != c->left->get_type() ||
nc->right->get_type() != c->right->get_type() ) {
- fprintf(stderr, "ERROR: in hfst-compiler.yy:"
+ fprintf(stderr, "ERROR: in sfst-compiler.yy:"
" context transducers do not have the same type.\n");
- exit(1);
+ throw HfstException();
}
nc->next = c;
return nc;
}
- void HfstCompiler::warn(const char *msg) {
+ void SfstCompiler::warn(const char *msg) {
std::cerr << "\nWarning: " << msg << "!\n";
}
- HfstTransducer * HfstCompiler::negation( HfstTransducer *t )
+ HfstTransducer * SfstCompiler::negation( HfstTransducer *t )
{
if (RS.size() > 0 || RSS.size() > 0)
warn("agreement operation inside of negation");
@@ -356,8 +408,8 @@ namespace hfst
// go through all symbol pairs in TheAlphabet and copy them to sps
StringPairSet sps;
- for( HfstAlphabet::const_iterator it=TheAlphabet.begin(); it!=TheAlphabet.end(); it++ ) {
- HfstAlphabet::NumberPair l=*it;
+ for( SfstAlphabet::const_iterator it=TheAlphabet.begin(); it!=TheAlphabet.end(); it++ ) {
+ SfstAlphabet::NumberPair l=*it;
sps.insert(StringPair( TheAlphabet.code2symbol(l.first),
TheAlphabet.code2symbol(l.second)) );
}
@@ -370,7 +422,7 @@ namespace hfst
}
- HfstTransducer * HfstCompiler::explode( HfstTransducer *t ) {
+ HfstTransducer * SfstCompiler::explode( HfstTransducer *t ) {
//fprintf(stderr,"explode...\n");
if (RS.size() == 0 && RSS.size() == 0) {
@@ -414,9 +466,9 @@ namespace hfst
delete vt;
// transform paths to a vector of transducers
- for (HfstTwoLevelPaths::const_iterator it = paths.begin(); it != paths.end(); it++) {
+ for (HfstTwoLevelPaths::const_iterator it = paths.begin(); it != paths.end(); it++) {
HfstTransducer * path = new HfstTransducer(it->second, t->get_type());
- //HfstTransducer * path
+ //HfstTransducer * path
// = new HfstTransducer(wp.istring, wp.ostring, TOK, t->get_type());
path->set_final_weights(it->first);
transducer_paths.push_back(path);
@@ -433,7 +485,7 @@ namespace hfst
//cerr << ti;
ti.substitute(StringPair(std::string(name[i]), std::string(name[i])), *(transducer_paths[j]));
//fprintf(stderr, " ...substituted\n");
- delete transducer_paths[j];
+ delete transducer_paths[j];
nt->disjunct(ti);
}
free(name[i]); // svar_value was given a copy of name[i], so the value is freed here
@@ -451,7 +503,7 @@ namespace hfst
// replace all agreement variables
for( size_t i=0; i<name.size(); i++ ) {
//printf("substituting range agreement variable \"%s\"\n", name[i]);
- HfstTransducer *nt = new HfstTransducer(t->get_type());
+ HfstTransducer *nt = new HfstTransducer(t->get_type());
Range *r=svar_value(strdup(name[i])); // svar_value frees its argument
// insert each character
@@ -461,7 +513,7 @@ namespace hfst
HfstTransducer ti(*t);
// agreement variable marker should always appear on both sides of the tape..
//printf("substituting agreement range variable %s with %s\n", name[i], TheAlphabet.code2symbol(r->character));
- ti.substitute(std::string(name[i]), TheAlphabet.code2symbol(r->character));
+ ti.substitute(std::string(name[i]), TheAlphabet.code2symbol(r->character));
nt->disjunct(ti);
Range *next = r->next;
@@ -471,19 +523,19 @@ namespace hfst
free(name[i]); // svar_value was given a copy of name[i], so the value is freed here
delete t;
t = nt;
- }
+ }
return t;
}
- HfstTransducer * HfstCompiler::restriction( HfstTransducer * t, Twol_Type type, Contexts *c, int direction ) {
+ HfstTransducer * SfstCompiler::restriction( HfstTransducer * t, Twol_Type type, Contexts *c, int direction ) {
StringPairSet sps;
- for( HfstAlphabet::const_iterator it=TheAlphabet.begin(); it!=TheAlphabet.end(); it++ ) {
- HfstAlphabet::NumberPair l=*it;
+ for( SfstAlphabet::const_iterator it=TheAlphabet.begin(); it!=TheAlphabet.end(); it++ ) {
+ SfstAlphabet::NumberPair l=*it;
sps.insert(StringPair( TheAlphabet.code2symbol(l.first),
TheAlphabet.code2symbol(l.second)) );
- }
+ }
HfstTransducerPairVector contexts;
Contexts *p = c;
@@ -496,10 +548,10 @@ namespace hfst
return new HfstTransducer( hfst::rules::restriction(
contexts, *t, sps,
- (hfst::rules::TwolType)type, direction ) );
+ (hfst::rules::TwolType)type, direction ) );
}
- HfstTransducer * HfstCompiler::make_rule( HfstTransducer * lc, Range * lower_range, Twol_Type type,
+ HfstTransducer * SfstCompiler::make_rule( HfstTransducer * lc, Range * lower_range, Twol_Type type,
Range * upper_range, HfstTransducer * rc, ImplementationType implementation_type ) {
if (RS.size() > 0 || RSS.size() > 0)
@@ -508,7 +560,7 @@ namespace hfst
if (!Alphabet_Defined) {
fprintf(stderr, "\nERROR:"
" Two level rules require the definition of an alphabet!\n");
- exit(1);
+ throw HfstException();
}
if (lc == NULL)
@@ -519,11 +571,11 @@ namespace hfst
HfstTransducerPair tr_pair(*(lc), *(rc));
StringPairSet sps;
- for( HfstAlphabet::const_iterator it=TheAlphabet.begin(); it!=TheAlphabet.end(); it++ ) {
- HfstAlphabet::NumberPair l=*it;
+ for( SfstAlphabet::const_iterator it=TheAlphabet.begin(); it!=TheAlphabet.end(); it++ ) {
+ SfstAlphabet::NumberPair l=*it;
sps.insert(StringPair( TheAlphabet.code2symbol(l.first),
TheAlphabet.code2symbol(l.second)) );
- }
+ }
StringPairSet mappings;
Range * r1 = lower_range;
@@ -533,11 +585,11 @@ namespace hfst
if (!Alphabet_Defined) {
fprintf(stderr, "ERROR: The wildcard symbol '.'"
" requires the definition of an alphabet");
- exit(1);
+ throw HfstException();
}
// one of the ranges was '.'
- for(HfstAlphabet::const_iterator it=TheAlphabet.begin();
+ for(SfstAlphabet::const_iterator it=TheAlphabet.begin();
it!=TheAlphabet.end(); it++) {
if ((r1 == NULL || in_range(it->first, r1)) &&
(r2 == NULL || in_range(it->second, r2))) {
@@ -545,7 +597,7 @@ namespace hfst
TheAlphabet.code2symbol(it->first),
TheAlphabet.code2symbol(it->second) ) );
}
- }
+ }
}
else {
for (;;) {
@@ -577,7 +629,7 @@ namespace hfst
}
- HfstTransducer * HfstCompiler::read_words(const char *folder, char *filename,
+ HfstTransducer * SfstCompiler::read_words(const char *folder, char *filename,
ImplementationType type) {
std::string filestr("");
@@ -600,7 +652,7 @@ namespace hfst
HfstTransducer * retval_hfst = NULL;
hfst::implementations::HfstBasicTransducer retval_fsm;
- if (type != FOMA_TYPE &&
+ if (type != FOMA_TYPE &&
type != TROPICAL_OPENFST_TYPE &&
type != LOG_OPENFST_TYPE)
retval_hfst = new HfstTransducer(type);
@@ -625,11 +677,11 @@ namespace hfst
StringPairVector spv;
char *bufptr = buffer;
- std::pair<unsigned int, unsigned int> np =
+ std::pair<unsigned int, unsigned int> np =
TheAlphabet.next_label(bufptr, true);
while (np.first != 0 || np.second != 0) {
spv.push_back(StringPair
- (std::string(TheAlphabet.code2symbol(np.first)),
+ (std::string(TheAlphabet.code2symbol(np.first)),
std::string(TheAlphabet.code2symbol(np.second)) ) );
np = TheAlphabet.next_label(bufptr, true);
}
@@ -658,7 +710,7 @@ namespace hfst
}
}
- HfstTransducer * HfstCompiler::read_transducer(const char *folder, char *filename, ImplementationType type) {
+ HfstTransducer * SfstCompiler::read_transducer(const char *folder, char *filename, ImplementationType type) {
std::string filestr("");
if (NULL != folder) {
@@ -680,7 +732,7 @@ namespace hfst
return t;
}
- void HfstCompiler::write_to_file(HfstTransducer *t, const char* folder, char* filename) {
+ void SfstCompiler::write_to_file(HfstTransducer *t, const char* folder, char* filename) {
std::string filestr("");
if (NULL != folder) {
@@ -696,16 +748,16 @@ namespace hfst
return;
}
- HfstTransducer * HfstCompiler::replace_in_context(HfstTransducer * mapping, Repl_Type repl_type, Contexts *contexts, bool optional) {
+ HfstTransducer * SfstCompiler::replace_in_context(HfstTransducer * mapping, Repl_Type repl_type, Contexts *contexts, bool optional) {
HfstTransducerPair tr_pair(*(contexts->left), *(contexts->right));
StringPairSet sps;
- for( HfstAlphabet::const_iterator it=TheAlphabet.begin(); it!=TheAlphabet.end(); it++ ) {
- HfstAlphabet::NumberPair l=*it;
+ for( SfstAlphabet::const_iterator it=TheAlphabet.begin(); it!=TheAlphabet.end(); it++ ) {
+ SfstAlphabet::NumberPair l=*it;
sps.insert(StringPair( TheAlphabet.code2symbol(l.first),
TheAlphabet.code2symbol(l.second)) );
- }
- switch (repl_type)
+ }
+ switch (repl_type)
{
case repl_up:
return new HfstTransducer(rules::replace_up(tr_pair, *mapping, optional, sps));
@@ -723,15 +775,15 @@ namespace hfst
return NULL;
}
- HfstTransducer * HfstCompiler::replace(HfstTransducer * mapping, Repl_Type repl_type, bool optional) {
+ HfstTransducer * SfstCompiler::replace(HfstTransducer * mapping, Repl_Type repl_type, bool optional) {
StringPairSet sps;
- for( HfstAlphabet::const_iterator it=TheAlphabet.begin(); it!=TheAlphabet.end(); it++ ) {
- HfstAlphabet::NumberPair l=*it;
+ for( SfstAlphabet::const_iterator it=TheAlphabet.begin(); it!=TheAlphabet.end(); it++ ) {
+ SfstAlphabet::NumberPair l=*it;
sps.insert(StringPair( TheAlphabet.code2symbol(l.first),
TheAlphabet.code2symbol(l.second)) );
- }
- switch (repl_type)
+ }
+ switch (repl_type)
{
case repl_up:
return new HfstTransducer(rules::replace_up(*mapping, optional, sps));
@@ -741,13 +793,13 @@ namespace hfst
break;
default:
fprintf(stderr, "ERROR: invalid replace type requested\n");
- exit(1);
+ throw HfstException();
}
- fprintf(stderr, "ERROR: in function HfstCompiler::replace\n");
- exit(1);
+ fprintf(stderr, "ERROR: in function SfstCompiler::replace\n");
+ throw HfstException();
}
- HfstTransducer * HfstCompiler::make_mapping( Ranges *list1, Ranges *list2, ImplementationType type ) {
+ HfstTransducer * SfstCompiler::make_mapping( Ranges *list1, Ranges *list2, ImplementationType type ) {
Ranges *l1=list1;
Ranges *l2=list2;
@@ -757,7 +809,7 @@ namespace hfst
StringPairSet sps;
for( Range *r1=l1->range; r1; r1=r1->next )
for( Range *r2=l2->range; r2; r2=r2->next ) {
- sps.insert(StringPair(TheAlphabet.code2symbol(r1->character),
+ sps.insert(StringPair(TheAlphabet.code2symbol(r1->character),
TheAlphabet.code2symbol(r2->character)) );
}
spsv.push_back(sps);
@@ -767,7 +819,7 @@ namespace hfst
while (l1) {
StringPairSet sps;
for( Range *r1=l1->range; r1; r1=r1->next ) {
- sps.insert(StringPair(TheAlphabet.code2symbol(r1->character),
+ sps.insert(StringPair(TheAlphabet.code2symbol(r1->character),
TheAlphabet.code2symbol(0)) );
}
spsv.push_back(sps);
@@ -776,7 +828,7 @@ namespace hfst
while (l2) {
StringPairSet sps;
for( Range *r2=l2->range; r2; r2=r2->next ) {
- sps.insert(StringPair(TheAlphabet.code2symbol(0),
+ sps.insert(StringPair(TheAlphabet.code2symbol(0),
TheAlphabet.code2symbol(r2->character)) );
}
spsv.push_back(sps);
@@ -791,7 +843,7 @@ namespace hfst
}
- HfstTransducer * HfstCompiler::result( HfstTransducer *t, bool switch_flag) {
+ HfstTransducer * SfstCompiler::result( HfstTransducer *t, bool switch_flag) {
t = explode(t);
@@ -814,7 +866,7 @@ namespace hfst
return t;
}
- void HfstCompiler::def_alphabet( HfstTransducer *tr )
+ void SfstCompiler::def_alphabet( HfstTransducer *tr )
{
tr = explode(tr);
tr->minimize();
@@ -834,13 +886,13 @@ namespace hfst
if (it->first.compare("<>") == 0)
inumber=0;
else
- inumber = TheAlphabet.symbol2code(it->first.c_str());
+ inumber = TheAlphabet.symbol2code(it->first.c_str());
if (it->second.compare("<>") == 0)
onumber=0;
else
onumber = TheAlphabet.symbol2code(it->second.c_str());
- TheAlphabet.insert(HfstAlphabet::NumberPair(inumber,onumber));
+ TheAlphabet.insert(SfstAlphabet::NumberPair(inumber,onumber));
}
Alphabet_Defined = 1;
}
@@ -852,11 +904,11 @@ namespace hfst
for (HfstBasicTransducer::const_iterator it = t.begin();
it != t.end(); it++)
{
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
+ for (hfst::implementations::HfstBasicTransitions::const_iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
- TheAlphabet.insert(HfstAlphabet::NumberPair
+ TheAlphabet.insert(SfstAlphabet::NumberPair
(
TheAlphabet.symbol2code
(tr_it->get_input_symbol().c_str())
diff --git a/tools/src/HfstCompiler.h b/libhfst/src/parsers/SfstCompiler.h
similarity index 89%
rename from tools/src/HfstCompiler.h
rename to libhfst/src/parsers/SfstCompiler.h
index 862b0c0..c195b55 100644
--- a/tools/src/HfstCompiler.h
+++ b/libhfst/src/parsers/SfstCompiler.h
@@ -14,13 +14,13 @@
#define _HFST_COMPILER_H_
#include "HfstDataTypes.h"
-#include "HfstAlphabet.h"
-#include <iostream>
+#include "SfstAlphabet.h"
+#include <iosfwd>
#include <fstream>
namespace hfst
{
- using implementations::HfstAlphabet;
+ using implementations::SfstAlphabet;
typedef unsigned int Character;
@@ -47,7 +47,7 @@ namespace hfst
and the HFST library and contains some extra functions needed by the parser.
Most of these functions either directly use the SFST interface or are generalized
versions of corresponding SFST functions that work on all types of HfstTransducers. */
- class HfstCompiler {
+ class SfstCompiler {
struct ltstr {
bool operator()(const char* s1, const char* s2) const
@@ -68,19 +68,28 @@ namespace hfst
SVarMap SVM;
RVarSet RS;
RVarSet RSS;
+ HfstTransducer * result_;
public:
bool Verbose;
bool Alphabet_Defined;
- HfstAlphabet TheAlphabet;
+ SfstAlphabet TheAlphabet;
ImplementationType compiler_type;
-
+ std::string filename;
+ std::string foldername;
+ int switch_;
- HfstCompiler( ImplementationType type, bool verbose=false ) :
- Verbose(verbose), Alphabet_Defined(false), compiler_type(type)
- {}
+ SfstCompiler( ImplementationType type, bool verbose=false );
public:
+ void set_result(HfstTransducer * tr);
+ HfstTransducer * get_result();
+ void set_input(FILE * input);
+ void set_filename(const std::string & name);
+ void set_foldername(const std::string & name);
+ void set_switch(int value);
+ void parse();
+
HfstTransducer * make_transducer(Range *r1, Range *r2, ImplementationType type);
static void warn(const char *msg);
HfstTransducer *new_transducer( Range*, Range*, ImplementationType );
@@ -101,15 +110,15 @@ namespace hfst
static Range *add_var_values( char *name, Range*);
Range *add_values( unsigned int, unsigned int, Range*);
static Range *append_values( Range *r2, Range *r );
- static Ranges *add_range( Range*, Ranges* );
+ static Ranges *add_range( Range*, Ranges* );
//static void add_alphabet( HfstTransducer* );
static Contexts *make_context( HfstTransducer *l, HfstTransducer *r );
- static Contexts *add_context( Contexts *nc, Contexts *c );
+ static Contexts *add_context( Contexts *nc, Contexts *c );
HfstTransducer * substitute(HfstTransducer *t, Character old_char, Character new_char);
HfstTransducer * substitute(HfstTransducer *t, Character old_char_in, Character old_char_out,
- Character new_char_in, Character new_char_out);
+ Character new_char_in, Character new_char_out);
HfstTransducer * substitute(HfstTransducer *t, Character old_char_in, Character old_char_out,
HfstTransducer *tr);
@@ -123,7 +132,7 @@ namespace hfst
HfstTransducer * replace_in_context(HfstTransducer * mapping, Repl_Type repl_type, Contexts *contexts, bool optional);
HfstTransducer * replace(HfstTransducer * mapping, Repl_Type repl_type, bool optional);
HfstTransducer * restriction( HfstTransducer * t, Twol_Type type, Contexts *c, int direction );
- HfstTransducer * make_rule( HfstTransducer * lc, Range * lower_range, Twol_Type type,
+ HfstTransducer * make_rule( HfstTransducer * lc, Range * lower_range, Twol_Type type,
Range * upper_range, HfstTransducer * rc, ImplementationType implementation_type );
void def_alphabet( HfstTransducer *a );
@@ -145,7 +154,7 @@ namespace hfst
static void error( const char *message );
static void error2( const char *message, char *input );
- };
+ };
}
#endif
diff --git a/tools/src/HfstUtf8.cc b/libhfst/src/parsers/SfstUtf8.cc
similarity index 98%
rename from tools/src/HfstUtf8.cc
rename to libhfst/src/parsers/SfstUtf8.cc
index a7f41fc..45b24ea 100644
--- a/tools/src/HfstUtf8.cc
+++ b/libhfst/src/parsers/SfstUtf8.cc
@@ -11,10 +11,9 @@
#include "string.h"
-#include "HfstUtf8.h"
+#include "SfstUtf8.h"
-namespace hfst {
-namespace hfst_utf8 {
+namespace sfst_utf8 {
const unsigned char get3LSbits=7;
const unsigned char get4LSbits=15;
@@ -149,4 +148,4 @@ namespace hfst_utf8 {
}
}
-}
+
diff --git a/tools/src/HfstUtf8.h b/libhfst/src/parsers/SfstUtf8.h
similarity index 90%
rename from tools/src/HfstUtf8.h
rename to libhfst/src/parsers/SfstUtf8.h
index b8711cd..eeb8f6c 100644
--- a/tools/src/HfstUtf8.h
+++ b/libhfst/src/parsers/SfstUtf8.h
@@ -9,19 +9,17 @@
/* */
/*******************************************************************/
-#ifndef _HFST_UTF8_H_
-#define _HFST_UTF8_H_
+#ifndef _SFST_UTF8_H_
+#define _SFST_UTF8_H_
/* @file HfstUtf8.h
\brief Functions for handling utf-8 strings. */
-namespace hfst {
-namespace hfst_utf8 {
+namespace sfst_utf8 {
unsigned int utf8toint( char *s );
unsigned int utf8toint( char **s );
char *int2utf8( unsigned int );
}
-}
#endif
diff --git a/libhfst/src/parsers/TwolcCompiler.cc b/libhfst/src/parsers/TwolcCompiler.cc
new file mode 100644
index 0000000..8e46011
--- /dev/null
+++ b/libhfst/src/parsers/TwolcCompiler.cc
@@ -0,0 +1,116 @@
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 3 of the Licence.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+#include "commandline_src/CommandLine.h"
+#include "HfstTwolcDefs.h"
+#include "io_src/InputReader.h"
+#include "grammar_defs.h"
+#include "rule_src/TwolCGrammar.h"
+#include "rule_src/OtherSymbolTransducer.h"
+#include "TwolcCompiler.h"
+
+namespace hfst {
+ namespace twolcpre1 {
+ int parse();
+ void set_input(std::istream & istr);
+ void set_output(std::ostream & ostr);
+ void set_warning_stream(std::ostream & ostr);
+ void set_error_stream(std::ostream & ostr);
+ }
+}
+
+namespace hfst {
+ namespace twolcpre2 {
+ int parse();
+ void set_input(std::istream & istr);
+ void complete_alphabet(void);
+ const HandyDeque<std::string> & get_total_alphabet_symbol_queue();
+ const HandyDeque<std::string> & get_non_alphabet_symbol_queue();
+ void set_warning_stream(std::ostream & ostr);
+ void set_error_stream(std::ostream & ostr);
+ }
+}
+
+namespace hfst {
+ namespace twolcpre3 {
+ int parse();
+ void set_input(std::istream & istr);
+ void set_grammar(TwolCGrammar * grammar);
+ TwolCGrammar * get_grammar();
+ void set_silent(bool val);
+ void set_verbose(bool val);
+ void message(const std::string &m);
+ void set_warning_stream(std::ostream & ostr);
+ void set_error_stream(std::ostream & ostr);
+ }
+}
+
+namespace hfst {
+
+ TwolcCompiler::TwolcCompiler(const CommandLine & cl, std::ostream & warn, std::ostream & error):
+ command_line(cl), warning_stream(warn), error_stream(error) {};
+
+ void TwolcCompiler::compile()
+ {
+ hfst::twolcpre1::set_input(command_line.set_input_file());
+ std::ostringstream oss1;
+ hfst::twolcpre1::set_output(oss1);
+ hfst::twolcpre1::set_warning_stream(this->warning_stream);
+ hfst::twolcpre1::set_error_stream(this->error_stream);
+ if (hfst::twolcpre1::parse() != 0)
+ {
+ throw HfstException();
+ }
+
+ std::istringstream iss1(oss1.str());
+ hfst::twolcpre2::set_input(iss1);
+ hfst::twolcpre2::set_warning_stream(this->warning_stream);
+ hfst::twolcpre2::set_error_stream(this->error_stream);
+
+ if (hfst::twolcpre2::parse() != 0)
+ {
+ throw HfstException();
+ }
+ hfst::twolcpre2::complete_alphabet();
+ std::ostringstream oss2;
+ oss2 << hfst::twolcpre2::get_total_alphabet_symbol_queue() << " ";
+ oss2 << hfst::twolcpre2::get_non_alphabet_symbol_queue();
+
+ std::istringstream iss2(oss2.str());
+ hfst::twolcpre3::set_input(iss2);
+ hfst::twolcpre3::set_warning_stream(this->warning_stream);
+ hfst::twolcpre3::set_error_stream(this->error_stream);
+
+ OtherSymbolTransducer::set_transducer_type(command_line.format);
+ hfst::twolcpre3::set_silent(command_line.be_quiet);
+ hfst::twolcpre3::set_verbose(command_line.be_verbose);
+
+ TwolCGrammar twolc_grammar(command_line.be_quiet,
+ command_line.be_verbose,
+ command_line.resolve_left_conflicts,
+ command_line.resolve_right_conflicts);
+ hfst::twolcpre3::set_grammar(&twolc_grammar);
+ if (hfst::twolcpre3::parse() != 0)
+ {
+ throw HfstException();
+ }
+ }
+
+ void TwolcCompiler::store(HfstOutputStream & ostr)
+ {
+ //hfst::twolcpre3::message("Compiling and storing rules.");
+ hfst::twolcpre3::get_grammar()->compile_and_store(ostr);
+ }
+
+}
+
+
diff --git a/tools/src/hfst-twolc/src/variable_src/VariableValues.cc b/libhfst/src/parsers/TwolcCompiler.h
similarity index 53%
copy from tools/src/hfst-twolc/src/variable_src/VariableValues.cc
copy to libhfst/src/parsers/TwolcCompiler.h
index dc25f03..71cb781 100644
--- a/tools/src/hfst-twolc/src/variable_src/VariableValues.cc
+++ b/libhfst/src/parsers/TwolcCompiler.h
@@ -1,7 +1,3 @@
-//! @file VariableValues.cc
-//!
-//! @author Miikka Silfverberg
-
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 3 of the Licence.
@@ -14,13 +10,25 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
-#include "VariableValues.h"
+//#include "commandline_src/CommandLine.h"
+//#include "HfstTwolcDefs.h"
+//#include "io_src/InputReader.h"
+//#include "grammar_defs.h"
+//#include "rule_src/TwolCGrammar.h"
+//#include "rule_src/OtherSymbolTransducer.h"
-void VariableValues::set_variable(const std::string &variable)
-{ this->variable = variable; }
+namespace hfst {
-VariableValues::const_iterator VariableValues::begin(void) const
-{ return const_iterator(variable,std::vector<std::string>::begin()); }
+ class TwolcCompiler
+ {
+ private:
+ CommandLine command_line;
+ std::ostream & warning_stream;
+ std::ostream & error_stream;
+ public:
+ TwolcCompiler(const CommandLine & cl, std::ostream & warn, std::ostream & error);
+ void compile();
+ void store(hfst::HfstOutputStream & ostr);
+ };
-VariableValues::const_iterator VariableValues::end(void) const
-{ return const_iterator(variable,std::vector<std::string>::end()); }
+}
diff --git a/libhfst/src/parsers/XfstCompiler.cc b/libhfst/src/parsers/XfstCompiler.cc
index 6f49570..2658da7 100644
--- a/libhfst/src/parsers/XfstCompiler.cc
+++ b/libhfst/src/parsers/XfstCompiler.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
//! @file XfstCompiler.cc
@@ -34,6 +34,8 @@ using std::stack;
#ifndef _WIN32
#include <glob.h>
#else
+ #pragma warning(disable: 4161)
+ #pragma warning(disable: 4103)
#include <windows.h>
#endif // WINDOWS
@@ -67,7 +69,7 @@ typedef yy_buffer_state *YY_BUFFER_STATE;
extern YY_BUFFER_STATE hxfst_scan_string(const char*);
extern void hxfst_delete_buffer(YY_BUFFER_STATE);
-#include "implementations/HfstTransitionGraph.h"
+#include "implementations/HfstBasicTransducer.h"
using hfst::implementations::HfstBasicTransducer;
using hfst::implementations::HfstBasicTransition;
@@ -78,10 +80,10 @@ using hfst::implementations::HfstBasicTransition;
#define GET_TOP(x) HfstTransducer * x = this->top(); if ((x) == NULL) { xfst_lesser_fail(); return *this; }
#define PROMPT_AND_RETURN_THIS prompt(); return *this;
#define PRINT_INFO_PROMPT_AND_RETURN_THIS print_transducer_info(); prompt(); return *this;
-#define IF_NULL_PROMPT_AND_RETURN_THIS(x) if (x == NULL) { if(variables_["quit-on-fail"] == "ON") { /*exit(EXIT_FAILURE);*/ this->fail_flag_ = true; } prompt(); return *this; }
-#define MAYBE_MINIMIZE(x) if (variables_["minimal"] == "ON") { x->minimize(); }
-#define MAYBE_ASSERT(assertion, value) if (!value && ((variables_["assert"] == "ON" || assertion) && (variables_["quit-on-fail"] == "ON"))) { /*exit(EXIT_FAILURE);*/ this->fail_flag_ = true; }
-#define MAYBE_QUIT if(variables_["quit-on-fail"] == "ON") { /*exit(EXIT_FAILURE);*/ this->fail_flag_ = true; }
+#define IF_NULL_PROMPT_AND_RETURN_THIS(x) if (x == NULL) { if(variables_["quit-on-fail"] == "ON") { this->fail_flag_ = true; } prompt(); return *this; }
+#define MAYBE_MINIMIZE(x) x->optimize();
+#define MAYBE_ASSERT(assertion, value) if (!value && ((variables_["assert"] == "ON" || assertion) && (variables_["quit-on-fail"] == "ON"))) { this->fail_flag_ = true; }
+#define MAYBE_QUIT if(variables_["quit-on-fail"] == "ON") { this->fail_flag_ = true; }
#define EMPTY_STACK error() << "Empty stack." << std::endl; flush(&error());
@@ -91,9 +93,12 @@ using hfst::implementations::HfstBasicTransition;
#include "xfst_help_message.h"
-namespace hfst {
+namespace hfst {
namespace xfst {
+ // whether we need to reset the lexc parser before reading lexc
+ static bool has_lexc_been_read_ = false;
+
static std::map<std::string, std::string> variable_explanations_;
static void initialize_variable_explanations()
@@ -146,6 +151,7 @@ namespace xfst {
lexc_(hfst::TROPICAL_OPENFST_TYPE),
format_(hfst::TROPICAL_OPENFST_TYPE),
verbose_(false),
+ verbose_prompt_(false),
latest_regex_compiled(NULL),
quit_requested_(false),
fail_flag_(false),
@@ -156,7 +162,7 @@ namespace xfst {
winoss_stdout_(std::ostringstream())
// redirected_stream_(NULL)
#endif
- {
+ {
xre_.set_expand_definitions(true);
xre_.set_verbosity(this->verbose_);
xre_.set_flag_harmonization(false);
@@ -173,8 +179,8 @@ namespace xfst {
variables_["flag-is-epsilon"] = "OFF";
variables_["harmonize-flags"] = "OFF";
variables_["hopcroft-min"] = "ON";
- variables_["lexc-minimize-flags"] = "OFF";
- variables_["lexc-rename-flags"] = "OFF";
+ variables_["lexc-minimize-flags"] = "OFF";
+ variables_["lexc-rename-flags"] = "OFF";
variables_["lexc-with-flags"] = "OFF";
variables_["lookup-cycle-cutoff"] = LOOKUP_CYCLE_CUTOFF;
variables_["maximum-weight"] = "OFF";
@@ -211,6 +217,7 @@ namespace xfst {
lexc_(impl),
format_(impl),
verbose_(false),
+ verbose_prompt_(false),
latest_regex_compiled(NULL),
quit_requested_(false),
fail_flag_(false),
@@ -221,7 +228,7 @@ namespace xfst {
winoss_stdout_(std::ostringstream())
//redirected_stream_(NULL)
#endif
- {
+ {
xre_.set_expand_definitions(true);
xre_.set_verbosity(this->verbose_);
xre_.set_flag_harmonization(false);
@@ -238,8 +245,8 @@ namespace xfst {
variables_["flag-is-epsilon"] = "OFF";
variables_["harmonize-flags"] = "OFF";
variables_["hopcroft-min"] = "ON";
- variables_["lexc-minimize-flags"] = "OFF";
- variables_["lexc-rename-flags"] = "OFF";
+ variables_["lexc-minimize-flags"] = "OFF";
+ variables_["lexc-rename-flags"] = "OFF";
variables_["lexc-with-flags"] = "OFF";
variables_["lookup-cycle-cutoff"] = LOOKUP_CYCLE_CUTOFF;
variables_["maximum-weight"] = "OFF";
@@ -268,6 +275,24 @@ namespace xfst {
prompt();
}
+ XfstCompiler::~XfstCompiler()
+ {
+ while(!stack_.empty())
+ {
+ delete(stack_.top());
+ stack_.pop();
+ }
+ for (std::map<std::string, hfst::HfstTransducer*>::const_iterator it
+ = definitions_.begin(); it != definitions_.end(); it++)
+ {
+ delete it->second;
+ }
+ if (latest_regex_compiled != NULL)
+ {
+ delete latest_regex_compiled;
+ }
+ }
+
int XfstCompiler::xfst_fclose(FILE * f, const char * name)
{
int retval = fclose(f);
@@ -282,7 +307,7 @@ namespace xfst {
FILE * XfstCompiler::xfst_fopen(const char* path, const char* mode)
{
- FILE * f = fopen(path, mode);
+ FILE * f = hfst::hfst_fopen(path, mode);
if (f == NULL)
{
error() << "could not open file " << path << std::endl;
@@ -324,19 +349,17 @@ namespace xfst {
void XfstCompiler::xfst_fail()
{
- if (variables_["quit-on-fail"] == "ON")
+ if (variables_["quit-on-fail"] == "ON")
{
this->fail_flag_ = true;
- //exit(EXIT_FAILURE);
}
}
void XfstCompiler::xfst_lesser_fail()
{
- if (variables_["quit-on-fail"] == "ON" && !read_interactive_text_from_stdin_)
+ if (variables_["quit-on-fail"] == "ON" && !read_interactive_text_from_stdin_)
{
this->fail_flag_ = true;
- //exit(EXIT_FAILURE);
}
}
@@ -383,7 +406,7 @@ namespace xfst {
{
return "";
}
- if (strcmp(hfst::internal_unknown.c_str(), symbol) == 0 ||
+ if (strcmp(hfst::internal_unknown.c_str(), symbol) == 0 ||
strcmp(hfst::internal_identity.c_str(), symbol) == 0)
{
return "?";
@@ -444,20 +467,21 @@ namespace xfst {
negative_values.insert(feat);
break;
case 'R': // require
- if (val == "") // empty require
- if (values[feat] == "")
+ if (val == "") { // empty require
+ if (values[feat] == "")
{ return false; }
else { // nonempty require
- if (is_negatively_set || (values[feat] != val)) { return false; }
+ if (is_negatively_set || (values[feat] != val)) { return false; }
}
+ }
break;
case 'D': // disallow
if (val == "") { // empty disallow
- if (values[feat] != "") {
+ if (values[feat] != "") {
return false;
}
}
- else {
+ else {
if ((!is_negatively_set) && (values[feat] == val)) { // nonempty disallow
return false;
}
@@ -480,9 +504,9 @@ namespace xfst {
{
values[feat] = val;
}
- else
+ else
{
- return false;
+ return false;
}
break;
default:
@@ -495,10 +519,10 @@ namespace xfst {
return true;
}
- bool
+ bool
XfstCompiler::print_paths
- (const hfst::HfstOneLevelPaths &paths,
- std::ostream * oss_ /* =cout */,
+ (const hfst::HfstOneLevelPaths &paths,
+ std::ostream * oss_ /* =cout */,
int n /* = -1*/)
{
std::ostream * oss = get_stream(oss_);
@@ -525,7 +549,7 @@ namespace xfst {
// see if symbol separator (space) is needed
if (variables_["print-space"] == "ON" && // print space required
- something_printed && // not first symbol shown
+ something_printed && // not first symbol shown
strcmp(print_symbol, "") != 0) // something to show
{
*oss << " "; // hfst_fprintf(outfile, " ");
@@ -560,10 +584,10 @@ namespace xfst {
bool
XfstCompiler::print_paths
- (const hfst::HfstTwoLevelPaths &paths,
- std::ostream * oss_ /* =cout */,
+ (const hfst::HfstTwoLevelPaths &paths,
+ std::ostream * oss_ /* =cout */,
int n /* = -1*/)
- {
+ {
std::ostream * oss = get_stream(oss_);
bool retval = false; // if anything was printed
oss->precision(get_precision());
@@ -662,46 +686,51 @@ namespace xfst {
tok.add_multichar_symbol(*it);
}
StringVector lookup_path = tok.tokenize_one_level(std::string(token));
+ free(token);
size_t cutoff = -1;
- if (t->is_lookup_infinitely_ambiguous(lookup_path))
+ if (t->is_lookup_infinitely_ambiguous(lookup_path, variables_["obey-flags"] == "ON"))
{
cutoff = string_to_size_t(variables_["lookup-cycle-cutoff"]);
if (verbose_)
{
error() << "warning: lookup is infinitely ambiguous, limiting the number of cycles to " << cutoff << std::endl;
flush(&error());
- //hfst_fprintf(warnstream_,
- // "warning: lookup is infinitely ambiguous, limiting the number of cycles to " SIZE_T_SPECIFIER "\n", cutoff);
}
}
HfstTwoLevelPaths results;
- // todo: variables_["obey-flags"] == ["ON"|"OFF"]
-
if (variables_["maximum-weight"] == "OFF")
- t->lookup_fd(lookup_path, results, &cutoff, NULL);
+ t->lookup(lookup_path, results, &cutoff, NULL, (variables_["obey-flags"] == "ON"));
else
{
float max_weight = string_to_float(variables_["maximum-weight"]);
- t->lookup_fd(lookup_path, results, &cutoff, &max_weight);
+ t->lookup(lookup_path, results, &cutoff, &max_weight, (variables_["obey-flags"] == "ON"));
}
- HfstOneLevelPaths paths = extract_output_paths(results);
+ bool printed = false; // if anything was printed
+
+ if (variables_["print-pairs"] == "OFF")
+ {
+ HfstOneLevelPaths paths = extract_output_paths(results);
+ printed = this->print_paths(paths);
+ }
+ else
+ {
+ printed = this->print_paths(results);
+ }
- bool printed = this->print_paths(paths);
if (!printed)
{
output() << "???" << std::endl;
- //hfst_fprintf(outstream_, "???\n");
}
flush(&output());
return *this;
}
- std::ostream & XfstCompiler::output()
+ std::ostream & XfstCompiler::output()
{
return *get_stream(output_);
}
@@ -728,7 +757,7 @@ namespace xfst {
bool printed = this->print_paths(*paths);
if (!printed)
{
- output() << "???" << std::endl;
+ output() << "???" << std::endl;
flush(&output());
//hfst_fprintf(outstream_, "???\n");
}
@@ -772,7 +801,7 @@ namespace xfst {
if (verbose_)
{
- error() << "converting transducer type from " << hfst::implementation_type_to_format(t->get_type()) << " to "
+ error() << "converting transducer type from " << hfst::implementation_type_to_format(t->get_type()) << " to "
<< hfst::implementation_type_to_format(to_format) << ", this might take a while..." << std::endl;
flush(&error());
//hfst_fprintf(warnstream_, "converting transducer type from %s to %s, this might take a while...\n",
@@ -821,7 +850,7 @@ namespace xfst {
if (verbose_)
{
- error() << "converting transducer type from " << hfst::implementation_type_to_format(t->get_type()) << " to "
+ error() << "converting transducer type from " << hfst::implementation_type_to_format(t->get_type()) << " to "
<< hfst::implementation_type_to_format(format_) << ", this might take a while..." << std::endl;
flush(&error());
//hfst_fprintf(warnstream_, "converting transducer type from %s to %s, this might take a while...\n",
@@ -880,7 +909,7 @@ namespace xfst {
{
error() << "warning: transducer is infinitely ambiguous, limiting number of cycles to " << ol_cutoff << std::endl;
flush(&error());
- //hfst_fprintf(warnstream_,
+ //hfst_fprintf(warnstream_,
// "warning: transducer is infinitely ambiguous, limiting number of cycles to " SIZE_T_SPECIFIER "\n", ol_cutoff);
}
}
@@ -900,7 +929,7 @@ namespace xfst {
flush(&error());
}
t = new HfstTransducer(*(stack_.top()));
- t->invert().minimize();
+ t->invert().minimize(); // the user has been warned for possible slow performance
HfstBasicTransducer fsm(*t);
this->lookup(line, &fsm);
delete t;
@@ -955,7 +984,7 @@ namespace xfst {
return *this;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::add_props(FILE* infile)
{
char* line = 0;
@@ -969,8 +998,8 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
- XfstCompiler::add_props(const char* indata)
+ XfstCompiler&
+ XfstCompiler::add_props(const char* indata)
{
char* s = strdup(indata);
char* line = strtok(s, "\n");
@@ -1048,7 +1077,7 @@ namespace xfst {
if (t->get_type() == hfst::HFST_OL_TYPE ||
t->get_type() == hfst::HFST_OLW_TYPE)
{
- error() << "Operation not supported for optimized lookup format. Consider 'remove-optimization' to convert into ordinary format." << std::endl;
+ error() << "Operation not supported for optimized lookup format. Consider 'remove-optimization' to convert into ordinary format." << std::endl;
flush(&error());
prompt();
return *this;
@@ -1062,7 +1091,7 @@ namespace xfst {
flush(&error());
}
t = new HfstTransducer(*(stack_.top()));
- t->invert().minimize();
+ t->invert().minimize(); // the user has been warned for possible slow performance
}
if (t->get_type() != hfst::HFST_OL_TYPE && t->get_type() != hfst::HFST_OLW_TYPE)
@@ -1085,7 +1114,7 @@ namespace xfst {
char * line = NULL;
// prompt is printed only when reading from the user
- const char * promptstr
+ const char * promptstr
= ((infile == stdin) && verbose_)? get_apply_prompt(direction) : "";
int ind = current_history_index(); // readline history to return to
@@ -1205,7 +1234,7 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::define_list(const char* name, const char* start,
const char* end)
{
@@ -1264,27 +1293,27 @@ namespace xfst {
//if (!was_defined) {
xre_.define(name, *top); // XRE
//}
- HfstTransducer * nu = new HfstTransducer(*top);
+ HfstTransducer * nu = new HfstTransducer(*top);
stack_.pop(); // todo: delete?
if (variables_["name-nets"] == "ON") {
nu->set_name(name);
}
definitions_[name] = nu;
- if (verbose_)
+ if (verbose_)
{
if (was_defined)
output() << "Redefined"; // hfst_fprintf(outstream_, "Redefined");
else
output() << "Defined"; // hfst_fprintf(outstream_, "Defined");
- output() << " '" << std::string(name) << "'" << std::endl; // hfst_fprintf(outstream_, " '%s'\n", name);
+ output() << " '" << std::string(name) << "'" << std::endl; // hfst_fprintf(outstream_, " '%s'\n", name);
flush(&output());
- }
+ }
original_definitions_[name] = "<net taken from stack>";
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::define(const char* name, const char* xre)
{
/*if (hfst::xfst::nametoken_to_number(name) >= 0)
@@ -1319,24 +1348,24 @@ namespace xfst {
error() << "Could not define variable '" << std::string(name) << "'" << std::endl;
flush(&error());
// << xre_.get_error_message() << std::endl;
- //hfst_fprintf(errorstream_, "Could not define variable %s:\n%s\n",
+ //hfst_fprintf(errorstream_, "Could not define variable %s:\n%s\n",
// name, xre_.get_error_message().c_str());
xfst_fail();
}
- HfstTransducer * nu = new HfstTransducer(*latest_regex_compiled);
+ HfstTransducer * nu = new HfstTransducer(*latest_regex_compiled);
if (variables_["name-nets"] == "ON")
nu->set_name(name);
definitions_[name] = nu;
- if (verbose_)
+ if (verbose_)
{
if (was_defined)
output() << "Redefined" << std::endl; // hfst_fprintf(outstream_, "Redefined");
else
output() << "Defined" << std::endl; // hfst_fprintf(outstream_, "Defined");
- output() << " '" << std::string(name) << "'" << std::endl; // hfst_fprintf(outstream_, " '%s'\n", name);
+ output() << " '" << std::string(name) << "'" << std::endl; // hfst_fprintf(outstream_, " '%s'\n", name);
flush(&output());
- }
+ }
original_definitions_[name] = xre;
}
else
@@ -1344,7 +1373,7 @@ namespace xfst {
error() << "Could not define variable '" << std::string(name) << "'" << std::endl;
flush(&error());
// << xre_.get_error_message() << std::endl;
- //hfst_fprintf(errorstream_, "Could not define variable %s:\n%s\n",
+ //hfst_fprintf(errorstream_, "Could not define variable %s:\n%s\n",
// name, xre_.get_error_message().c_str());
xfst_fail();
}
@@ -1390,16 +1419,16 @@ namespace xfst {
for ( ; prototype[i] != ')'; i++)
{
if (prototype[i] == '\0') { // no closing parenthesis found
- return false;
+ return false;
}
else if (prototype[i] == ' ') { // skip whitespace
- }
+ }
else if (prototype[i] == ',') { // end of argument
args.push_back(arg);
arg = "";
}
else {
- arg += prototype[i];
+ arg += prototype[i];
}
}
// last argument
@@ -1408,9 +1437,9 @@ namespace xfst {
return true;
}
- /* Convert each argument name in \a arguments in \a xre with a special symbol
+ /* Convert each argument name in \a arguments in \a xre with a special symbol
"@function_name(argno@" where argno is a number defining which argument
- of the function is being replaced.
+ of the function is being replaced.
For example, if we have a function Foo(x, y, z) defined with regex [ x y+ z x* ],
the regex is converted into ["@Foo(1@" "@Foo(2@"+ "@Foo(3@" "@Foo(1@"* ].
@@ -1419,16 +1448,16 @@ namespace xfst {
(const std::vector<std::string> & arguments,
const std::string & xre,
const std::string & function_name,
- hfst::xre::XreCompiler & xre_,
+ hfst::xre::XreCompiler & xre_,
bool user_friendly_argument_names=false)
{
std::string retval(xre);
unsigned int arg_number = 1;
- for (std::vector<std::string>::const_iterator arg
+ for (std::vector<std::string>::const_iterator arg
= arguments.begin(); arg != arguments.end(); arg++)
{
- std::set<unsigned int> arg_positions;
+ std::set<unsigned int> arg_positions;
if (! xre_.get_positions_of_symbol_in_xre
(*arg, retval, arg_positions) ) // XRE
{
@@ -1465,14 +1494,14 @@ namespace xfst {
new_retval.append(substituting_argument);
// skip rest of the original symbol by advancing i to
// point to the last char in the original symbol
- for (unsigned int offset=1;
+ for (unsigned int offset=1;
offset < arg->length(); offset++)
{
++i;
}
}
// else, just copy
- else
+ else
{
new_retval += retval[i];
}
@@ -1512,7 +1541,7 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- std::string xre_converted
+ std::string xre_converted
= convert_argument_symbols(arguments, xre, name, xre_);
if (xre_converted == std::string(""))
{
@@ -1525,7 +1554,7 @@ namespace xfst {
bool was_defined = xre_.is_function_definition(name);
- if (! xre_.define_function(name, arguments.size(), xre_converted)) // XRE
+ if (! xre_.define_function(name, hfst::size_t_to_uint(arguments.size()), xre_converted)) // XRE
{
error() << "Error when defining function" << std::endl;
flush(&error());
@@ -1534,25 +1563,25 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- if (verbose_)
+ if (verbose_)
{
if (was_defined) {
output() << "Redefined";
- //hfst_fprintf(stderr, "Redefined");
+ //hfst_fprintf(stderr, "Redefined");
}
else {
- output() << "Defined"; // hfst_fprintf(stderr, "Defined");
+ output() << "Defined"; // hfst_fprintf(stderr, "Defined");
}
output() << " function '" << name << "@" << (int)arguments.size() << ")" << std::endl;
flush(&output());
- //hfst_fprintf(stderr, " function '%s@%i)'\n",
- // name.c_str(), (int)arguments.size());
+ //hfst_fprintf(stderr, " function '%s@%i)'\n",
+ // name.c_str(), (int)arguments.size());
}
- function_arguments_[name] = arguments.size();
- function_definitions_[std::string(name)]
+ function_arguments_[name] = hfst::size_t_to_uint(arguments.size());
+ function_definitions_[std::string(name)]
= convert_argument_symbols(arguments, xre, "", xre_, true);
- original_function_definitions_[prototype] = xre;
+ original_function_definitions_[prototype] = xre;
PROMPT_AND_RETURN_THIS;
}
@@ -1584,14 +1613,14 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- /*XfstCompiler&
+ /*XfstCompiler&
XfstCompiler::name(const char* name)
{
names_[name] = stack_.top();
PROMPT_AND_RETURN_THIS;
}*/
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::load_definitions(const char * infilename)
{
return this->load_stack_or_definitions
@@ -1636,8 +1665,8 @@ namespace xfst {
// Punctuation characters and upper/lower case are handled in this function.
static bool string_found(const std::string & str_, const std::string & text_)
{
- std::string str = to_upper_case(str_);
- std::string text = to_upper_case(text_);
+ std::string str = to_upper_case(str_);
+ std::string text = to_upper_case(text_);
std::size_t pos = text.find(str);
if (pos == std::string::npos)
{
@@ -1655,7 +1684,7 @@ namespace xfst {
}
#endif // FOO
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::apropos(const char* text )
{
std::string message;
@@ -1676,7 +1705,7 @@ namespace xfst {
XfstCompiler&
XfstCompiler::describe(const char* text)
{
- int help_mode = (strcmp(text, "") == 0)?
+ int help_mode = (strcmp(text, "") == 0)?
HELP_MODE_ALL_COMMANDS : HELP_MODE_ONE_COMMAND;
std::string message;
if (!get_help_message(text, message, help_mode))
@@ -1693,16 +1722,23 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::clear()
{
- while (!stack_.empty()) {
- stack_.pop();
- }
+ while(!stack_.empty())
+ {
+ delete(stack_.top());
+ stack_.pop();
+ }
+ if (latest_regex_compiled != NULL)
+ {
+ delete latest_regex_compiled;
+ latest_regex_compiled = NULL;
+ }
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::pop()
{
if (stack_.empty()) {
@@ -1714,7 +1750,7 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::push(const char* name)
{
if (definitions_.find(name) == definitions_.end())
@@ -1725,16 +1761,16 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- stack_.push(definitions_[name]);
+ stack_.push(new HfstTransducer(*(definitions_[name])));
PRINT_INFO_PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::push()
{
- for (map<string,HfstTransducer*>::const_iterator def
+ for (map<string,HfstTransducer*>::const_iterator def
= definitions_.begin(); def != definitions_.end();
- ++def)
+ ++def)
{
stack_.push(new HfstTransducer(*(def->second)));
}
@@ -1742,7 +1778,7 @@ namespace xfst {
PRINT_INFO_PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::turn()
{
queue<HfstTransducer*> tmp;
@@ -1780,10 +1816,10 @@ namespace xfst {
static const char * to_filename(const char * file)
{
if (file == 0) {
- return "<stdin>";
+ return "<stdin>";
}
else {
- return file;
+ return file;
}
}
@@ -1800,7 +1836,7 @@ namespace xfst {
// xfst_lesser_fail(); ???
return *this;
}
- std::map<std::string, HfstTransducer*>::const_iterator it
+ std::map<std::string, HfstTransducer*>::const_iterator it
= definitions_.find(def_name);
if (it != definitions_.end())
{
@@ -1866,7 +1902,7 @@ namespace xfst {
{
assert(infilename != NULL);
- FILE * infile = fopen(infilename, "r");
+ FILE * infile = hfst::hfst_fopen(infilename, "r");
if (infile == NULL)
{
error() << "Could not open file " << infilename << std::endl;
@@ -1883,7 +1919,7 @@ namespace xfst {
}
HfstInputStream* instream = 0;
- try
+ try
{
instream = (infilename != 0) ?
new HfstInputStream(infilename):
@@ -1893,7 +1929,7 @@ namespace xfst {
{
error() << "Unable to read transducers from " << to_filename(infilename) << std::endl;
flush(&error());
- //hfst_fprintf(errorstream_, "Unable to read transducers from %s\n",
+ //hfst_fprintf(errorstream_, "Unable to read transducers from %s\n",
// to_filename(infilename));
xfst_fail();
return NULL;
@@ -1943,13 +1979,13 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::load_stack(const char* infilename)
{
return this->load_stack_or_definitions(infilename, false);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::collect_epsilon_loops()
{
error() << "cannot collect epsilon loops" << std::endl;
@@ -1958,7 +1994,7 @@ namespace xfst {
// __LINE__);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::compact_sigma()
{
GET_TOP(top);
@@ -1966,12 +2002,12 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::eliminate_flag(const char* name)
{
GET_TOP(tmp);
std::string name_(name);
- try
+ try
{
tmp->eliminate_flag(name);
}
@@ -1984,7 +2020,7 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::eliminate_flags()
{
GET_TOP(tmp);
@@ -2000,7 +2036,7 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::echo(const char* text)
{
output() << text << std::endl;
@@ -2009,7 +2045,7 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::quit(const char* message)
{
if (verbose_ && (strcmp(message, "dodongo") == 0))
@@ -2030,7 +2066,7 @@ namespace xfst {
return *this;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::system(const char* command)
{
int rv = ::system(command);
@@ -2043,7 +2079,7 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::set(const char* name, const char* text)
{
if (variables_.find(name) == variables_.end())
@@ -2061,7 +2097,7 @@ namespace xfst {
//hfst_fprintf(outstream_, "variable %s = %s\n", "flag-is-epsilon", text);
}
PROMPT_AND_RETURN_THIS;
- }
+ }
else
{
error() << "no such variable: '" << name << "'" << std::endl;
@@ -2107,6 +2143,13 @@ namespace xfst {
if (strcmp(text, "OFF") == 0)
hfst::set_flag_is_epsilon_in_composition(false);
}
+ if (strcmp(name, "minimal") == 0)
+ {
+ if (strcmp(text, "ON") == 0)
+ hfst::set_minimization(true);
+ if (strcmp(text, "OFF") == 0)
+ hfst::set_minimization(false);
+ }
if (verbose_)
{
@@ -2118,7 +2161,7 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::set(const char* name, unsigned int number)
{
if (variables_.find(name) == variables_.end())
@@ -2131,7 +2174,8 @@ namespace xfst {
}
char* num = static_cast<char*>(malloc(sizeof(char)*31));
sprintf(num, "%u", number);
- variables_[name] = num;
+ variables_[name] = std::string(num);
+ free(num);
PROMPT_AND_RETURN_THIS;
}
@@ -2143,7 +2187,7 @@ namespace xfst {
return std::string(variables_[name]);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::show(const char* name)
{
if (variables_.find(name) == variables_.end())
@@ -2160,7 +2204,7 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::show()
{
for (map<string,string>::const_iterator var = variables_.begin();
@@ -2173,15 +2217,15 @@ namespace xfst {
output().width(20);
output() << var->first << ": " << var->second << std::endl;
//hfst_fprintf(stderr, "%20s: %s\n", var->first.c_str(),
- // var->second.c_str());
+ // var->second.c_str());
}
else
{
output().width(20);
- output() << var->first << ": ";
+ output() << var->first << ": ";
output().width(6);
output() << var->second << ": " << variable_explanations_[var->first] << std::endl;
- // hfst_fprintf(stderr, "%20s:%6s: %s\n",
+ // hfst_fprintf(stderr, "%20s:%6s: %s\n",
// var->first.c_str(), var->second.c_str(),
// variable_explanations_[var->first].c_str());
}
@@ -2190,8 +2234,8 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
- XfstCompiler::test_eq(bool assertion)
+ XfstCompiler&
+ XfstCompiler::test_eq(bool assertion)
{
if (stack_.size() < 2)
{
@@ -2218,7 +2262,7 @@ namespace xfst {
XfstCompiler&
XfstCompiler::print_bool(bool value)
{
- int printval = (value)? 1 : 0;
+ int printval = (value)? 1 : 0;
output() << printval << ", (1 = TRUE, 0 = FALSE)" << std::endl;
flush(&output());
//hfst_fprintf(outstream_, "%i, (1 = TRUE, 0 = FALSE)\n", printval);
@@ -2247,9 +2291,9 @@ namespace xfst {
return NULL;
}
return retval;
- }
+ }
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::test_funct(bool assertion)
{
error() << "test funct missing" << std::endl;
@@ -2276,7 +2320,7 @@ namespace xfst {
MAYBE_ASSERT(assertion, result);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::test_upper_bounded(bool assertion)
{
HfstTransducer * temp = this->top();
@@ -2286,14 +2330,14 @@ namespace xfst {
HfstTransducer tmp(*temp);
tmp.output_project();
- tmp.minimize();
+ tmp.remove_epsilons(); // needed for testing cyclicity
bool result = ! tmp.is_cyclic();
this->print_bool(result);
MAYBE_ASSERT(assertion, result);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::test_uni(Level level, bool assertion)
{
HfstTransducer * temp = this->top();
@@ -2321,12 +2365,12 @@ namespace xfst {
MAYBE_ASSERT(assertion, value);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::test_upper_uni(bool assertion)
{
return this->test_uni(UPPER_LEVEL, assertion);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::test_lower_bounded(bool assertion)
{
HfstTransducer * temp = this->top();
@@ -2336,24 +2380,24 @@ namespace xfst {
HfstTransducer tmp(*temp);
tmp.input_project();
- tmp.minimize();
+ tmp.remove_epsilons(); // needed for testing cyclicity
bool result = ! tmp.is_cyclic();
this->print_bool(result);
MAYBE_ASSERT(assertion, result);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::test_lower_uni(bool assertion)
{
return this->test_uni(LOWER_LEVEL, assertion);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::test_nonnull(bool assertion)
{
return this->test_null(true, assertion);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::test_null(bool invert_test_result, bool assertion)
{
HfstTransducer * tmp = this->top();
@@ -2383,7 +2427,7 @@ namespace xfst {
xfst_lesser_fail();
PROMPT_AND_RETURN_THIS;
}
- std::stack<HfstTransducer*> copied_stack(stack_);
+ std::stack<HfstTransducer*> copied_stack(stack_);
HfstTransducer topmost_transducer(*(copied_stack.top()));
copied_stack.pop();
@@ -2431,17 +2475,17 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::test_overlap(bool assertion)
{
return this->test_operation(TEST_OVERLAP_, assertion);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::test_sublanguage(bool assertion)
{
return this->test_operation(TEST_SUBLANGUAGE_, assertion);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::test_unambiguous(bool assertion)
{
error() << "test unambiguous missing" << std::endl;
@@ -2450,7 +2494,7 @@ namespace xfst {
//MAYBE_ASSERT(assertion, result);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::test_infinitely_ambiguous(bool assertion)
{
HfstTransducer * tmp = this->top();
@@ -2518,18 +2562,18 @@ namespace xfst {
return sp;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::substitute_named(const char* variable, const char* label)
{
GET_TOP(top);
- std::map<std::string, HfstTransducer*>::const_iterator it
+ std::map<std::string, HfstTransducer*>::const_iterator it
= definitions_.find(variable);
if (it == definitions_.end())
{
error() << "no such definition '" << variable << "', cannot substitute" << std::endl;
flush(&error());
- //hfst_fprintf(errorstream_, "no such definition '%s', cannot substitute\n",
+ //hfst_fprintf(errorstream_, "no such definition '%s', cannot substitute\n",
// variable);
MAYBE_QUIT;
PROMPT_AND_RETURN_THIS;
@@ -2546,7 +2590,7 @@ namespace xfst {
{
error() << "no occurrences of label '" << label << "', cannot substitute" << std::endl;
flush(&error());
- //hfst_fprintf(errorstream_, "no occurrences of label '%s', cannot substitute\n",
+ //hfst_fprintf(errorstream_, "no occurrences of label '%s', cannot substitute\n",
// label);
MAYBE_QUIT;
PROMPT_AND_RETURN_THIS;
@@ -2554,15 +2598,15 @@ namespace xfst {
HfstBasicTransducer fsm(*top);
- for (HfstBasicTransducer::const_iterator it = fsm.begin();
- it != fsm.end(); it++ )
- {
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
- = it->begin(); tr_it != it->end(); tr_it++)
+ for (HfstBasicTransducer::const_iterator it = fsm.begin();
+ it != fsm.end(); it++ )
+ {
+ for (hfst::implementations::HfstBasicTransitions::const_iterator tr_it
+ = it->begin(); tr_it != it->end(); tr_it++)
{
std::string isymbol = tr_it->get_input_symbol();
std::string osymbol = tr_it->get_output_symbol();
- if (isymbol != osymbol &&
+ if (isymbol != osymbol &&
(isymbol == labelstr || osymbol == labelstr))
{
error() << "label '" << label << "' is used as a symbol on one side of an arc, cannot substitute" << std::endl;
@@ -2589,7 +2633,7 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::substitute_label(const char* list, const char* target)
{
GET_TOP(top);
@@ -2605,13 +2649,14 @@ namespace xfst {
{
// tokenize labels into string pairs
StringVector sv = tokenize_string(it->c_str(), ':');
- try
+ try
{
StringPair sp = symbol_vector_to_symbol_pair(sv);
symbol_pairs.insert(sp);
}
catch (const char * msg)
{
+ (void)msg;
error() << "error: could not substitute with '" << list << "'" << std::endl;
flush(&error());
//hfst_fprintf(errorstream_, "error: could not substitute with '%s'\n", list);
@@ -2623,18 +2668,18 @@ namespace xfst {
// tokenize target label into string pair
StringVector target_vector = tokenize_string(target, ':');
- try
+ try
{
StringPair target_label = symbol_vector_to_symbol_pair(target_vector);
HfstBasicTransducer fsm(*top);
bool target_label_found = false;
- for (HfstBasicTransducer::const_iterator it = fsm.begin();
- it != fsm.end() && !target_label_found; it++ )
- {
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
- = it->begin(); tr_it != it->end(); tr_it++)
+ for (HfstBasicTransducer::const_iterator it = fsm.begin();
+ it != fsm.end() && !target_label_found; it++ )
+ {
+ for (hfst::implementations::HfstBasicTransitions::const_iterator tr_it
+ = it->begin(); tr_it != it->end(); tr_it++)
{
if (target_label.first == tr_it->get_input_symbol() &&
target_label.second == tr_it->get_output_symbol())
@@ -2648,7 +2693,7 @@ namespace xfst {
{
error() << "no occurrences of '" << target_label.first << ":" << target_label.second << "', cannot substitute" << std::endl;
flush(&error());
- //hfst_fprintf(errorstream_, "no occurrences of '%s:%s', cannot substitute\n",
+ //hfst_fprintf(errorstream_, "no occurrences of '%s:%s', cannot substitute\n",
// target_label.first.c_str(), target_label.second.c_str());
PROMPT_AND_RETURN_THIS;
}
@@ -2657,6 +2702,7 @@ namespace xfst {
}
catch (const char * msg)
{
+ (void)msg;
error() << "error: could not substitute '" << target << "'" << std::endl;
flush(&error());
//hfst_fprintf(errorstream_, "error: could not substitute '%s'\n", target);
@@ -2667,7 +2713,7 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::substitute_symbol(const char* list, const char* target)
{
GET_TOP(top);
@@ -2707,12 +2753,11 @@ namespace xfst {
flush(&error());
//hfst_fprintf(errorstream_, "fatal error in substitution\n");
this->fail_flag_ = true;
- //exit(EXIT_FAILURE);
}
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_aliases(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -2723,14 +2768,14 @@ namespace xfst {
// HERE
oss->width(10);
*oss << "alias " << alias->first << " " << alias->second;
- //hfst_fprintf(outfile, "alias %10s %s",
+ //hfst_fprintf(outfile, "alias %10s %s",
// alias->first.c_str(), alias->second.c_str());
}
flush(oss);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_arc_count(const char* level, std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -2740,7 +2785,7 @@ namespace xfst {
flush(oss);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_arc_count(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -2750,15 +2795,15 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_defined(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
bool definitions = false;
- for (map<string,string>::const_iterator def
+ for (map<string,string>::const_iterator def
= original_definitions_.begin(); def != original_definitions_.end();
++def)
- {
+ {
// HERE
definitions = true;
oss->width(10);
@@ -2777,7 +2822,7 @@ namespace xfst {
definitions = true;
oss->width(10);
*oss << func->first << " " << func->second << std::endl;
- //hfst_fprintf(outfile, "%10s %s\n", func->first.c_str(),
+ //hfst_fprintf(outfile, "%10s %s\n", func->first.c_str(),
// func->second.c_str());
}
if (!definitions)
@@ -2787,7 +2832,7 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_dir(const char* globdata, std::ostream * oss)
{
#ifndef _WIN32
@@ -2815,7 +2860,7 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_flags(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -2832,11 +2877,11 @@ namespace xfst {
return this->print_labels(oss, topmost);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_labels(const char* name, std::ostream * oss_)
- {
+ {
std::ostream * oss = get_stream(oss_);
- std::map<std::string, HfstTransducer*>::const_iterator it
+ std::map<std::string, HfstTransducer*>::const_iterator it
= definitions_.find(name);
if (it == definitions_.end())
{
@@ -2851,18 +2896,18 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_labels(std::ostream * oss_, HfstTransducer* tr)
{
std::ostream * oss = get_stream(oss_);
std::set<std::pair<std::string, std::string> > label_set;
HfstBasicTransducer fsm(*tr);
- for (HfstBasicTransducer::const_iterator it = fsm.begin();
- it != fsm.end(); it++ )
- {
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
- = it->begin(); tr_it != it->end(); tr_it++)
+ for (HfstBasicTransducer::const_iterator it = fsm.begin();
+ it != fsm.end(); it++ )
+ {
+ for (hfst::implementations::HfstBasicTransitions::const_iterator tr_it
+ = it->begin(); tr_it != it->end(); tr_it++)
{
std::pair<std::string, std::string> label_pair
(tr_it->get_input_symbol(), tr_it->get_output_symbol());
@@ -2889,7 +2934,7 @@ namespace xfst {
flush(oss);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_labelmaps(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -2899,7 +2944,7 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_label_count(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -2908,11 +2953,11 @@ namespace xfst {
std::map<std::pair<std::string, std::string>, unsigned int > label_map;
HfstBasicTransducer fsm(*topmost);
- for (HfstBasicTransducer::const_iterator it = fsm.begin();
- it != fsm.end(); it++ )
- {
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
- = it->begin(); tr_it != it->end(); tr_it++)
+ for (HfstBasicTransducer::const_iterator it = fsm.begin();
+ it != fsm.end(); it++ )
+ {
+ for (hfst::implementations::HfstBasicTransitions::const_iterator tr_it
+ = it->begin(); tr_it != it->end(); tr_it++)
{
std::pair<std::string, std::string> label_pair
(tr_it->get_input_symbol(), tr_it->get_output_symbol());
@@ -2921,7 +2966,7 @@ namespace xfst {
}
unsigned int index=1;
- for(std::map<std::pair<std::string, std::string>, unsigned int >::const_iterator
+ for(std::map<std::pair<std::string, std::string>, unsigned int >::const_iterator
it= label_map.begin(); it != label_map.end(); it++)
{
if (it != label_map.begin())
@@ -2969,7 +3014,7 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_list(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -3004,14 +3049,14 @@ namespace xfst {
XfstCompiler&
XfstCompiler::shortest_string
- (const hfst::HfstTransducer * transducer,
+ (const hfst::HfstTransducer * transducer,
hfst::HfstTwoLevelPaths & paths)
- {
+ {
transducer->extract_shortest_paths(paths);
return *this;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_shortest_string(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -3033,7 +3078,7 @@ namespace xfst {
flush(oss);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_shortest_string_size(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -3083,10 +3128,10 @@ namespace xfst {
GET_TOP(topmost);
// Variables needed to find out some properties about the transducer
- HfstTransducer tmp_lower(*topmost);
+ HfstTransducer tmp_lower(*topmost);
HfstTransducer tmp_upper(*topmost);
- tmp_lower.output_project().minimize();
- tmp_upper.input_project().minimize();
+ tmp_lower.output_project().remove_epsilons();
+ tmp_upper.input_project().remove_epsilons();
HfstTwoLevelPaths paths_upper;
HfstTwoLevelPaths paths_lower;
@@ -3095,20 +3140,22 @@ namespace xfst {
bool transducer_is_empty = false;
try { // Transducer is empty if neither upper..
- transducer_is_empty =
- ! tmp_upper.extract_longest_paths(paths_upper,
+ transducer_is_empty =
+ ! tmp_upper.extract_longest_paths(paths_upper,
variables_["obey-flags"] == "ON");
}
catch (const TransducerIsCyclicException & e) {
+ (void)e;
upper_is_cyclic = true;
}
try { // ..nor lower paths can be extracted.
- transducer_is_empty =
- ! tmp_lower.extract_longest_paths(paths_lower,
+ transducer_is_empty =
+ ! tmp_lower.extract_longest_paths(paths_lower,
variables_["obey-flags"] == "ON");
}
catch (const TransducerIsCyclicException & e) {
+ (void)e;
lower_is_cyclic = true;
}
@@ -3122,11 +3169,11 @@ namespace xfst {
output() << "transducer is empty" << std::endl; // hfst_fprintf(outstream_, "transducer is empty\n");
flush(&output());
}
- // then the usual:
+ // then the usual:
else {
// warn about flag diacritics
- if (variables_["show-flags"] == "OFF" &&
- (tmp_upper.has_flag_diacritics() ||
+ if (variables_["show-flags"] == "OFF" &&
+ (tmp_upper.has_flag_diacritics() ||
tmp_lower.has_flag_diacritics()) )
{
error() << "warning: longest string may have flag diacritics that are not shown" << std::endl;
@@ -3155,25 +3202,25 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_longest_string(std::ostream * oss)
{
return print_longest_string_or_its_size(oss, false);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_longest_string_size(std::ostream * oss)
{
return print_longest_string_or_its_size(oss, true);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_lower_words(const char * name, unsigned int number,
std::ostream * oss)
{
return print_words(name, number, oss, LOWER_LEVEL);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_random_lower(const char * name, unsigned int number, std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -3182,14 +3229,14 @@ namespace xfst {
HfstTransducer tmp(format_);
if (name == NULL)
{
- HfstTransducer * temp = this->top();
- if (temp == NULL)
+ HfstTransducer * temp = this->top();
+ if (temp == NULL)
{ return *this; }
tmp = *temp;
}
else
{
- std::map<std::string, HfstTransducer*>::const_iterator it
+ std::map<std::string, HfstTransducer*>::const_iterator it
= definitions_.find(name);
if (it == definitions_.end())
{
@@ -3211,7 +3258,7 @@ namespace xfst {
flush(oss);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_upper_words(const char * name, unsigned int number,
std::ostream * oss)
{
@@ -3226,14 +3273,14 @@ namespace xfst {
HfstTransducer tmp(format_);
if (name == NULL)
{
- HfstTransducer * temp = this->top();
- if (temp == NULL)
+ HfstTransducer * temp = this->top();
+ if (temp == NULL)
{ return *this; }
tmp = *temp;
}
else
{
- std::map<std::string, HfstTransducer*>::const_iterator it
+ std::map<std::string, HfstTransducer*>::const_iterator it
= definitions_.find(name);
if (it == definitions_.end())
{
@@ -3256,14 +3303,14 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_words(const char * name, unsigned int number,
std::ostream * oss)
{
return print_words(name, number, oss, BOTH_LEVELS);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_words(const char * name, unsigned int number,
std::ostream * oss_, Level level)
{
@@ -3271,14 +3318,14 @@ namespace xfst {
HfstTransducer temp(format_);
if (name == NULL)
{
- HfstTransducer * tmp = this->top();
- if (tmp == NULL)
+ HfstTransducer * tmp = this->top();
+ if (tmp == NULL)
{ return *this; }
temp = *tmp;
}
else
{
- std::map<std::string, HfstTransducer*>::const_iterator it
+ std::map<std::string, HfstTransducer*>::const_iterator it
= definitions_.find(name);
if (it == definitions_.end())
{
@@ -3324,7 +3371,8 @@ namespace xfst {
}
catch (const TransducerIsCyclicException & e)
{
- int cutoff = string_to_size_t(variables_["print-words-cycle-cutoff"]);
+ (void)e;
+ int cutoff = hfst::size_t_to_uint(string_to_size_t(variables_["print-words-cycle-cutoff"]));
error() << "warning: transducer is cyclic, limiting the number of cycles to " << cutoff << std::endl;
flush(&error());
//hfst_fprintf(warnstream_, "warning: transducer is cyclic, limiting the number of cycles to %i\n", cutoff);
@@ -3339,24 +3387,24 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_random_words(const char * name, unsigned int number, std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
const HfstTransducer * tmp = NULL;
if (name == NULL)
{
- tmp = this->top();
- if (tmp == NULL)
+ tmp = this->top();
+ if (tmp == NULL)
{ return *this; }
}
else
{
- std::map<std::string, HfstTransducer*>::const_iterator it
+ std::map<std::string, HfstTransducer*>::const_iterator it
= definitions_.find(name);
if (it == definitions_.end())
{
- *oss << "no such definition '" << std::string(name) << "'" << std::endl;
+ *oss << "no such definition '" << std::string(name) << "'" << std::endl;
//hfst_fprintf(outfile, "no such definition '%s'\n", name);
flush(oss);
prompt();
@@ -3410,13 +3458,13 @@ namespace xfst {
#endif
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_name(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
GET_TOP(tmp);
- for (std::map<std::string, HfstTransducer*>::const_iterator it
+ for (std::map<std::string, HfstTransducer*>::const_iterator it
= names_.begin(); it != names_.end(); it++)
{
if (tmp == it->second)
@@ -3450,7 +3498,7 @@ namespace xfst {
error() << "Writing net in dot format to temporary file '" << dotfilename << "'." << std::endl;
flush(&error());
}
- FILE * dotfile = fopen(dotfilename, "wb");
+ FILE * dotfile = hfst::hfst_fopen(dotfilename, "wb");
hfst::print_dot(dotfile, *tmp);
fclose(dotfile);
if (false || verbose_)
@@ -3479,20 +3527,23 @@ namespace xfst {
#endif // WINDOWS
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_net(std::ostream * oss_)
{
- std::ostream * oss = get_stream(oss_);
+ if (variables_["print-sigma"] == "ON")
+ {
+ this->print_sigma(oss_, false /*do not prompt*/);
+ }
GET_TOP(tmp);
HfstBasicTransducer basic(*tmp);
+ std::ostream * oss = get_stream(oss_);
basic.write_in_xfst_format(*oss, variables_["print-weight"] == "ON");
flush(oss);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_net(const char* name, std::ostream * oss_)
{
- std::ostream * oss = get_stream(oss_);
//hfst_fprintf(outfile, "missing print net %s:%d\n", __FILE__, __LINE__);
std::map<std::string,hfst::HfstTransducer*>::const_iterator it =
definitions_.find(name);
@@ -3505,16 +3556,23 @@ namespace xfst {
}
else
{
+ if (variables_["print-sigma"] == "ON")
+ {
+ stack_.push(it->second);
+ this->print_sigma(oss_, false /*do not prompt*/);
+ stack_.pop();
+ }
HfstBasicTransducer basic(*(it->second));
+ std::ostream * oss = get_stream(oss_);
basic.write_in_xfst_format(*oss, variables_["print-weight"] == "ON");
+ flush(oss);
+ PROMPT_AND_RETURN_THIS;
}
- flush(oss);
- PROMPT_AND_RETURN_THIS;
}
static bool is_special_symbol(const std::string &s)
{
- if (s == hfst::internal_epsilon || s == hfst::internal_unknown ||
+ if (s == hfst::internal_epsilon || s == hfst::internal_unknown ||
s == hfst::internal_identity ) {
return true;
}
@@ -3548,7 +3606,7 @@ namespace xfst {
bool first_symbol = true;
for (StringSet::const_iterator it = alpha.begin(); it != alpha.end(); it++)
{
- if (! is_special_symbol(*it))
+ if (! is_special_symbol(*it))
{
if (!first_symbol || unknown || identity)
*oss << ", "; //hfst_fprintf(outfile, ", ");
@@ -3576,12 +3634,12 @@ namespace xfst {
identity = false;
HfstBasicTransducer fsm(*t);
- for (HfstBasicTransducer::const_iterator it = fsm.begin();
- it != fsm.end(); it++ )
- {
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
- = it->begin(); tr_it != it->end(); tr_it++)
- {
+ for (HfstBasicTransducer::const_iterator it = fsm.begin();
+ it != fsm.end(); it++ )
+ {
+ for (hfst::implementations::HfstBasicTransitions::const_iterator tr_it
+ = it->begin(); tr_it != it->end(); tr_it++)
+ {
std::string istr = tr_it->get_input_symbol();
std::string ostr = tr_it->get_input_symbol();
if (istr == hfst::internal_unknown ||
@@ -3603,7 +3661,7 @@ namespace xfst {
}
// todo: flags?
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_sigma(std::ostream * oss_, bool prompt)
{
std::ostream * oss = get_stream(oss_);
@@ -3622,7 +3680,7 @@ namespace xfst {
return *this;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_sigma(const char* /*name*/, std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -3631,7 +3689,7 @@ namespace xfst {
flush(oss);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_sigma_count(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -3640,7 +3698,7 @@ namespace xfst {
flush(oss);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_sigma_word_count(const char* level, std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -3650,7 +3708,7 @@ namespace xfst {
flush(oss);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_sigma_word_count(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -3659,7 +3717,7 @@ namespace xfst {
flush(oss);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_size(const char* name, std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -3671,7 +3729,7 @@ namespace xfst {
flush(oss);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_size(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -3680,7 +3738,7 @@ namespace xfst {
flush(oss);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::print_stack(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -3704,7 +3762,7 @@ namespace xfst {
flush(oss);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::write_dot(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -3728,7 +3786,7 @@ namespace xfst {
xfst_lesser_fail();
PROMPT_AND_RETURN_THIS;
}
- FILE * outfile = fopen(name, "wb");
+ FILE * outfile = hfst::hfst_fopen(name, "wb");
if (outfile == NULL)
{
error() << "Could not open file " << name << std::endl;
@@ -3740,7 +3798,7 @@ namespace xfst {
hfst::print_dot(outfile, *tmp);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::write_prolog(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -3774,7 +3832,7 @@ namespace xfst {
flush(oss);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::write_spaced(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -3783,7 +3841,7 @@ namespace xfst {
flush(oss);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::write_text(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -3792,7 +3850,7 @@ namespace xfst {
flush(oss);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::write_function(const char* name, const char* outfile)
{
/*if (outfile == 0)
@@ -3801,7 +3859,7 @@ namespace xfst {
}*/
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::write_definition(const char* name, const char* outfile)
{
if (definitions_.find(name) == definitions_.end())
@@ -3823,7 +3881,7 @@ namespace xfst {
delete outstream;
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::write_definitions(const char* outfile)
{
if (definitions_.empty())
@@ -3879,7 +3937,7 @@ namespace xfst {
outstream->close();
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::read_props(FILE* infile)
{
char* line = 0;
@@ -3892,7 +3950,7 @@ namespace xfst {
free(line);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::read_props(const char* indata)
{
char* s = strdup(indata);
@@ -3905,7 +3963,7 @@ namespace xfst {
free(s);
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::read_regex(FILE* infile)
{
#define MAX_FILE_SIZE 10000000
@@ -3944,18 +4002,19 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::compile_regex(const char* indata, unsigned int & chars_read)
{
if (latest_regex_compiled != NULL)
{
delete latest_regex_compiled;
+ latest_regex_compiled = NULL;
}
latest_regex_compiled = xre_.compile_first(indata, chars_read); // XRE
return *this;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::read_regex(const char* indata)
{
// When calling this function, the regex \a indata should already have
@@ -3973,23 +4032,23 @@ namespace xfst {
error() << "Error reading regex '" << indata << "'." << std::endl;
flush(&error());
// << xre_.get_error_message() << std::endl;
- //hfst_fprintf(errorstream_, "Error reading regex '%s':\n%s\n", indata,
+ //hfst_fprintf(errorstream_, "Error reading regex '%s':\n%s\n", indata,
// xre_.get_error_message().c_str());
xfst_fail();
}
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::read_prolog(FILE* infile)
{
try {
unsigned int linecount=0;
- HfstBasicTransducer tr =
+ HfstBasicTransducer tr =
HfstBasicTransducer::read_in_prolog_format(infile, linecount);
stack_.push(new HfstTransducer(tr, format_));
MAYBE_MINIMIZE(stack_.top());
PRINT_INFO_PROMPT_AND_RETURN_THIS;
- }
+ }
catch (const NotValidPrologFormatException & e)
{
error() << e() << std::endl;
@@ -3998,7 +4057,7 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::read_prolog(const char* /* indata */)
{
error() << "missing read prolog" << std::endl;
@@ -4006,12 +4065,12 @@ namespace xfst {
//hfst_fprintf(stderr, "missing read prolog %s:%d\n", __FILE__, __LINE__);
PRINT_INFO_PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::read_spaced_from_file(const char * filename)
{
return this->read_text_or_spaced(filename, true); // spaces are used
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::read_spaced(const char* /* indata */)
{
error() << "missing read spaced" << std::endl;
@@ -4019,10 +4078,10 @@ namespace xfst {
//hfst_fprintf(stderr, "missing read spaced %s:%d\n", __FILE__, __LINE__);
PRINT_INFO_PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::read_text_or_spaced(const char * filename, bool spaces)
{
- FILE * infile = fopen(filename, "r");
+ FILE * infile = hfst::hfst_fopen(filename, "r");
if (infile == NULL)
{
error() << "Could not open file " << filename << std::endl;
@@ -4053,16 +4112,16 @@ namespace xfst {
xfst_fail();
}
- tmp->minimize();
+ tmp->minimize(); // a trie should be easily minimizable
stack_.push(tmp);
PRINT_INFO_PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::read_text_from_file(const char * filename)
{
return this->read_text_or_spaced(filename, false); // spaces are not used
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::read_text(const char* /* indata */)
{
error() << "missing read text" << std::endl;
@@ -4071,7 +4130,7 @@ namespace xfst {
PRINT_INFO_PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::cleanup_net()
{
error() << "cannot cleanup net" << std::endl;
@@ -4086,7 +4145,7 @@ namespace xfst {
}
PRINT_INFO_PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::complete_net()
{
GET_TOP(topmost);
@@ -4101,18 +4160,18 @@ namespace xfst {
PRINT_INFO_PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::compose_net()
{
return this->apply_binary_operation_iteratively(COMPOSE_NET);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::concatenate_net()
{
return this->apply_binary_operation_iteratively(CONCATENATE_NET);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::crossproduct_net()
{
return this->apply_binary_operation(CROSSPRODUCT_NET);
@@ -4124,7 +4183,7 @@ namespace xfst {
GET_TOP(result);
this->pop();
- try
+ try
{
switch (operation)
{
@@ -4156,7 +4215,7 @@ namespace xfst {
result->reverse();
break;
case MINIMIZE_NET:
- result->minimize();
+ result->minimize(); // implicit minimization requested, do not use optimize()
break;
case PRUNE_NET_:
result->prune();
@@ -4168,12 +4227,16 @@ namespace xfst {
break;
}
- MAYBE_MINIMIZE(result);
+ if (operation != MINIMIZE_NET && operation != DETERMINIZE_NET && operation != EPSILON_REMOVE_NET)
+ {
+ MAYBE_MINIMIZE(result);
+ }
stack_.push(result);
print_transducer_info();
}
catch (const FunctionNotImplementedException & e)
{
+ (void)e;
error() << "function not available" << std::endl;
flush(&error());
//hfst_fprintf(stderr, "function not available.\n");
@@ -4210,13 +4273,14 @@ namespace xfst {
result->subtract(*another);
break;
case (CROSSPRODUCT_NET):
- try
+ try
{
result->cross_product(*another);
break;
}
catch (const TransducersAreNotAutomataException & e)
{
+ (void)e;
error() << "transducers are not automata" << std::endl;
flush(&error());
//hfst_fprintf(stderr, "transducers are not automata\n");
@@ -4299,12 +4363,13 @@ namespace xfst {
}
}
- try
+ try
{
result->compose(*t);
}
catch (const FlagDiacriticsAreNotIdentitiesException & e)
{
+ (void)e;
error() << "Error: flag diacritics must be identities in composition if flag-is-epsilon is ON." << std::endl
<< "I.e. only FLAG:FLAG is allowed, not FLAG1:FLAG2, FLAG:bar or foo:FLAG" << std::endl
<< "Apply twosided flag-diacritics (tfd) before composition." << std::endl;
@@ -4342,32 +4407,32 @@ namespace xfst {
PRINT_INFO_PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::determinize_net()
{
return this->apply_unary_operation(DETERMINIZE_NET);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::epsilon_remove_net()
{
return this->apply_unary_operation(EPSILON_REMOVE_NET);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::ignore_net()
{
return this->apply_binary_operation(IGNORE_NET);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::intersect_net()
{
return this->apply_binary_operation_iteratively(INTERSECT_NET);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::invert_net()
{
return this->apply_unary_operation(INVERT_NET);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::label_net()
{
GET_TOP(topmost);
@@ -4377,11 +4442,11 @@ namespace xfst {
std::set<std::pair<std::string, std::string> > label_set;
HfstBasicTransducer fsm(*topmost);
- for (HfstBasicTransducer::const_iterator it = fsm.begin();
- it != fsm.end(); it++ )
- {
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
- = it->begin(); tr_it != it->end(); tr_it++)
+ for (HfstBasicTransducer::const_iterator it = fsm.begin();
+ it != fsm.end(); it++ )
+ {
+ for (hfst::implementations::HfstBasicTransitions::const_iterator tr_it
+ = it->begin(); tr_it != it->end(); tr_it++)
{
std::pair<std::string, std::string> label_pair
(tr_it->get_input_symbol(), tr_it->get_output_symbol());
@@ -4396,7 +4461,7 @@ namespace xfst {
result->disjunct(label_tr);
}
- result->minimize();
+ result->minimize(); // should be safe to minimize
stack_.pop();
delete topmost;
stack_.push(result);
@@ -4404,27 +4469,27 @@ namespace xfst {
PRINT_INFO_PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::lower_side_net()
{
return this->apply_unary_operation(LOWER_SIDE_NET);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::upper_side_net()
{
return this->apply_unary_operation(UPPER_SIDE_NET);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::minimize_net()
{
return this->apply_unary_operation(MINIMIZE_NET);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::minus_net()
{
return this->apply_binary_operation(MINUS_NET);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::name_net(const char* s)
{
if (stack_.size() < 1)
@@ -4439,7 +4504,7 @@ namespace xfst {
names_[s] = t;
PRINT_INFO_PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::negate_net()
{
if (stack_.size() < 1)
@@ -4456,7 +4521,7 @@ namespace xfst {
(hfst::internal_unknown, hfst::internal_unknown, format_);
result->disjunct(unk2unk);
result->repeat_star();
- result->minimize();
+ result->minimize(); // should be safe to minimize
HfstTransducer* t = stack_.top();
stack_.pop();
@@ -4467,12 +4532,12 @@ namespace xfst {
stack_.push(result);
PRINT_INFO_PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::one_plus_net()
{
return this->apply_unary_operation(ONE_PLUS_NET);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::zero_plus_net()
{
return this->apply_unary_operation(ZERO_PLUS_NET);
@@ -4482,22 +4547,22 @@ namespace xfst {
{
return this->apply_unary_operation(OPTIONAL_NET);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::prune_net()
{
return this->apply_unary_operation(PRUNE_NET_);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::reverse_net()
{
return this->apply_unary_operation(REVERSE_NET);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::shuffle_net()
{
return this->apply_binary_operation_iteratively(SHUFFLE_NET);
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::sigma_net()
{
GET_TOP(tmp);
@@ -4513,7 +4578,7 @@ namespace xfst {
stack_.push(sigma);
PRINT_INFO_PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::sort_net()
{
error() << "missing sort net" << std::endl;
@@ -4521,7 +4586,7 @@ namespace xfst {
//hfst_fprintf(stderr, "cannot sort net %s:%d\n", __FILE__, __LINE__);
PRINT_INFO_PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::substring_net()
{
error() << "missing substring net" << std::endl;
@@ -4554,17 +4619,17 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
- // For 'inspect_net': print to outstream_ all arcs in
+ // For 'inspect_net': print to outstream_ all arcs in
// \a transitions. Return the number of arcs.
unsigned int XfstCompiler::print_arcs
- (const HfstBasicTransducer::HfstTransitions & transitions)
+ (const hfst::implementations::HfstBasicTransitions & transitions)
{
bool first_loop = true;
unsigned int arc_number = 1;
- for (HfstBasicTransducer::HfstTransitions::const_iterator it
+ for (hfst::implementations::HfstBasicTransitions::const_iterator it
= transitions.begin(); it != transitions.end(); it++)
{
- if (first_loop)
+ if (first_loop)
{
output() << "Arcs:";
//hfst_fprintf(outstream_, "Arcs:");
@@ -4587,7 +4652,7 @@ namespace xfst {
else
{
output() << " " << arc_number << ". " << isymbol << ":" << osymbol;
- //hfst_fprintf(outstream_, " %i. %s:%s", arc_number,
+ //hfst_fprintf(outstream_, " %i. %s:%s", arc_number,
// isymbol.c_str(), osymbol.c_str());
}
flush(&output());
@@ -4633,7 +4698,7 @@ namespace xfst {
shortest_path.push_back(state);
}
- // For 'inspect_net': return to level \a level.
+ // For 'inspect_net': return to level \a level.
// Return whether the operation succeeded.
static bool return_to_level
(std::vector<unsigned int> & whole_path,
@@ -4753,7 +4818,7 @@ namespace xfst {
void XfstCompiler::ignore_history_after_index(int index)
{
#ifdef HAVE_READLINE
- for (unsigned int i=(history_length - 1);
+ for (unsigned int i=(history_length - 1);
i > (index - 1); i--)
{
remove_history(i);
@@ -4761,7 +4826,7 @@ namespace xfst {
#endif
}
- // whether arc \a number can be followed in a state
+ // whether arc \a number can be followed in a state
// that has \a number_of_arcs arcs.
bool XfstCompiler::can_arc_be_followed
(int number, unsigned int number_of_arcs)
@@ -4773,12 +4838,12 @@ namespace xfst {
//hfst_fprintf(outstream_, "could not read arc number\n");
return false;
}
- else if (number < 1 || number > number_of_arcs)
+ else if (number < 1 || number > (int)number_of_arcs)
{
- if (number_of_arcs < 1)
+ if (number_of_arcs < 1)
output() << "state has no arcs" << std::endl; //hfst_fprintf(outstream_, "state has no arcs\n");
- else
- output() << "arc number must be between 1 and " << number_of_arcs << std::endl;
+ else
+ output() << "arc number must be between 1 and " << number_of_arcs << std::endl;
// hfst_fprintf(outstream_, "arc number must be between %i and %i\n",
// 1, number_of_arcs);
flush(&output());
@@ -4798,7 +4863,7 @@ namespace xfst {
// "(type '0' if you wish to exit program)\n");
return false;
}
- else if (level < 0 || level > whole_path_length)
+ else if (level < 0 || level > (int)whole_path_length)
{
output() << "no such level: '" << level << "' (current lievel is " << (int)whole_path_length << ")" << std::endl;
flush(&output());
@@ -4809,7 +4874,7 @@ namespace xfst {
return true;
}
- static const char * inspect_net_help_msg =
+ static const char * inspect_net_help_msg =
"'N' transits arc N, '-N' returns to level N, '<' "
"to previous level, '0' quits.\n";
@@ -4839,7 +4904,7 @@ namespace xfst {
flush(&output());
// transitions of current state
- HfstBasicTransducer::HfstTransitions transitions = net[0];
+ hfst::implementations::HfstBasicTransitions transitions = net[0];
// number of arcs in current state
unsigned int number_of_arcs = print_arcs(transitions);
@@ -4856,16 +4921,18 @@ namespace xfst {
if (whole_path.size() < 2) // exit if already in the start state
{
ignore_history_after_index(ind);
+ free(line);
PROMPT_AND_RETURN_THIS;
}
- else if (! return_to_level(whole_path, shortest_path,
- whole_path.size() - 1))
+ else if (! return_to_level(whole_path, shortest_path,
+ hfst::size_t_to_uint(whole_path.size() - 1)))
{
- error() << "FATAL ERROR: could not return to level '" << (int)(whole_path.size() - 1) << "'" << std::endl;
+ error() << "FATAL ERROR: could not return to level '" << (int)(whole_path.size() - 1) << "'" << std::endl;
flush(&error());
- //hfst_fprintf(errorstream_, "FATAL ERROR: could not return to level '%i'\n",
+ //hfst_fprintf(errorstream_, "FATAL ERROR: could not return to level '%i'\n",
// (int)(whole_path.size() - 1));
ignore_history_after_index(ind);
+ free(line);
PROMPT_AND_RETURN_THIS;
}
}
@@ -4875,6 +4942,7 @@ namespace xfst {
int level = atoi(line+1); // skip '-'
if (! can_level_be_reached(level, whole_path.size()))
{
+ free(line);
continue;
}
else if (! return_to_level(whole_path, shortest_path, level))
@@ -4883,6 +4951,7 @@ namespace xfst {
flush(&error());
//hfst_fprintf(errorstream_, "FATAL ERROR: could not return to level '%i'\n", level);
ignore_history_after_index(ind);
+ free(line);
PROMPT_AND_RETURN_THIS;
}
}
@@ -4890,6 +4959,7 @@ namespace xfst {
else if (strcmp(line, "0\n") == 0 || strcmp(line, "0") == 0)
{
ignore_history_after_index(ind);
+ free(line);
PROMPT_AND_RETURN_THIS;
}
// case (4): follow arc
@@ -4904,13 +4974,13 @@ namespace xfst {
{
HfstBasicTransition tr = transitions[number - 1];
output() << " " << tr.get_input_symbol() << ":" << tr.get_output_symbol() << " --> ";
- //hfst_fprintf(outstream_, " %s:%s --> ", tr.get_input_symbol().c_str(),
+ //hfst_fprintf(outstream_, " %s:%s --> ", tr.get_input_symbol().c_str(),
// tr.get_output_symbol().c_str());
append_state_to_paths(whole_path, shortest_path, tr.get_target_state());
}
}
- // update transitions and number of arcs and print information about current level
+ // update transitions and number of arcs and print information about current level
transitions = net[whole_path.back()];
print_level(whole_path, shortest_path);
if (net.is_final_state(whole_path.back()))
@@ -4964,13 +5034,13 @@ namespace xfst {
HfstTransducer * well_formed = contains_regexps(xre_);
// subtract those paths from copy of t
HfstTransducer tc(*t);
- tc.subtract(*well_formed).minimize();
+ tc.subtract(*well_formed);
delete well_formed;
// all paths that contain one or more ^[ or ^]
HfstTransducer * brackets = xre_.compile("$[ \"^[\" | \"^]\" ] ;");
// test if the result is empty
- tc.intersect(*brackets).minimize();
+ tc.intersect(*brackets);
delete(brackets);
HfstTransducer empty(tc.get_type());
bool value = empty.compare(tc, false);
@@ -4992,7 +5062,7 @@ namespace xfst {
return pathstr;
}
- static HfstTransducer * to_literal_transducer(const hfst::StringPairVector & path, hfst::xre::XreCompiler & xre_)
+ /* static HfstTransducer * to_literal_transducer(const hfst::StringPairVector & path, hfst::xre::XreCompiler & xre_)
{
std::string pathstr("[");
for (hfst::StringPairVector::const_iterator it = path.begin(); it != path.end(); it++)
@@ -5028,7 +5098,7 @@ namespace xfst {
HfstTransducer * retval = xre_.compile(p); // XRE
free(p);
return retval;
- }
+ }*/
static std::string to_regexp(const hfst::StringPairVector & path, bool input_side)
{
@@ -5087,7 +5157,7 @@ namespace xfst {
}
HfstBasicTransducer fsm(*tmp);
- try
+ try
{
HfstReplacementsMap replacement_map = fsm.find_replacements((level == UPPER_LEVEL));
@@ -5124,7 +5194,7 @@ namespace xfst {
return *this;
}
- replacement->minimize();
+ replacement->optimize();
HfstBasicTransducer repl(*replacement);
delete replacement;
fsm.insert_transducer(start_state, end_state, repl);
@@ -5140,9 +5210,9 @@ namespace xfst {
// filter out regexps
HfstTransducer * cr = contains_regexp_markers_on_one_side(xre_, (level == UPPER_LEVEL) /*input side*/);
- cr->minimize();
+ cr->optimize();
- result->subtract(*cr).minimize();
+ result->subtract(*cr).optimize();
delete cr;
stack_.pop();
delete tmp;
@@ -5191,7 +5261,7 @@ namespace xfst {
}
}
- FILE * infile = fopen(filename, "r");
+ FILE * infile = hfst::hfst_fopen(filename, "r");
if (infile == NULL)
{
error() << "could not read lexc file" << std::endl;
@@ -5201,8 +5271,14 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
+ if (has_lexc_been_read_)
+ lexc_.reset();
+ else
+ has_lexc_been_read_ = true;
+
lexc_.parse(infile);
t = lexc_.compileLexical();
+
if (0 != fclose(infile))
{
error() << "Could not close file " << filename << std::endl;
@@ -5216,7 +5292,7 @@ namespace xfst {
hfst_fprintf(errorstream_, "foma back-end not enabled, cannot read lexc files\n");
xfst_fail();
PROMPT_AND_RETURN_THIS;
- }
+ }
t = HfstTransducer::read_lexc_ptr(std::string(filename), hfst::FOMA_TYPE, verbose_);
if (t != NULL)
@@ -5241,7 +5317,7 @@ namespace xfst {
XfstCompiler&
XfstCompiler::read_att_from_file(const char * filename)
{
- FILE * infile = fopen(filename, "r");
+ FILE * infile = hfst::hfst_fopen(filename, "r");
if (infile == NULL)
{
error() << "could not read att file " << filename << std::endl;
@@ -5254,7 +5330,7 @@ namespace xfst {
try
{
HfstTransducer * tmp = NULL;
- if (variables_["att-epsilon"].compare("@0@ | @_EPSILON_SYMBOL_@") == 0)
+ if (variables_["att-epsilon"].compare("@0@ | @_EPSILON_SYMBOL_@") == 0)
tmp = new HfstTransducer(infile, format_);
else
tmp = new HfstTransducer(infile, format_, variables_["att-epsilon"]);
@@ -5264,6 +5340,7 @@ namespace xfst {
}
catch (const HfstException & e)
{
+ (void)e;
if (0 != fclose(infile)) {
error() << "Could not close file " << filename << std::endl;
flush(&error()); }
@@ -5281,7 +5358,7 @@ namespace xfst {
}
PROMPT_AND_RETURN_THIS;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::write_att(std::ostream * oss_)
{
std::ostream * oss = get_stream(oss_);
@@ -5292,7 +5369,7 @@ namespace xfst {
PROMPT_AND_RETURN_THIS;
}
const std::stack<HfstTransducer*>&
- XfstCompiler::get_stack() const
+ XfstCompiler::get_stack() const
{
return stack_;
}
@@ -5307,7 +5384,7 @@ namespace xfst {
int
XfstCompiler::parse(const char* filename)
{
- hxfstin = fopen(filename, "r");
+ hxfstin = hfst::hfst_fopen(filename, "r");
if (hxfstin == NULL)
{
error() << "could not open " << filename << " for reading" << std::endl;
@@ -5390,7 +5467,7 @@ namespace xfst {
{
return output_to_console_;
}
- XfstCompiler&
+ XfstCompiler&
XfstCompiler::setVerbosity(bool verbosity)
{
verbose_ = verbosity;
diff --git a/libhfst/src/parsers/XfstCompiler.h b/libhfst/src/parsers/XfstCompiler.h
index a63555c..76c45b2 100644
--- a/libhfst/src/parsers/XfstCompiler.h
+++ b/libhfst/src/parsers/XfstCompiler.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
//! @file XfstCompiler.h
@@ -33,7 +33,7 @@
#include "XreCompiler.h"
#include "LexcCompiler.h"
-namespace hfst {
+namespace hfst {
//! @brief hfst::xfst namespace contains all functions needed to parse XFST scritpts
namespace xfst {
@@ -67,6 +67,8 @@ class XfstCompiler
XfstCompiler();
//! @brief Create compiler for @a impl format transducers
XfstCompiler(hfst::ImplementationType impl);
+ //! @brief Destructor.
+ ~XfstCompiler();
//! @brief Add properties from file, one property per line
//! @todo properties cannot be stored in HFST automata
@@ -448,10 +450,10 @@ class XfstCompiler
/* Compile a regex string starting from \a indata, store the resulting
- transducer to variable XfstCompiler::latest_regex_compiled and
+ transducer to variable XfstCompiler::latest_regex_compiled and
store the number of characters read from \a indata to \a chars_read.
- This function is used by the xfst lexer to determine where a regex
+ This function is used by the xfst lexer to determine where a regex
starting from \a indata ends. For example, if we have the input
regex foo:bar ! this is a comment with a semicolon;
@@ -461,7 +463,7 @@ class XfstCompiler
the xfst lexer would read "regex" and then call 'compile_regex' on
the substring starting right after "regex". 'compile_regex' would
then read until the semicolon ending the regex, i.e. the one right
- after "baz" and store the resulting transducer to
+ after "baz" and store the resulting transducer to
'latest_regex_compiled'.
The xfst lexer could then put back all characters after index
@@ -507,7 +509,7 @@ class XfstCompiler
// Needed in interactive mode where user input is read line by line.
bool quit_requested() const;
//! @brief Handle unknown command \a s.
- // @return Whether the parser should go on, 0 signifying true.
+ // @return Whether the parser should go on, 0 signifying true.
int unknown_command(const char * s);
// For xfst parser.
bool get_fail_flag() const;
@@ -530,7 +532,7 @@ class XfstCompiler
On Unix and Mac, output() and error() just return members output_ and error_,
and flush(std::ostream *) does nothing. On windows, it is possible that
output() and error() return an ostringstream which is actually printed when
- flush(std::ostream *) is called.
+ flush(std::ostream *) is called.
This delayed printing is needed because Windows makes a difference between
standard output and error streams and console output and error streams.
@@ -547,25 +549,25 @@ class XfstCompiler
void flush(std::ostream * oss);
protected:
- //! @brief Get the prompt that is used when applying up or down
+ //! @brief Get the prompt that is used when applying up or down
//! (as specified by \a direction).
const char* get_apply_prompt(ApplyDirection direction);
//! @brief Get the print symbol for \a symbol.
//! @see print_flags
const char* get_print_symbol(const char* symbol);
- //! @brief Print \a n first paths (or all, if n is negative)
+ //! @brief Print \a n first paths (or all, if n is negative)
//! from \a paths to \a outfile.
- bool print_paths(const hfst::HfstTwoLevelPaths &paths,
+ bool print_paths(const hfst::HfstTwoLevelPaths &paths,
std::ostream * oss = &std::cout, int n=-1);
- //! @brief Print \a n first paths (or all, if n is negative)
+ //! @brief Print \a n first paths (or all, if n is negative)
//! from \a paths to \a outfile.
- bool print_paths(const hfst::HfstOneLevelPaths &paths,
+ bool print_paths(const hfst::HfstOneLevelPaths &paths,
std::ostream * oss = &std::cout, int n=-1);
// A method used by function print_longest_string_or_its_size.
XfstCompiler& print_one_string_or_its_size
(std::ostream * oss, const HfstTwoLevelPaths & paths, const char * level, bool print_size);
//! @brief Print the longest string of topmost transducer in the stack
- //! (if print_size is false) or the size of that string (if print_size is true)
+ //! (if print_size is false) or the size of that string (if print_size is true)
//! to \a outfile.
XfstCompiler& print_longest_string_or_its_size(std::ostream * oss, bool print_size);
//! @brief Try to extract a maximum of \a number paths from topmost
@@ -579,7 +581,7 @@ class XfstCompiler
XfstCompiler& read_text_or_spaced(const char * filename, bool spaces);
//! @brief Convert format of \a t read from file \a filename to common
- //! format used by this xfst compiler and print a warning message
+ //! format used by this xfst compiler and print a warning message
//! about loss of information during conversion, if needed.
void convert_to_common_format
(HfstTransducer * t, const char * filename = NULL);
@@ -593,7 +595,7 @@ class XfstCompiler
//! (if \a definitions is true).
//!
//! Transducers are pushed to the stack in the order they are read from the file.
- //! This is because 'write_stack' writes the transducers starting from the
+ //! This is because 'write_stack' writes the transducers starting from the
//! bottom-most transducer in the stack.
//!
//! Definitions are added using the function 'add_loaded_definition'.
@@ -606,7 +608,7 @@ class XfstCompiler
//! In both cases, a warning message is printed.
XfstCompiler& add_loaded_definition(HfstTransducer * t);
- //! @brief Set fail flag to true if quit-on-fail is ON,
+ //! @brief Set fail flag to true if quit-on-fail is ON,
//! else do nothing.
void xfst_fail();
@@ -627,10 +629,10 @@ class XfstCompiler
bool can_arc_be_followed(int number, unsigned int number_of_arcs);
- unsigned int print_arcs(const HfstBasicTransducer::HfstTransitions & transitions);
+ unsigned int print_arcs(const hfst::implementations::HfstBasicTransitions & transitions);
//! @brief Perform lookup on the top transducer using strings in \a infile.
- //! \a direction specifies whether apply is done on input (up) or output (down)
+ //! \a direction specifies whether apply is done on input (up) or output (down)
//! side. If infile is stdin, interactive mode with prompts is used.
//! The results are printed to standard output.
XfstCompiler& apply(FILE* infile, ApplyDirection direction);
@@ -639,8 +641,8 @@ class XfstCompiler
//! If the stack is empty, print a warning.
XfstCompiler& apply_unary_operation(UnaryOperation operation);
//! @brief Apply \a operation on two top transducers in the stack.
- //! The top transducers are popped, the operation is applied
- //! (the topmost transducer is the first transducer in the operation),
+ //! The top transducers are popped, the operation is applied
+ //! (the topmost transducer is the first transducer in the operation),
//! and the result is pushed to the top of the stack.
//! If the stack has less than two transducers, print a warning.
XfstCompiler& apply_binary_operation(BinaryOperation operation);
@@ -664,7 +666,7 @@ class XfstCompiler
//! Use \a promptstr as prompt for readline, or print it to stderr if readline is not in use.
char * xfst_getline(FILE * file, const std::string & promptstr = "");
- //! @brief Remove newline ('\n' and '\r') from the end of \a str.
+ //! @brief Remove newline ('\n' and '\r') from the end of \a str.
char * remove_newline(char * str);
//! @brief Print weight.
@@ -673,7 +675,7 @@ class XfstCompiler
//! @brief Get current readline history index.
int current_history_index();
- //! @brief Remove all readline history after \a index.
+ //! @brief Remove all readline history after \a index.
void ignore_history_after_index(int index);
//! @brief Get the precision that is used when printing weights.
@@ -723,7 +725,6 @@ class XfstCompiler
std::map<std::string,std::string> variables_;
std::map<std::string,std::string> properties_;
std::map<std::string,std::set<std::string> > lists_;
- hfst::HfstTransducer* last_defined_;
hfst::ImplementationType format_;
bool verbose_;
bool verbose_prompt_;
diff --git a/libhfst/src/parsers/XreCompiler.cc b/libhfst/src/parsers/XreCompiler.cc
index b159da1..6e10cf9 100644
--- a/libhfst/src/parsers/XreCompiler.cc
+++ b/libhfst/src/parsers/XreCompiler.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
//! @file XreCompiler.cc
@@ -17,13 +17,14 @@
#ifdef WINDOWS
#include "hfst-string-conversions.h"
-#endif
+#endif
#ifndef UNIT_TEST
namespace hfst { namespace xre {
- unsigned int cr=0; // chars read from xre input
+ unsigned int cr=0; // number of chars read from xre input
+ unsigned int lr=1; // number of lines read from xre input
std::set<unsigned int> positions;
char * position_symbol = NULL;
std::ostream * error_(&std::cerr);
@@ -33,8 +34,9 @@ namespace hfst { namespace xre {
bool output_to_console_(false);
#endif
bool verbose_(false);
+ std::set<std::string> * defined_multichar_symbols_(NULL);
-XreCompiler::XreCompiler() :
+XreCompiler::XreCompiler() :
definitions_(),
function_definitions_(),
function_arguments_(),
@@ -70,6 +72,14 @@ XreCompiler::XreCompiler(hfst::ImplementationType impl) :
#endif
{}
+ XreCompiler::~XreCompiler()
+ {
+ for(std::map<std::string,hfst::HfstTransducer*>::iterator it
+ = definitions_.begin(); it != definitions_.end(); it++)
+ {
+ delete it->second;
+ }
+ }
void XreCompiler::set_verbosity(bool verbose)
{
@@ -152,13 +162,13 @@ XreCompiler::define(const std::string& name, const std::string& xre)
HfstTransducer* compiled = compile(xre);
if (compiled == NULL)
{
- //fprintf(stderr, "error in XreCompiler::define: xre '%s' could not be parsed, leaving %s undefined\n",
+ //fprintf(stderr, "error in XreCompiler::define: xre '%s' could not be parsed, leaving %s undefined\n",
// xre.c_str(), name.c_str());
//*errorstream_ << "error in XreCompiler::define: xre '" << xre << "' could not be parsed, leaving " << name << "undefined" << std::endl;
if (this->verbose_)
{
std::ostream * err = get_stream(get_error_stream());
- *err << "error: could not parse '" << xre << "', leaving '" << name << "' undefined" << std::endl;
+ *err << "error: could not parse '" << xre << "', leaving '" << name << "' undefined" << std::endl;
flush(err);
}
return false;
@@ -167,7 +177,7 @@ XreCompiler::define(const std::string& name, const std::string& xre)
return true;
}
-void
+void
XreCompiler::define_list(const std::string& name, const std::set<std::string>& symbol_list)
{
list_definitions_[name] = symbol_list;
@@ -180,10 +190,10 @@ XreCompiler::define(const std::string& name, const HfstTransducer & transducer)
}
bool
-XreCompiler::define_function(const std::string& name,
+XreCompiler::define_function(const std::string& name,
unsigned int arguments,
const std::string& xre)
-{
+{
//fprintf(stderr, "XreCompiler: defining function %s(@%i) = [%s]\n", name.c_str(), arguments, xre.c_str()); // DEBUG
function_arguments_[name] = arguments;
function_definitions_[name] = xre;
@@ -207,14 +217,35 @@ XreCompiler::is_function_definition(const std::string & name)
}
void
-XreCompiler::undefine(const std::string& name)
+XreCompiler::undefine(const std::string& name)
{
if (definitions_.find(name) != definitions_.end())
{
+ delete definitions[name];
definitions_.erase(name);
}
}
+void
+XreCompiler::remove_defined_multichar_symbols()
+{
+ if (defined_multichar_symbols_ != NULL)
+ {
+ delete defined_multichar_symbols_;
+ defined_multichar_symbols_ = NULL;
+ }
+}
+
+void
+XreCompiler::add_defined_multichar_symbol(const std::string & symbol)
+{
+ if (defined_multichar_symbols_ == NULL)
+ {
+ defined_multichar_symbols_ = new std::set<std::string>();
+ }
+ defined_multichar_symbols_->insert(symbol);
+}
+
extern bool expand_definitions;
extern bool harmonize_;
extern bool harmonize_flags_;
@@ -249,9 +280,19 @@ XreCompiler::compile(const std::string& xre)
//std::cerr << "XreCompiler: " << this << " : compile(\"" << xre << "\")" << std::endl;
unsigned int cr_before = cr;
cr = 0;
- HfstTransducer * retval = hfst::xre::compile(xre, definitions_, function_definitions_, function_arguments_, list_definitions_, format_);
- cr = cr_before;
- return retval;
+ try
+ {
+ HfstTransducer * retval = hfst::xre::compile(xre, definitions_, function_definitions_, function_arguments_, list_definitions_, format_);
+ cr = cr_before;
+ return retval;
+ }
+ catch (const char * msg)
+ {
+ if (strcmp(msg, "Allocation of memory failed in Mem::add_buffer!") == 0) // sfst backend
+ HFST_THROW_MESSAGE(HfstException, "Allocation of memory failed in SFST backend.");
+ else
+ HFST_THROW_MESSAGE(HfstException, msg);
+ }
}
HfstTransducer*
@@ -261,10 +302,20 @@ XreCompiler::compile_first(const std::string& xre, unsigned int & chars_read)
//std::cerr << "XreCompiler: " << this << " : compile_first(\"" << xre << "\"";
unsigned int cr_before = cr;
cr = 0;
- HfstTransducer * retval = hfst::xre::compile_first(xre, definitions_, function_definitions_, function_arguments_, list_definitions_, format_, chars_read);
- //std::cerr << ", " << chars_read << ")" << std::endl;
- cr = cr_before;
- return retval;
+ try
+ {
+ HfstTransducer * retval = hfst::xre::compile_first(xre, definitions_, function_definitions_, function_arguments_, list_definitions_, format_, chars_read);
+ //std::cerr << ", " << chars_read << ")" << std::endl;
+ cr = cr_before;
+ return retval;
+ }
+ catch (const char * msg)
+ {
+ if (strcmp(msg, "Allocation of memory failed in Mem::add_buffer!") == 0) // sfst backend
+ HFST_THROW_MESSAGE(HfstException, "Allocation of memory failed in SFST backend.");
+ else
+ HFST_THROW_MESSAGE(HfstException, msg);
+ }
}
bool XreCompiler::get_positions_of_symbol_in_xre
@@ -274,17 +325,18 @@ bool XreCompiler::get_positions_of_symbol_in_xre
positions.clear();
unsigned int cr_before = cr;
cr = 0;
- HfstTransducer * compiled =
+ HfstTransducer * compiled =
hfst::xre::compile(xre, definitions_, function_definitions_, function_arguments_, list_definitions_, format_);
free(position_symbol);
position_symbol = NULL;
if (compiled == NULL)
{
/*fprintf(stderr, "error in XreCompiler::get_positions_of_symbol_in_xre: xre '%s' "
- "could not be parsed, positions of symbol %s not found\n",
+ "could not be parsed, positions of symbol %s not found\n",
xre.c_str(), symbol.c_str());*/
return false;
}
+ delete compiled;
positions_ = positions;
cr = cr_before;
return true;
@@ -305,19 +357,19 @@ int
main(int, char**)
{
std::cout << "Unit tests for " __FILE__ ":";
- std::cout << std::endl << "constructors: ";
- std::cout << " (default)...";
+ std::cout << std::endl << "constructors: " << std::endl;
+ std::cout << " (default)..." << std::endl;;
XreCompiler defaultXre();
#if HAVE_SFST
- std::cout << " (SFST)...";
+ std::cout << " (SFST)..." << std::endl;;
XreCompiler sfstXre = XreCompiler(SFST_TYPE);
#endif
#if HAVE_OPENFST
- std::cout << " (OpenFst)...";
+ std::cout << " (OpenFst)..." << std::endl;;
XreCompiler ofstXre = XreCompiler(TROPICAL_OPENFST_TYPE);
#endif
#if HAVE_FOMA
- std::cout << " (Foma)...";
+ std::cout << " (Foma)..." << std::endl;;
XreCompiler fomaXre = XreCompiler(FOMA_TYPE);
#endif
HfstBasicTransducer basicCat;
@@ -363,87 +415,87 @@ main(int, char**)
basicAaOrBc.add_transition(0, HfstBasicTransition(3, "b", "b", 0));
basicAaOrBc.add_transition(3, HfstBasicTransition(2, "c", "c", 0));
basicAaOrBc.set_final_weight(2, 0);
- std::cout << std::endl << "compilation: ";
+ std::cout << std::endl << "compilation: " << std::endl;;
#if HAVE_SFST
- std::cout << "sfst compile(c a t)...";
+ std::cout << " sfst compile(c a t)..." << std::endl;;
HfstTransducer* sfstCat = sfstXre.compile("c a t");
assert(sfstCat != 0);
assert(sfstCat->compare(HfstTransducer(basicCat, SFST_TYPE)));
delete sfstCat;
- std::cout << "(f i:o 0:u g h t)...";
+ std::cout << " (f i:o 0:u g h t)..." << std::endl;;
HfstTransducer* sfstFight = sfstXre.compile("f i:o 0:u g h t");
assert(sfstFight != 0);
assert(sfstFight->compare(HfstTransducer(basicFight, SFST_TYPE)));
delete sfstFight;
- std::cout << "(c a t | dog)...";
+ std::cout << " (c a t | dog)..." << std::endl;;
HfstTransducer* sfstCatOrDog = sfstXre.compile("c a t | dog");
assert(sfstCatOrDog != 0);
assert(sfstCatOrDog->compare(HfstTransducer(basicCatOrDog, SFST_TYPE)));
delete sfstCatOrDog;
- std::cout << "(a a | b c)...";
+ std::cout << " (a a | b c)..." << std::endl;;
HfstTransducer* sfstAaOrBc = sfstXre.compile("a a | b c");
assert(sfstAaOrBc != 0);
assert(sfstAaOrBc->compare(HfstTransducer(basicAaOrBc, SFST_TYPE)));
delete sfstAaOrBc;
#endif
#if HAVE_OPENFST
- std::cout << "ofst compile(c a t)...";
+ std::cout << " ofst compile(c a t)..." << std::endl;;
HfstTransducer* ofstCat = ofstXre.compile("c a t");
assert(ofstCat != 0);
assert(ofstCat->compare(HfstTransducer(basicCat, TROPICAL_OPENFST_TYPE)));
delete ofstCat;
- std::cout << "(f i:o 0:u g h t)...";
+ std::cout << " (f i:o 0:u g h t)..." << std::endl;;
HfstTransducer* ofstFight = ofstXre.compile("f i:o 0:u g h t");
assert(ofstFight != 0);
assert(ofstFight->compare(HfstTransducer(basicFight,
TROPICAL_OPENFST_TYPE)));
delete ofstFight;
- std::cout << "(c a t | dog)...";
+ std::cout << " (c a t | dog)..." << std::endl;;
HfstTransducer* ofstCatOrDog = ofstXre.compile("c a t | dog");
assert(ofstCatOrDog != 0);
assert(ofstCatOrDog->compare(HfstTransducer(basicCatOrDog,
TROPICAL_OPENFST_TYPE)));
delete ofstCatOrDog;
- std::cout << "(a a | b c)...";
+ std::cout << " (a a | b c)..." << std::endl;;
HfstTransducer* ofstAaOrBc = ofstXre.compile("a a | b c");
assert(ofstAaOrBc != 0);
- assert(ofstAaOrBc->compare(HfstTransducer(basicAaOrBc,
+ assert(ofstAaOrBc->compare(HfstTransducer(basicAaOrBc,
TROPICAL_OPENFST_TYPE)));
delete ofstAaOrBc;
#endif
#if HAVE_FOMA
- std::cout << "foma compile(c a t)...";
+ std::cout << " foma compile(c a t)..." << std::endl;;
HfstTransducer* fomaCat = fomaXre.compile("c a t");
assert(fomaCat != 0);
assert(fomaCat->compare(HfstTransducer(basicCat, FOMA_TYPE)));
delete fomaCat;
- std::cout << "(f i:o 0:u g h t)...";
+ std::cout << " (f i:o 0:u g h t)..." << std::endl;;
HfstTransducer* fomaFight = fomaXre.compile("f i:o 0:u g h t");
assert(fomaFight != 0);
assert(fomaFight->compare(HfstTransducer(basicFight, FOMA_TYPE)));
delete fomaFight;
- std::cout << "(c a t | dog)...";
+ std::cout << " (c a t | dog)..." << std::endl;;
HfstTransducer* fomaCatOrDog = fomaXre.compile("c a t | dog");
assert(fomaCatOrDog != 0);
assert(fomaCatOrDog->compare(HfstTransducer(basicCatOrDog, FOMA_TYPE)));
delete fomaCatOrDog;
- std::cout << "(a a | b c)...";
+ std::cout << " (a a | b c)..." << std::endl;;
HfstTransducer* fomaAaOrBc = fomaXre.compile("a a | b c");
assert(fomaAaOrBc != 0);
assert(fomaAaOrBc->compare(HfstTransducer(basicAaOrBc, FOMA_TYPE)));
delete fomaAaOrBc;
#endif
- std::cout << std::endl << "define:";
+ std::cout << std::endl << "define:" << std::endl;;
#if HAVE_SFST
- std::cout << "sfst define(vowels, a | e | i | o | u | y)...";
+ std::cout << " sfst define(vowels, a | e | i | o | u | y)..." << std::endl;;
sfstXre.define("vowels", "a | e | i | o | u | y");
#endif
#if HAVE_OPENFST
- std::cout << "openfst define(vowels, a | e | i | o | u | y)...";
+ std::cout << " openfst define(vowels, a | e | i | o | u | y)..." << std::endl;;
ofstXre.define("vowels", "a | e | i | o | u | y");
#endif
#if HAVE_FOMA
- std::cout << "foma define(vowels, a | e | i | o | u | y)...";
+ std::cout << " foma define(vowels, a | e | i | o | u | y)..." << std::endl;;
fomaXre.define("vowels", "a | e | i | o | u | y");
#endif
std::cout << "ok." << std::endl;
diff --git a/libhfst/src/parsers/XreCompiler.h b/libhfst/src/parsers/XreCompiler.h
index 70b885c..2e9b718 100644
--- a/libhfst/src/parsers/XreCompiler.h
+++ b/libhfst/src/parsers/XreCompiler.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
//! @file XreCompiler.h
@@ -31,8 +31,8 @@
#include <cstdio>
#include "../HfstDataTypes.h"
-namespace hfst {
-//! @brief hfst::xre namespace is used for all functions related to Xerox
+namespace hfst {
+//! @brief hfst::xre namespace is used for all functions related to Xerox
//! Regular Expresisions (XRE) parsing.
namespace xre {
@@ -70,6 +70,8 @@ class XreCompiler
XreCompiler(hfst::ImplementationType impl);
// ...
XreCompiler(const struct XreConstructorArguments & args);
+ // ...
+ ~XreCompiler();
//! @brief Add a definition macro.
//! Compilers will replace arcs labeled @a name, with the transducer
@@ -87,13 +89,13 @@ class XreCompiler
//! @param xre The regex defining the function. Function arguments must be named
//! as '"@name(N@"' where name is \a name (without the left parenthesis)
//! and N the order of the argument.
- //! For example a definition
+ //! For example a definition
//! define_function("Concat(", 2, " [\"@Concat(1@\" \"@Concat(2@\"] ");
//! defines a function that calculates the concatenation of its first and second arguments.
//! A call
//! compile.("[ Concat(foo, bar) ];");
//! then returns a transducer [ foo bar ].
- bool define_function(const std::string& name,
+ bool define_function(const std::string& name,
unsigned int arguments,
const std::string& xre);
@@ -108,6 +110,9 @@ class XreCompiler
//! @brief Remove a definition macro.
void undefine(const std::string& name);
+ void add_defined_multichar_symbol(const std::string & symbol);
+ void remove_defined_multichar_symbols();
+
//! @brief Compile a transducer defined by @a xre.
//! May return a pointer to @e empty transducer on non-fatal error.
//! A null pointer is returned on fatal error, if abort is not called.
@@ -121,7 +126,7 @@ class XreCompiler
HfstTransducer* compile_first(const std::string& xre, unsigned int & chars_read);
//! @brief Whether the last regex compiled contained only comments.
- //!
+ //!
//! In that case, the last call to compile ot compile_first has returned NULL,
//! which also signals an error during regex compilation.
diff --git a/tools/src/hfst-twolc/src/alphabet_src/Alphabet.cc b/libhfst/src/parsers/alphabet_src/Alphabet.cc
similarity index 91%
rename from tools/src/hfst-twolc/src/alphabet_src/Alphabet.cc
rename to libhfst/src/parsers/alphabet_src/Alphabet.cc
index 1506312..e14d77e 100644
--- a/tools/src/hfst-twolc/src/alphabet_src/Alphabet.cc
+++ b/libhfst/src/parsers/alphabet_src/Alphabet.cc
@@ -34,7 +34,7 @@ SymbolPairVector * Alphabet::get_symbol_pair_vector(const SymbolPair &pair)
}
const OtherSymbolTransducer &Alphabet::get_transducer(const SymbolPair &pair)
-{ return alphabet.has_key(pair) ? alphabet[pair] : compute(pair); }
+{ return alphabet.has_key(pair) ? alphabet[pair] : compute(pair); }
bool Alphabet::is_pair(const std::string &input,const std::string &output)
{
@@ -63,8 +63,8 @@ void Alphabet::define_singleton_set(const std::string &name)
bool Alphabet::is_set_pair(const SymbolPair &pair) const
{
- return
- pair.first.find("__HFST_TWOLC_SET_NAME=") != std::string::npos ||
+ return
+ pair.first.find("__HFST_TWOLC_SET_NAME=") != std::string::npos ||
pair.second.find("__HFST_TWOLC_SET_NAME=") != std::string::npos;
}
@@ -81,17 +81,17 @@ const OtherSymbolTransducer &Alphabet::compute(const SymbolPair &pair)
OtherSymbolTransducer pair_transducer;
if (diacritics.has_element(input))
- {
+ {
pair_transducer.apply(&HfstTransducer::disjunct,
- OtherSymbolTransducer(input,input));
- if (input != output && output != TWOLC_EPSILON &&
+ OtherSymbolTransducer(input,input));
+ if (input != output && output != TWOLC_EPSILON &&
output != TWOLC_UNKNOWN)
{ std::cerr << "Warning: Diacritic " << input << " in pair "
<< input << ":" << output << " will correspond 0."
<< std::endl; }
}
else if (input == TWOLC_UNKNOWN && output == TWOLC_UNKNOWN)
- {
+ {
for (HandySet<SymbolPair>::const_iterator it = alphabet_set.begin();
it != alphabet_set.end();
++it)
@@ -121,7 +121,7 @@ const OtherSymbolTransducer &Alphabet::compute(const SymbolPair &pair)
{ continue; }
if (*it == jt->second)
- {
+ {
pair_transducer.apply(&HfstTransducer::disjunct,
OtherSymbolTransducer
(jt->first,jt->second)); }
@@ -144,7 +144,7 @@ const OtherSymbolTransducer &Alphabet::compute(const SymbolPair &pair)
{ continue; }
if (*it == jt->first)
- {
+ {
pair_transducer.apply(&HfstTransducer::disjunct,
OtherSymbolTransducer
(jt->first,jt->second)); }
@@ -156,16 +156,16 @@ const OtherSymbolTransducer &Alphabet::compute(const SymbolPair &pair)
const SymbolRange &input_set = sets[input];
const SymbolRange &output_set = sets[output];
- for (SymbolRange::const_iterator it = input_set.begin();
- it != input_set.end();
+ for (SymbolRange::const_iterator it = input_set.begin();
+ it != input_set.end();
++it)
{
- for (SymbolRange::const_iterator jt = output_set.begin();
+ for (SymbolRange::const_iterator jt = output_set.begin();
jt != output_set.end();
++jt)
{
if (is_pair(*it,*jt))
- {
+ {
pair_transducer.apply(&HfstTransducer::disjunct,
OtherSymbolTransducer(*it,*jt));
}
@@ -178,20 +178,20 @@ const OtherSymbolTransducer &Alphabet::compute(const SymbolPair &pair)
}
void Alphabet::define_alphabet_pair(const SymbolPair &pair)
-{
- alphabet_set.insert(pair);
+{
+ alphabet_set.insert(pair);
input_symbols.insert(pair.first);
output_symbols.insert(pair.second);
}
void Alphabet::define_diacritics(const SymbolRange &diacs)
-{
- diacritics.insert(diacs.begin(),diacs.end());
+{
+ diacritics.insert(diacs.begin(),diacs.end());
for (HandySet<std::string>::iterator it = diacritics.begin();
it != diacritics.end();
++it)
- {
- alphabet_set.erase(SymbolPair(*it,*it));
+ {
+ alphabet_set.erase(SymbolPair(*it,*it));
alphabet_set.erase(SymbolPair(*it,TWOLC_EPSILON));
input_symbols.erase(*it);
output_symbols.erase(*it);
@@ -208,23 +208,23 @@ int main(void)
{
bool have_openfst = false;
#if HAVE_OPENFST
- have_openfst = true;
+ have_openfst = true;
#endif //HAVE_OPENFST
bool have_sfst = false;
#if HAVE_SFST
- have_sfst = true;
+ have_sfst = true;
#endif //HAVE_SFST
bool have_foma = false;
#if HAVE_FOMA
- have_foma = true;
+ have_foma = true;
#endif // HAVE_FOMA
OtherSymbolTransducer::set_transducer_type
- (have_openfst ? hfst::TROPICAL_OPENFST_TYPE :
+ (have_openfst ? hfst::TROPICAL_OPENFST_TYPE :
have_sfst ? hfst::SFST_TYPE :
- have_foma ? hfst::FOMA_TYPE :
+ have_foma ? hfst::FOMA_TYPE :
hfst::ERROR_TYPE);
Alphabet alphabet;
diff --git a/tools/src/hfst-twolc/src/alphabet_src/Alphabet.h b/libhfst/src/parsers/alphabet_src/Alphabet.h
similarity index 100%
rename from tools/src/hfst-twolc/src/alphabet_src/Alphabet.h
rename to libhfst/src/parsers/alphabet_src/Alphabet.h
diff --git a/tools/src/hfst-twolc/src/commandline_src/CommandLine.cc b/libhfst/src/parsers/commandline_src/CommandLine.cc
similarity index 88%
rename from tools/src/hfst-twolc/src/commandline_src/CommandLine.cc
rename to libhfst/src/parsers/commandline_src/CommandLine.cc
index 0c4b25d..4be392e 100644
--- a/tools/src/hfst-twolc/src/commandline_src/CommandLine.cc
+++ b/libhfst/src/parsers/commandline_src/CommandLine.cc
@@ -1,32 +1,40 @@
//! @file CommandLine.cc
#include "CommandLine.h"
+#include <cstring>
+#include <fstream>
+
+#ifndef _MSC_VER
+# include <getopt.h>
+#else
+# include "../../../hfst-getopt.h"
+#endif
void CommandLine::print_version(void)
{
// c.f. http://www.gnu.org/prep/standards/standards.html#g_t_002d_002dversion
- std::cerr << std::endl << PROGRAM_NAME << " " << 0 << " ("
+ std::cerr << std::endl << PROGRAM_NAME << " " << 0 << " ("
<< PACKAGE_STRING << ")" << std::endl
<< "Copyright (C) 2010 University of Helsinki," << std::endl
<< "License GPLv3: GNU GPL version 3 " << std::endl
<< "<http://gnu.org/licenses/gpl.html>" << std::endl
- << "This is free software: you are free to change and "
+ << "This is free software: you are free to change and "
<< std::endl
<< "redistribute it." << std::endl
- << "There is NO WARRANTY, to the extent permitted by law."
+ << "There is NO WARRANTY, to the extent permitted by law."
<< std::endl << std::endl;
}
void CommandLine::print_usage(void)
{
std::cerr << std::endl
- << "Usage: " << PROGRAM_NAME << " [OPTIONS...] INFILE"
- << std::endl
- << "Usage: " << PROGRAM_NAME << " [OPTIONS...] -i INFILE"
- << std::endl
- << "Usage: " << PROGRAM_NAME << " [OPTIONS...] --input=INFILE"
- << std::endl
- << "Usage: cat INFILE | " << PROGRAM_NAME << " [OPTIONS...]"
+ << "Usage: " << PROGRAM_NAME << " [OPTIONS...] INFILE"
+ << std::endl
+ << "Usage: " << PROGRAM_NAME << " [OPTIONS...] -i INFILE"
+ << std::endl
+ << "Usage: " << PROGRAM_NAME << " [OPTIONS...] --input=INFILE"
+ << std::endl
+ << "Usage: cat INFILE | " << PROGRAM_NAME << " [OPTIONS...]"
<< std::endl
<< "An input file has to be given either using the option -i or"
<< std::endl
@@ -47,8 +55,8 @@ void CommandLine::print_help(void)
<< "written to STDOUT." << std::endl << std::endl;
std::cerr << "Common options:\n"
- << " -h, --help Print help message" << std::endl
- << " -V, --version Print version info" << std::endl
+ << " -h, --help Print help message" << std::endl
+ << " -V, --version Print version info" << std::endl
<< " -u, --usage Print usage" << std::endl
<< " -v, --verbose Print verbosely while processing"
<< std::endl
@@ -62,18 +70,18 @@ void CommandLine::print_help(void)
<< std::endl;
std::cerr << "TwolC grammar options:" << std::endl
- << " -R, --resolve Resolve left-arrow conflicts."
+ << " -R, --resolve Resolve left-arrow conflicts."
<< std::endl
- << " -D, --dont-resolve-right Don't resolve right-arrow conflicts."
+ << " -D, --dont-resolve-right Don't resolve right-arrow conflicts."
<< std::endl
- << " -f, --format=FORMAT Store result in format FORMAT."
+ << " -f, --format=FORMAT Store result in format FORMAT."
<< std::endl << std::endl;
std::cerr << "Format may be one of openfst-log, openfst-tropical, foma or sfst."
<< std::endl << std::endl;
std::cerr << "By default format is openfst-tropical. By default right arrow "
- << std::endl
+ << std::endl
<< "conflicts are resolved and left arrow conflicts are not resolved."
<< std::endl << std::endl;
}
@@ -87,7 +95,7 @@ int CommandLine::parse_options(int argc, char** argv)
char * outfilename = NULL;
bool outputNamed = false;
bool inputNamed = false;
- bool isDebug = false;
+ bool isDebug = false;
char * infilename = NULL;
char * debug_file_name = NULL;
ImplementationType form = hfst::TROPICAL_OPENFST_TYPE;
@@ -97,13 +105,13 @@ int CommandLine::parse_options(int argc, char** argv)
{
static const struct option long_options[] =
{
- {"help", no_argument, 0, 'h'},
- {"version", no_argument, 0, 'V'},
- {"verbose", no_argument, 0, 'v'},
- {"quiet", no_argument, 0, 'q'},
- {"silent", no_argument, 0, 's'},
+ {"help", no_argument, 0, 'h'},
+ {"version", no_argument, 0, 'V'},
+ {"verbose", no_argument, 0, 'v'},
+ {"quiet", no_argument, 0, 'q'},
+ {"silent", no_argument, 0, 's'},
{"usage", no_argument, 0, 'u'},
- {"input", required_argument, 0, 'i'},
+ {"input", required_argument, 0, 'i'},
{"output", required_argument, 0, 'o'},
{"resolve-left",no_argument, 0, 'R'},
{"dont-resolve-right",no_argument, 0, 'D'},
@@ -112,8 +120,8 @@ int CommandLine::parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv,
+ // add tool-specific options here
+ int c = getopt_long(argc, argv,
":hVvqsu" "i:o:" "RDi:d:f:",
long_options, &option_index);
if (-1 == c)
@@ -189,7 +197,7 @@ int CommandLine::parse_options(int argc, char** argv)
else if (std::string("unweighted") == optarg)
{ form = hfst::FOMA_TYPE; }
else
- {
+ {
std::cerr << "Unknown format \"" << optarg << "\"."
<< "Try running with option -h or --help."
<< std::endl;
@@ -197,13 +205,13 @@ int CommandLine::parse_options(int argc, char** argv)
}
break;
case ':':
- std::cerr << "Missing argument for -" << (char)optopt
- << ". Try using --help."
+ std::cerr << "Missing argument for -" << (char)optopt
+ << ". Try using --help."
<< std::endl;
exit(1);
default:
std::cerr << "Unknown commandline option: -" << (char)optopt
- << ". Try using --help."
+ << ". Try using --help."
<< std::endl;
exit(1);
}
@@ -213,23 +221,23 @@ int CommandLine::parse_options(int argc, char** argv)
if (! inputNamed)
{
if ((argc - optind) == 1)
- {
+ {
inputNamed = true;
- infilename = hfst_strdup(argv[optind]);
+ infilename = hfst_strdup(argv[optind]);
}
else if ((argc - optind) > 1)
- {
- std::cerr << "no more than one input rule file may be given"
- << std::endl;
+ {
+ std::cerr << "no more than one input rule file may be given"
+ << std::endl;
exit(1);
}
}
- else
+ else
{
if ((argc - optind) > 0)
- {
- std::cerr << "no more than one input rule file may be given"
- << std::endl;
+ {
+ std::cerr << "no more than one input rule file may be given"
+ << std::endl;
exit(1);
}
@@ -246,9 +254,9 @@ int CommandLine::parse_options(int argc, char** argv)
if (this->has_output_file)
{ this->output_file_name = outfilename; }
this->format = form;
- this->help = help;
- this->usage = usage;
- this->version = version;
+ //this->help = help;
+ //this->usage = usage;
+ //this->version = version;
free(infilename);
free(outfilename);
@@ -289,7 +297,7 @@ std::istream &CommandLine::set_input_file(void)
{
input_file = new std::ifstream(input_file_name.c_str());
if (! input_file->good())
- {
+ {
std::cerr << "File " << input_file_name << " could not be opened!"
<< std::endl;
std::cout << "__HFST_TWOLC_DIE";
@@ -306,11 +314,11 @@ std::ostream &CommandLine::set_output_file(void)
{
output_file = new std::ofstream(output_file_name.c_str());
if (! output_file->good())
- {
+ {
std::cerr << "File " << output_file_name << " could not be opened!"
<< std::endl;
std::cout << "__HFST_TWOLC_DIE";
- exit(1);
+ exit(1);
}
return *output_file;
}
@@ -322,7 +330,7 @@ std::ostream &set_output_file(void);
#ifdef TEST_COMMAND_LINE
std::ostream &operator<<(std::ostream &out,const CommandLine &command_line)
{
- out
+ out
<< "VERBOSE:\t" << command_line.be_verbose << std::endl
<< "QUIET:\t\t" << command_line.be_quiet << std::endl
<< "INFILE EXIST:\t" << command_line.has_input_file << std::endl
@@ -334,7 +342,7 @@ std::ostream &operator<<(std::ostream &out,const CommandLine &command_line)
return out;
}
-int main(int argc, char * argv[])
+int main(int argc, char * argv[])
{
CommandLine command_line(argc,argv);
std::cout << command_line << std::endl;
diff --git a/tools/src/hfst-twolc/src/commandline_src/CommandLine.h b/libhfst/src/parsers/commandline_src/CommandLine.h
similarity index 94%
rename from tools/src/hfst-twolc/src/commandline_src/CommandLine.h
rename to libhfst/src/parsers/commandline_src/CommandLine.h
index 61f993b..a9d79c4 100644
--- a/tools/src/hfst-twolc/src/commandline_src/CommandLine.h
+++ b/libhfst/src/parsers/commandline_src/CommandLine.h
@@ -23,13 +23,7 @@
#include <iostream>
-#ifndef _MSC_VER
-# include <getopt.h>
-#else
-# include "../../../hfst-getopt.h"
-#endif
-
-#include "HfstTransducer.h"
+#include "HfstDataTypes.h"
using hfst::ImplementationType;
/** successful return value for argument parsing routine */
diff --git a/tools/src/hfst-twolc/src/common_globals.h b/libhfst/src/parsers/common_globals.h
similarity index 100%
rename from tools/src/hfst-twolc/src/common_globals.h
rename to libhfst/src/parsers/common_globals.h
diff --git a/tools/src/hfst-twolc/src/grammar_defs.h b/libhfst/src/parsers/grammar_defs.h
similarity index 92%
rename from tools/src/hfst-twolc/src/grammar_defs.h
rename to libhfst/src/parsers/grammar_defs.h
index cd91ce3..2ca4ce9 100644
--- a/tools/src/hfst-twolc/src/grammar_defs.h
+++ b/libhfst/src/parsers/grammar_defs.h
@@ -1,7 +1,7 @@
//! @file grammar_defs.h
-//!
+//!
//! @author Miikka Silfverberg
-//!
+//!
//! @brief Simple data structures for basic twolc grammar concepts.
// This program is free software: you can redistribute it and/or modify
@@ -37,10 +37,10 @@ typedef std::pair<std::string,std::string> SymbolPair;
//! @brief A range of symbols e.g. in a set definition.
typedef std::vector<std::string> SymbolRange;
-//! @brief A range of symbol pairs e.g. in the alphabet.
+//! @brief A range of symbol pairs e.g. in the alphabet.
typedef std::vector<SymbolPair> SymbolPairRange;
-//! @brief A range of symbol pairs e.g. in the alphabet.
+//! @brief A range of symbol pairs e.g. in the alphabet.
typedef std::vector<SymbolPair> SymbolPairVector;
//! @brief A symbols set consisting of a name and a @a SymbolRange.
diff --git a/tools/src/hfst-twolc/src/scanner1.ll b/libhfst/src/parsers/htwolcpre1-lexer.ll
similarity index 51%
rename from tools/src/hfst-twolc/src/scanner1.ll
rename to libhfst/src/parsers/htwolcpre1-lexer.ll
index 9e58bce..5444fd0 100644
--- a/tools/src/hfst-twolc/src/scanner1.ll
+++ b/libhfst/src/parsers/htwolcpre1-lexer.ll
@@ -1,4 +1,4 @@
-%option 8Bit batch yylineno noyywrap nounput
+%option 8Bit batch yylineno noyywrap nounput prefix="htwolcpre1"
%{
// This program is free software: you can redistribute it and/or modify
@@ -20,9 +20,9 @@
// The parser. Supplies flex with its symbol tables.
#ifdef YACC_USE_PARSER_H_EXTENSION
- #include "htwolcpre1.h"
+ #include "htwolcpre1-parser.h"
#else
- #include "htwolcpre1.hh"
+ #include "htwolcpre1-parser.hh"
#endif
// Functions for manipulating strings.
@@ -30,23 +30,42 @@
// input_defs.h declares the inputHandler, which is
// an object that overrides flex' default input reading.
- #include "io_src/input_defs.h"
+ //#include "io_src/input_defs.h"
+
+extern int htwolcpre1error(const char*);
+
+#undef YY_FATAL_ERROR
+#define YY_FATAL_ERROR(msg) htwolcpre1error(msg);
+
+#include "io_src/InputReader.h"
+extern InputReader htwolcpre1_input_reader;
+#undef YY_INPUT
+#define YY_INPUT( buf, result, max_size ) { \
+char c = htwolcpre1_input_reader.input(); \
+if (c == 0) { \
+result = YY_NULL; \
+} \
+ else { \
+buf[0] = c; \
+result = 1; \
+} \
+}
// symbol_queue is used to pass strings from Flex to Bison.
#include "HfstTwolcDefs.h"
- extern HandyDeque<std::string> symbol_queue;
+ extern HandyDeque<std::string> htwolcpre1_symbol_queue;
// This tells Bison that regular expression sections commenced.
bool regexp_start = false;
// This tells Bison that the rule section commenced.
- bool rules_start = false;
+ bool htwolcpre1_rules_start = false;
// Tells whether we are in the where-part of a rule or not.
bool where_seen = false;
// Tells whether we are inside a ( .. ). For variable rules.
- extern bool inside_parenthesis;
+ extern bool htwolcpre1_inside_parenthesis;
void reduce_queue(bool variable_symbol=false);
@@ -59,55 +78,55 @@ FREE_SYMBOL [^*+/\\=\"$?|&^\-\{\}\[\]\(\):;_!%\r\t\n~ ]
%%
-Alphabet/{RESERVED_SYMBOL} {
+Alphabet/{RESERVED_SYMBOL} {
// Alphabet declaration.
- symbol_queue.push_back("__HFST_TWOLC_Alphabet");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_Alphabet");
reduce_queue();
- return ALPHABET_DECLARATION;
+ return ALPHABET_DECLARATION;
}
Diacritics/{RESERVED_SYMBOL} {
// Diacritics declaration.
- symbol_queue.push_back("__HFST_TWOLC_Diacritics");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_Diacritics");
reduce_queue();
- return DIACRITICS_DECLARATION;
-}
-Rule-variables/{RESERVED_SYMBOL} {
+ return DIACRITICS_DECLARATION;
+}
+Rule-variables/{RESERVED_SYMBOL} {
// Rule-variables declaration, is not necessary,
// but is supported for backwards compatibility.
- return VARIABLE_DECLARATION;
+ return VARIABLE_DECLARATION;
}
Definitions/{RESERVED_SYMBOL} {
// Definitions declaration.
- symbol_queue.push_back("__HFST_TWOLC_Definitions");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_Definitions");
reduce_queue();
regexp_start = true;
- return DEFINITION_DECLARATION;
+ return DEFINITION_DECLARATION;
}
-Sets/{RESERVED_SYMBOL} {
+Sets/{RESERVED_SYMBOL} {
// Sets declaration.
- symbol_queue.push_back("__HFST_TWOLC_Sets");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_Sets");
reduce_queue();
- return SETS_DECLARATION;
+ return SETS_DECLARATION;
}
Rules/{RESERVED_SYMBOL} {
// Rules declaration.
- symbol_queue.push_back("__HFST_TWOLC_Rules");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_Rules");
reduce_queue();
- rules_start = true;
+ htwolcpre1_rules_start = true;
regexp_start = true;
- return RULES_DECLARATION;
+ return RULES_DECLARATION;
}
where/{RESERVED_SYMBOL} {
// The symbols until AND occur in rules with variables.
// When the symbols are encountered, nothing is printed,
// since the purpose is to reduce rules with variables
// into sub-case rules without variables.
- where_seen = true;
- return WHERE;
+ where_seen = true;
+ return WHERE;
}
except/{RESERVED_SYMBOL} {
// Separates negative contexts from positive contexts.
- symbol_queue.push_back("__HFST_TWOLC_except");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_except");
reduce_queue();
return EXCEPT;
}
@@ -119,251 +138,251 @@ in/{RESERVED_SYMBOL} { return IN; }
[!].* { /* comments: ignore */ }
[ \t\r] { /* spaces and tabs: ignore */ }
-\n {
+\n {
// For counting lines. Since flex skips ahead with reading symbols,
// it's better to increase the line count in the function reduce_queue().
- symbol_queue.push_back("__HFST_TWOLC_\\n");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_\\n");
}
-\"[^\"]+\" {
+\"[^\"]+\" {
// Rule name.
- symbol_queue.push_back
+ htwolcpre1_symbol_queue.push_back
(std::string("__HFST_TWOLC_RULE_NAME=")+
- replace_substr(yytext," ","__HFST_TWOLC_SPACE"));
+ replace_substr(htwolcpre1text," ","__HFST_TWOLC_SPACE"));
// reduce_queue();
- return RULE_NAME;
+ return RULE_NAME;
}
[*] {
// Kleene star operator.
- symbol_queue.push_back("__HFST_TWOLC_*");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_*");
reduce_queue();
- return STAR;
+ return STAR;
}
[+] {
// Kleene plus operator.
- symbol_queue.push_back("__HFST_TWOLC_+");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_+");
reduce_queue();
- return PLUS;
+ return PLUS;
}
[/] {
// Freely insert operator.
- symbol_queue.push_back("__HFST_TWOLC_/");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_/");
reduce_queue();
- return FREELY_INSERT;
+ return FREELY_INSERT;
}
[~] {
// Complement operator.
- symbol_queue.push_back("__HFST_TWOLC_~");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_~");
reduce_queue();
- return COMPLEMENT;
+ return COMPLEMENT;
}
[\\] {
// Term complement operator.
- symbol_queue.push_back("__HFST_TWOLC_\\");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_\\");
reduce_queue();
- return TERM_COMPLEMENT;
+ return TERM_COMPLEMENT;
}
[$][.] {
// Containment once operator.
- symbol_queue.push_back("__HFST_TWOLC_$.");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_$.");
reduce_queue();
- return CONTAINMENT_ONCE;
+ return CONTAINMENT_ONCE;
}
[$] {
// Containment at least once operator.
- symbol_queue.push_back("__HFST_TWOLC_$");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_$");
reduce_queue();
- return CONTAINMENT;
+ return CONTAINMENT;
}
[?] {
- // Any symbol.
- symbol_queue.push_back("__HFST_TWOLC_?");
- return SYMBOL;
+ // Any symbol.
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_?");
+ return SYMBOL;
}
[?]/[:] {
if (! regexp_start)
{ return QUESTION_MARK; }
- // Any symbol.
- symbol_queue.push_back("__HFST_TWOLC_?");
- return SYMBOL;
+ // Any symbol.
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_?");
+ return SYMBOL;
}
[?]/{RESERVED_EXC_COL} {
if (! regexp_start)
{ return QUESTION_MARK; }
- // Any symbol.
- symbol_queue.push_back("__HFST_TWOLC_?");
- return SYMBOL_SPACE;
+ // Any symbol.
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_?");
+ return SYMBOL_SPACE;
}
-[0]/[:] {
- symbol_queue.push_back("__HFST_TWOLC_0");
- return SYMBOL;
+[0]/[:] {
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_0");
+ return SYMBOL;
}
-[0]/{RESERVED_EXC_COL} {
+[0]/{RESERVED_EXC_COL} {
// Zero symbol.
- symbol_queue.push_back("__HFST_TWOLC_0");
- return SYMBOL_SPACE;
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_0");
+ return SYMBOL_SPACE;
}
[|] {
// Or.
- symbol_queue.push_back("__HFST_TWOLC_|");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_|");
reduce_queue();
- return UNION;
+ return UNION;
}
[&] {
// And.
- symbol_queue.push_back("__HFST_TWOLC_&");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_&");
reduce_queue();
- return INTERSECTION;
+ return INTERSECTION;
}
[\^] {
// Power operator.
- symbol_queue.push_back("__HFST_TWOLC_^");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_^");
reduce_queue();
- return POWER;
+ return POWER;
}
[\-] {
// Difference operator.
- symbol_queue.push_back("__HFST_TWOLC_-");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_-");
reduce_queue();
- return DIFFERENCE;
+ return DIFFERENCE;
}
[0-9],[0-9]+/{RESERVED_EXC_COL} {
- // Number.
- symbol_queue.push_back(+yytext);
- return NUMBER_SPACE;
+ // Number.
+ htwolcpre1_symbol_queue.push_back(+htwolcpre1text);
+ return NUMBER_SPACE;
}
[0-9]+/{RESERVED_EXC_COL} {
- // Number.
- symbol_queue.push_back(yytext);
- return NUMBER_SPACE;
+ // Number.
+ htwolcpre1_symbol_queue.push_back(htwolcpre1text);
+ return NUMBER_SPACE;
}
[0-9]+/[:] {
- // Number.
- symbol_queue.push_back(yytext);
- return NUMBER;
+ // Number.
+ htwolcpre1_symbol_queue.push_back(htwolcpre1text);
+ return NUMBER;
}
[.][#][.]/{RESERVED_SYMBOL} {
// Word boundary.
- symbol_queue.push_back("__HFST_TWOLC_.#.");
- return SYMBOL_SPACE;
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_.#.");
+ return SYMBOL_SPACE;
}
\[ {
// Beginning of a bracketed regex.
- symbol_queue.push_back("__HFST_TWOLC_[");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_[");
reduce_queue();
- return LEFT_SQUARE_BRACKET;
+ return LEFT_SQUARE_BRACKET;
}
\] {
// End of a bracketed regex.
- symbol_queue.push_back("__HFST_TWOLC_]");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_]");
reduce_queue();
- return RIGHT_SQUARE_BRACKET;
+ return RIGHT_SQUARE_BRACKET;
}
\<\[ {
// Beginning of a bracketed regex.
//
// For some bizarre reason [[ can't occur in the string-literal here...
- symbol_queue.push_back("__HFST_TWOLC_[" "[");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_[" "[");
reduce_queue();
- return RE_LEFT_SQUARE_BRACKET;
+ return RE_LEFT_SQUARE_BRACKET;
}
\]\> {
// End of a bracketed regex.
//
// For some bizarre reason ]] can't occur in the string-literal here...
- symbol_queue.push_back("__HFST_TWOLC_]" "]");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_]" "]");
reduce_queue();
- return RE_RIGHT_SQUARE_BRACKET;
+ return RE_RIGHT_SQUARE_BRACKET;
}
\{ {
// Beginning of a bracketed regex.
- symbol_queue.push_back("__HFST_TWOLC_[");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_[");
reduce_queue();
- return LEFT_CURLY_BRACKET;
+ return LEFT_CURLY_BRACKET;
}
\} {
// End of a bracketed regex.
- symbol_queue.push_back("__HFST_TWOLC_]");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_]");
reduce_queue();
- return RIGHT_CURLY_BRACKET;
+ return RIGHT_CURLY_BRACKET;
}
\( {
// Beginning of an optional bracketed regex.
- if (! where_seen)
+ if (! where_seen)
{
- symbol_queue.push_back("__HFST_TWOLC_(");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_(");
reduce_queue();
}
- return LEFT_PARENTHESIS;
+ return LEFT_PARENTHESIS;
}
\) {
// End of an optional bracketed regex.
- if (! where_seen)
+ if (! where_seen)
{
- symbol_queue.push_back("__HFST_TWOLC_)");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_)");
reduce_queue();
}
BEGIN 0;
- return RIGHT_PARENTHESIS;
+ return RIGHT_PARENTHESIS;
}
[/][<][=] {
// Restriction rule operator.
- symbol_queue.push_back("__HFST_TWOLC_/<=");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_/<=");
reduce_queue();
- return LEFT_RESTRICTION_ARROW;
+ return LEFT_RESTRICTION_ARROW;
}
[<][=] {
// Left rule operator.
- symbol_queue.push_back("__HFST_TWOLC_<=");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_<=");
reduce_queue();
- return LEFT_ARROW;
+ return LEFT_ARROW;
}
[=][>] {
// Right rule operator.
- symbol_queue.push_back("__HFST_TWOLC_=>");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_=>");
reduce_queue();
- return RIGHT_ARROW;
+ return RIGHT_ARROW;
}
[<][=][>] {
// Equivalence rule operator.
- symbol_queue.push_back("__HFST_TWOLC_<=>");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_<=>");
reduce_queue();
- return LEFT_RIGHT_ARROW;
+ return LEFT_RIGHT_ARROW;
}
[/][<][=][=] {
// Restriction regular expression rule operator.
- symbol_queue.push_back("__HFST_TWOLC_/<==");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_/<==");
reduce_queue();
- return RE_LEFT_RESTRICTION_ARROW;
+ return RE_LEFT_RESTRICTION_ARROW;
}
[<][=][=] {
// Left regular expression rule operator.
- symbol_queue.push_back("__HFST_TWOLC_<==");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_<==");
reduce_queue();
- return RE_LEFT_ARROW;
+ return RE_LEFT_ARROW;
}
[=][=][>] {
// Right regular expression rule operator.
- symbol_queue.push_back("__HFST_TWOLC_==>");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_==>");
reduce_queue();
- return RE_RIGHT_ARROW;
+ return RE_RIGHT_ARROW;
}
[<][=][=][>] {
// Equivalence regular expression rule operator.
- symbol_queue.push_back("__HFST_TWOLC_<==>");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_<==>");
reduce_queue();
- return RE_LEFT_RIGHT_ARROW;
+ return RE_LEFT_RIGHT_ARROW;
}
[:]/{RESERVED_EXC_PERC_AND_Q_MARK} {
// Pair separator in expressions like "[a:]".
- symbol_queue.push_back("__HFST_TWOLC_: ");
- symbol_queue.push_back("__HFST_TWOLC_?");
- return COLON_SPACE;
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_: ");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_?");
+ return COLON_SPACE;
}
[:] {
// Pair separator.
- symbol_queue.push_back("__HFST_TWOLC_:");
- return COLON;
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_:");
+ return COLON;
}
[;] {
// End-of-line symbol.
@@ -371,42 +390,42 @@ in/{RESERVED_SYMBOL} { return IN; }
{ where_seen = false; }
else
{
- symbol_queue.push_back("__HFST_TWOLC_;");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_;");
reduce_queue();
}
- return SEMI_COLON;
+ return SEMI_COLON;
}
[=] {
- // Equals sign used when defining sets and making definitions.
- symbol_queue.push_back("__HFST_TWOLC_=");
+ // Equals sign used when defining sets and making definitions.
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_=");
reduce_queue();
- return EQUALS;
+ return EQUALS;
}
[_] {
// Center-marker in a rule context.
- symbol_queue.push_back("__HFST_TWOLC__");
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC__");
reduce_queue();
- return CENTER_MARKER;
+ return CENTER_MARKER;
}
-(([%]({RESERVED_SYMBOL}|{FREE_SYMBOL}))|{FREE_SYMBOL})+/[:] {
+(([%]({RESERVED_SYMBOL}|{FREE_SYMBOL}))|{FREE_SYMBOL})+/[:] {
// A symbol which is the left side of a pair e.g. "a" in "a:b".
- std::string symbol =
- remove_white_space(replace_substr(yytext,"\n","__HFST_TWOLC_\\n"));
+ std::string symbol =
+ remove_white_space(replace_substr(htwolcpre1text,"\n","__HFST_TWOLC_\\n"));
if (symbol == "#")
- { symbol_queue.push_back("__HFST_TWOLC_#"); }
+ { htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_#"); }
else
- { symbol_queue.push_back(symbol); }
- return SYMBOL;
+ { htwolcpre1_symbol_queue.push_back(symbol); }
+ return SYMBOL;
}
-(([%]({RESERVED_SYMBOL}|{FREE_SYMBOL}))|{FREE_SYMBOL})+/{RESERVED_EXC_COL} {
+(([%]({RESERVED_SYMBOL}|{FREE_SYMBOL}))|{FREE_SYMBOL})+/{RESERVED_EXC_COL} {
// A symbol which is not the left side of a pair e.g. "b" in "a:b" or "[b]".
- std::string symbol =
- remove_white_space(replace_substr(yytext,"\n","__HFST_TWOLC_\\n"));
+ std::string symbol =
+ remove_white_space(replace_substr(htwolcpre1text,"\n","__HFST_TWOLC_\\n"));
if (symbol == "#")
{ symbol = "__HFST_TWOLC_#"; }
- symbol_queue.push_back(symbol);
- return SYMBOL_SPACE;
+ htwolcpre1_symbol_queue.push_back(symbol);
+ return SYMBOL_SPACE;
}
%%
diff --git a/tools/src/hfst-twolc/src/htwolcpre1.yy b/libhfst/src/parsers/htwolcpre1-parser.yy
similarity index 77%
rename from tools/src/hfst-twolc/src/htwolcpre1.yy
rename to libhfst/src/parsers/htwolcpre1-parser.yy
index 94725e9..41869f9 100644
--- a/tools/src/hfst-twolc/src/htwolcpre1.yy
+++ b/libhfst/src/parsers/htwolcpre1-parser.yy
@@ -27,33 +27,59 @@
#include "grammar_defs.h"
#include "variable_src/RuleSymbolVector.h"
#include "variable_src/RuleVariables.h"
-#include "commandline_src/CommandLine.h"
#include "common_globals.h"
-
- extern int yylineno;
- extern char * yytext;
- extern int yylineno;
- extern char * yytext;
- extern bool regexp_start;
- extern bool rules_start;
- void yyerror(const char * text );
- void warn(const char * warning );
- int yylex();
- int yyparse();
+#include "../HfstExceptionDefs.h"
+
+ extern int htwolcpre1lineno;
+ extern char * htwolcpre1text;
+ extern int htwolcpre1lineno;
+ extern char * htwolcpre1text;
+ extern bool htwolcpre1_rules_start;
+ void htwolcpre1error(const char * text );
+ void htwolcpre1warn(const char * warning );
+ int htwolcpre1lex();
+ int htwolcpre1parse();
void reduce_queue(bool variable_symbol=false);
void set_variable_values(void);
void reduce_symbol_pair(bool no_definitions = false);
void increase_line_counter(void);
std::string &get_symbol_queue_front(void);
void pop_symbol_queue(void);
+ std::ostream * output = NULL;
#define YYERROR_VERBOSE 1
// For displaying the line number in error messages and warnings.
- size_t line_number = 1;
+ size_t htwolcpre1_line_number = 1;
// For reading input one byte at a time.
- InputReader input_reader(line_number);
+ InputReader htwolcpre1_input_reader(htwolcpre1_line_number);
+
+namespace hfst {
+ namespace twolcpre1 {
+
+ void set_input(std::istream & istr)
+ {
+ htwolcpre1_input_reader.set_input(istr);
+ }
+ int parse()
+ {
+ return htwolcpre1parse();
+ }
+ void set_output(std::ostream & ostr)
+ {
+ output = &ostr;
+ }
+ void set_warning_stream(std::ostream & ostr)
+ {
+ htwolcpre1_input_reader.set_warning_stream(ostr);
+ }
+ void set_error_stream(std::ostream & ostr)
+ {
+ htwolcpre1_input_reader.set_error_stream(ostr);
+ }
+ }
+}
// For keeping track of values of variables.
VariableValueMap variable_value_map;
@@ -66,7 +92,7 @@
RuleSymbolVector rule_symbol_vector(variable_value_map);
// The latest symbol that was read is always the last element of this queue.
- HandyDeque<std::string> symbol_queue;
+ HandyDeque<std::string> htwolcpre1_symbol_queue;
// Stores symbol set names.
HandySet<std::string> sets;
@@ -84,7 +110,7 @@
std::vector<std::string> latest_set;
// Tells whether we are inside a ( .. ). For variable rules.
- bool inside_parenthesis = false;
+ bool htwolcpre1_inside_parenthesis = false;
// For temporarily storing a rule variable and its values
StringVector variable_vector;
@@ -94,12 +120,13 @@
%}
+%name-prefix "htwolcpre1"
-%union
+%union
{ int symbol_number; };
- /*
+ /*
All unary operators have stronger precedence than binary ones.
*/
@@ -111,21 +138,21 @@
/* Unary operators ordered by precedence from lowest to highest. */
%right <symbol_number> STAR PLUS
-%left <symbol_number> CONTAINMENT CONTAINMENT_ONCE TERM_COMPLEMENT COMPLEMENT
+%left <symbol_number> CONTAINMENT CONTAINMENT_ONCE TERM_COMPLEMENT COMPLEMENT
%right <symbol_number> POWER
/* "[", "]", "(", ")", "{", "}". */
-%right <symbol_number> RIGHT_SQUARE_BRACKET RIGHT_PARENTHESIS
+%right <symbol_number> RIGHT_SQUARE_BRACKET RIGHT_PARENTHESIS
%left <symbol_number> LEFT_SQUARE_BRACKET LEFT_PARENTHESIS
%right <symbol_number> RIGHT_CURLY_BRACKET
%left <symbol_number> LEFT_CURLY_BRACKET
/* Twolc rule operators */
-%token <symbol_number> LEFT_RESTRICTION_ARROW LEFT_ARROW RIGHT_ARROW
+%token <symbol_number> LEFT_RESTRICTION_ARROW LEFT_ARROW RIGHT_ARROW
%token <symbol_number> LEFT_RIGHT_ARROW
/* Twolc regular expression rule operators */
-%token <symbol_number> RE_LEFT_RESTRICTION_ARROW RE_LEFT_ARROW RE_RIGHT_ARROW
+%token <symbol_number> RE_LEFT_RESTRICTION_ARROW RE_LEFT_ARROW RE_RIGHT_ARROW
%token <symbol_number> RE_LEFT_RIGHT_ARROW
/* Twolc regular expression rule center brackets. */
@@ -133,10 +160,10 @@
%left <symbol_number> RE_LEFT_SQUARE_BRACKET
/* Basic tokens. */
-%token <symbol_number> ALPHABET_DECLARATION DIACRITICS_DECLARATION
+%token <symbol_number> ALPHABET_DECLARATION DIACRITICS_DECLARATION
%token <symbol_number> SETS_DECLARATION DEFINITION_DECLARATION
%token <symbol_number> RULES_DECLARATION VARIABLE_DECLARATION
-%token <symbol_number> COLON WHERE MATCHED_MATCHER
+%token <symbol_number> COLON WHERE MATCHED_MATCHER
%token <symbol_number> MIXED_MATCHER FREELY_MATCHER IN AND
%token <symbol_number> COLON_SPACE SYMBOL_SPACE
%token <symbol_number> SEMI_COLON EQUALS CENTER_MARKER
@@ -147,16 +174,16 @@
ALL: GRAMMAR {}
;
-GRAMMAR: ALPHABET GRAMMAR1
+GRAMMAR: ALPHABET GRAMMAR1
| GRAMMAR1
-GRAMMAR1: DIACRITICS GRAMMAR2
+GRAMMAR1: DIACRITICS GRAMMAR2
| GRAMMAR2
-GRAMMAR2: VARIABLES GRAMMAR3
-| GRAMMAR3
+GRAMMAR2: VARIABLES GRAMMAR3
+| GRAMMAR3
-GRAMMAR3: SETS GRAMMAR4
+GRAMMAR3: SETS GRAMMAR4
| GRAMMAR4
GRAMMAR4: DEFINITIONS GRAMMAR5
@@ -169,44 +196,44 @@ RULES:RULES_DECLARATION RULE_LIST
RULE_LIST: /* empty */
| RULE_LIST RULE
-RULE: RULE_NAME_DECL RULE_CENTER RULE_OPERATOR RULE_CONTEXTS
+RULE: RULE_NAME_DECL RULE_CENTER RULE_OPERATOR RULE_CONTEXTS
NEGATIVE_RULE_CONTEXTS RULE_VARIABLES
{
// If this rule didn't have variables, display it. Otherwise iterate
- // through its variable value combinations and display the rule using the
+ // through its variable value combinations and display the rule using the
// different combinations.
try
{
if (rule_variables.empty())
- { std::cout << rule_symbol_vector.replace_variables(); }
+ { *output << rule_symbol_vector.replace_variables(); }
else
{
for (RuleVariables::const_iterator it = rule_variables.begin();
it != rule_variables.end();
++it)
- {
- it.set_values(variable_value_map);
- std::cout << rule_symbol_vector.replace_variables();
+ {
+ it.set_values(variable_value_map);
+ *output << rule_symbol_vector.replace_variables();
}
}
}
catch (const UnequalSetSize &)
- {
+ {
std::string error
("Variable rules with keyword matched have to have equal length "
"variable value lists.");
- yyerror(error.c_str());
- exit(1);
+ htwolcpre1error(error.c_str());
+ throw HfstException();
}
// Clear all containers, so that we'll be ready to handle the next rule.
rule_symbol_vector.clear();
variable_value_map.clear();
rule_variables.clear();
}
-| RULE_NAME_DECL RE_RULE_CENTER RE_RULE_OPERATOR RULE_CONTEXTS
+| RULE_NAME_DECL RE_RULE_CENTER RE_RULE_OPERATOR RULE_CONTEXTS
NEGATIVE_RULE_CONTEXTS
-{
- std::cout << rule_symbol_vector.replace_variables();
+{
+ *output << rule_symbol_vector.replace_variables();
// Clear all containers, so that we'll be ready to handle the next rule.
rule_symbol_vector.clear();
@@ -215,9 +242,9 @@ NEGATIVE_RULE_CONTEXTS
}
RULE_NAME_DECL: RULE_NAME
-{
+{
// Add the rule name to rule_symbol_vector.
- reduce_queue();
+ reduce_queue();
}
RULE_CENTER: PAIR
@@ -293,7 +320,7 @@ DIACRITICS: DIACRITICS_DECLARATION DIACRITIC_LIST SEMI_COLON_LIST
VARIABLES: VARIABLE_DECLARATION VARIABLE_LIST SEMI_COLON_LIST
-SETS: SETS_DECLARATION SET_LIST
+SETS: SETS_DECLARATION SET_LIST
DEFINITIONS: DEFINITION_DECLARATION DEFINITION_LIST
@@ -313,9 +340,9 @@ RE_LIST: /* empty */
RE: PAIR
| RE POWER NUMBER_SPACE
-{
- symbol_queue.front() =
- std::string("__HFST_TWOLC_NUMBER=") + symbol_queue.front();
+{
+ htwolcpre1_symbol_queue.front() =
+ std::string("__HFST_TWOLC_NUMBER=") + htwolcpre1_symbol_queue.front();
reduce_queue();
}
| RE STAR
@@ -328,8 +355,8 @@ RE: PAIR
| LEFT_CURLY_BRACKET REGULAR_EXPRESSION RIGHT_CURLY_BRACKET
| LEFT_PARENTHESIS REGULAR_EXPRESSION RIGHT_PARENTHESIS
-SET_LIST: /* empty */
-| SET_LIST SET_DEFINITION
+SET_LIST: /* empty */
+| SET_LIST SET_DEFINITION
SYMBOL_LIST: /* empty */
| SYMBOL_LIST SET_SYMBOL
@@ -338,27 +365,27 @@ DIACRITIC_LIST: /* empty */
| DIACRITIC_LIST DIACRITIC_SYMBOL
SET_SYMBOL: GRAMMAR_SYMBOL_SPACE
-{
+{
// Push the set_symbol into latest_set, which contains symbols in the next
// set which will be defined.
if (sets.find(get_symbol_queue_front()) != sets.end())
{
- for (std::vector<std::string>::iterator it =
+ for (std::vector<std::string>::iterator it =
set_symbols
["__HFST_TWOLC_SET_NAME=" + get_symbol_queue_front()].begin();
it != set_symbols
["__HFST_TWOLC_SET_NAME=" + get_symbol_queue_front()].end();
++it)
- {
- std::cout << *it << " ";
+ {
+ *output << *it << " ";
latest_set.push_back(*it);
}
pop_symbol_queue();
}
else
- {
- latest_set.push_back(unescape(get_symbol_queue_front()));
- reduce_queue();
+ {
+ latest_set.push_back(unescape(get_symbol_queue_front()));
+ reduce_queue();
}
}
@@ -373,22 +400,22 @@ SET_DEFINITION: SET_NAME EQUALS SYMBOL_LIST SEMI_COLON_LIST
}
SET_NAME: SYMBOL_SPACE
-{
- // Store the set name in sets and push it at the back of
- // symbol_queue.
+{
+ // Store the set name in sets and push it at the back of
+ // symbol_queue.
sets.insert(get_symbol_queue_front());
- get_symbol_queue_front() =
+ get_symbol_queue_front() =
"__HFST_TWOLC_SET_NAME=" + get_symbol_queue_front();
- set_name = get_symbol_queue_front();
+ set_name = get_symbol_queue_front();
reduce_queue();
}
DEFINITION_NAME: SYMBOL_SPACE
{
- // Store the definition name in definitions and push it at the back of
+ // Store the definition name in definitions and push it at the back of
// symbol_queue.
definitions.insert(get_symbol_queue_front());
- get_symbol_queue_front() =
+ get_symbol_queue_front() =
"__HFST_TWOLC_DEFINITION_NAME=" + get_symbol_queue_front();
reduce_queue();
}
@@ -398,11 +425,11 @@ ALPHABET_PAIR_LIST: /* empty */
PAIR: GRAMMAR_SYMBOL COLON_SPACE
-{
+{
// For pairs "X:" and "X:?".
// Reduce the first three symbols "X", "__HFST_TWOLC_:" and "__HFST_TWOLC_?"
// from symbol_queue.
- reduce_symbol_pair(true);
+ reduce_symbol_pair(true);
}
| COLON GRAMMAR_SYMBOL_SPACE
{
@@ -410,15 +437,15 @@ PAIR: GRAMMAR_SYMBOL COLON_SPACE
// Push a "__HFST_TWOLC_?" onto symbol_queue.
// Reduce the three first symbols "__HFST_TWOLC_?", "__HFST_TWOLC_:" and "X"
// from symbol_queue.
- symbol_queue.push_front("__HFST_TWOLC_?");
- reduce_symbol_pair(true);
+ htwolcpre1_symbol_queue.push_front("__HFST_TWOLC_?");
+ reduce_symbol_pair(true);
}
| GRAMMAR_SYMBOL COLON GRAMMAR_SYMBOL_SPACE
-{
+{
// For pairs "X:Y".
// Reduce the first three symbols "X", "__HFST_TWOLC_:" and "Y" from
// symbol_queue.
- reduce_symbol_pair(true);
+ reduce_symbol_pair(true);
}
| COLON_SPACE
{
@@ -426,7 +453,7 @@ PAIR: GRAMMAR_SYMBOL COLON_SPACE
// Push a "__HFST_TWOLC_?" onto symbol_queue.
// Reduce the three first symbols "__HFST_TWOLC_?", "__HFST_TWOLC_:" and
// "__HFST_TWOLC_?" from symbol_queue.
- symbol_queue.push_front("__HFST_TWOLC_?");
+ htwolcpre1_symbol_queue.push_front("__HFST_TWOLC_?");
reduce_symbol_pair();
}
| GRAMMAR_SYMBOL_SPACE
@@ -438,18 +465,18 @@ PAIR: GRAMMAR_SYMBOL COLON_SPACE
std::string symbol = get_symbol_queue_front();
// Add the colon and output symbol.
- symbol_queue.push_front("__HFST_TWOLC_:");
- symbol_queue.push_front(symbol);
+ htwolcpre1_symbol_queue.push_front("__HFST_TWOLC_:");
+ htwolcpre1_symbol_queue.push_front(symbol);
reduce_symbol_pair();
}
ALPHABET_PAIR: GRAMMAR_SYMBOL COLON GRAMMAR_SYMBOL_SPACE
-{
+{
// For pairs "X:Y".
// Reduce the first three symbols "X", "__HFST_TWOLC_:" and "Y" from
// symbol_queue.
- reduce_symbol_pair();
+ reduce_symbol_pair();
}
| GRAMMAR_SYMBOL_SPACE
{
@@ -458,8 +485,8 @@ ALPHABET_PAIR: GRAMMAR_SYMBOL COLON GRAMMAR_SYMBOL_SPACE
// Reduce the first three symbols "X", "__HFST_TWOLC_:" and "X" from
// symbol_queue.
std::string symbol = get_symbol_queue_front();
- symbol_queue.push_back("__HFST_TWOLC_:");
- symbol_queue.push_back(symbol);
+ htwolcpre1_symbol_queue.push_back("__HFST_TWOLC_:");
+ htwolcpre1_symbol_queue.push_back(symbol);
reduce_symbol_pair();
}
@@ -480,16 +507,16 @@ SEMI_COLON_LIST: SEMI_COLON
%%
-// Print warning.
-void warn(const char * warning)
-{ input_reader.warn(warning); }
+// Print warning.
+void htwolcpre1warn(const char * warning)
+{ htwolcpre1_input_reader.warn(warning); }
-// Print error messge and exit 1.
-void yyerror(const char * text)
-{
- input_reader.error(text);
- std::cout << "__HFST_TWOLC_DIE";
- exit(1);
+// Print error message and throw exception.
+void htwolcpre1error(const char * text)
+{
+ htwolcpre1_input_reader.error(text);
+ *output << "__HFST_TWOLC_DIE";
+ throw HfstException();
}
// Set the variable of this variable initialization and set its values.
@@ -506,7 +533,7 @@ void set_variable_values(void)
else
{ rule_variables.add_value(*it); }
}
- variable_vector.clear();
+ variable_vector.clear();
}
// Pop the queue three times: once for the input symbol, once for the pair
@@ -518,11 +545,11 @@ void reduce_symbol_pair(bool no_definitions)
if (definitions.has_element(get_symbol_queue_front()))
{
std::string def = get_symbol_queue_front();
- std::string error =
- "Definition name " + def + " can't be used in pair expressions " +
+ std::string error =
+ "Definition name " + def + " can't be used in pair expressions " +
def + ":, :" + def + " and " + def + ":" + def + ".";
- yyerror(error.c_str());
+ htwolcpre1error(error.c_str());
}
}
@@ -534,11 +561,11 @@ void reduce_symbol_pair(bool no_definitions)
if (definitions.has_element(get_symbol_queue_front()))
{
std::string def = get_symbol_queue_front();
- std::string error =
- "Definition name " + def + " can't be used in pair expressions " +
+ std::string error =
+ "Definition name " + def + " can't be used in pair expressions " +
def + ":, :" + def + " and " + def + ":" + def + ".";
- yyerror(error.c_str());
+ htwolcpre1error(error.c_str());
}
}
@@ -549,45 +576,45 @@ void reduce_symbol_pair(bool no_definitions)
// encountered.
void increase_line_counter(void)
{
- while (! symbol_queue.empty() &&
- symbol_queue.front() == "__HFST_TWOLC_\\n")
+ while (! htwolcpre1_symbol_queue.empty() &&
+ htwolcpre1_symbol_queue.front() == "__HFST_TWOLC_\\n")
{
- ++line_number;
- symbol_queue.pop_front();
+ ++htwolcpre1_line_number;
+ htwolcpre1_symbol_queue.pop_front();
}
}
// First pop all "__HFST_TWOLC_\\n" in symbol_queue, while incrementing
-// line_counter. Then pop the first element != "__HFST_TWOLC_\\n" in
+// line_counter. Then pop the first element != "__HFST_TWOLC_\\n" in
// symbol_queue.
void pop_symbol_queue(void)
{
increase_line_counter();
- symbol_queue.pop_front();
+ htwolcpre1_symbol_queue.pop_front();
}
// First pop all "__HFST_TWOLC_\\n" in symbol_queue, while incrementing
-// line_counter. Then return a reference to the first element
+// line_counter. Then return a reference to the first element
// != "__HFST_TWOLC_\\n" in symbol_queue.
std::string &get_symbol_queue_front(void)
{
increase_line_counter();
- return symbol_queue.front();
+ return htwolcpre1_symbol_queue.front();
}
// Decide what to do with the next symbol in symbol_queue.
void reduce_queue(bool variable_symbol)
-{
+{
increase_line_counter();
// Test whether the next symbol is a set name and label it a set if it is.
if (sets.has_element(get_symbol_queue_front()))
- { get_symbol_queue_front() =
+ { get_symbol_queue_front() =
std::string("__HFST_TWOLC_SET_NAME=") + get_symbol_queue_front(); }
// Test whether the next symbol is a definition name and label it a
- // definition if it is.
+ // definition if it is.
if (definitions.has_element(get_symbol_queue_front()))
- { get_symbol_queue_front() =
+ { get_symbol_queue_front() =
std::string("__HFST_TWOLC_DEFINITION_NAME=") + get_symbol_queue_front();}
// Unescape the escaped characters in the symbol.
@@ -598,74 +625,26 @@ void reduce_queue(bool variable_symbol)
// 1. We display symbols, which aren't part of a rule.
// 2. We push back rule symbols into rule_symbol_vector, which stores
// string representations of rules.
- // 3. We push back variable symbols into rule_symbol_vector, which stores
+ // 3. We push back variable symbols into rule_symbol_vector, which stores
// rule-variable names and values.
if (! variable_symbol)
{
- if (! rules_start)
- {
- std::cout << get_symbol_queue_front() << " ";
+ if (! htwolcpre1_rules_start)
+ {
+ *output << get_symbol_queue_front() << " ";
pop_symbol_queue();
}
else
- {
+ {
rule_symbol_vector.push_back
(StringVector(get_symbol_queue_front()));
pop_symbol_queue();
}
}
else
- {
- variable_vector.push_back(get_symbol_queue_front());
- pop_symbol_queue();
- }
-}
-
-int main(int argc, char * argv[])
-{
-#ifdef WINDOWS
- _setmode(1, _O_BINARY);
-#endif
-
- CommandLine command_line(argc,argv);
-
- if (command_line.help || command_line.version)
- {
- if (command_line.version)
- { command_line.print_version(); }
- if (command_line.help)
- { command_line.print_help(); }
- exit(0);
- }
- if (command_line.usage)
{
- command_line.print_usage();
- exit(0);
- }
- if (! command_line.be_quiet)
- {
- if (! command_line.has_input_file)
- { std::cerr << "Reading input from STDIN." << std::endl; }
- else
- { std::cerr << "Reading input from " << command_line.input_file_name
- << "." << std::endl; }
- if (! command_line.has_output_file)
- { std::cerr << "Writing output to STDOUT." << std::endl; }
- else
- { std::cerr << "Writing output to " << command_line.output_file_name
- << "." << std::endl; }
+ variable_vector.push_back(get_symbol_queue_front());
+ pop_symbol_queue();
}
- if (command_line.be_verbose)
- { std::cerr << "Verbose mode." << std::endl; }
-
- input_reader.set_input(command_line.set_input_file());
-
- // Test that the output file is okay.
- (void)command_line.set_output_file();
-
- //yydebug = 1;
-
- return yyparse();
}
-
diff --git a/libhfst/src/parsers/htwolcpre2-lexer.ll b/libhfst/src/parsers/htwolcpre2-lexer.ll
new file mode 100644
index 0000000..d242be3
--- /dev/null
+++ b/libhfst/src/parsers/htwolcpre2-lexer.ll
@@ -0,0 +1,313 @@
+%option 8Bit batch yylineno noyywrap nounput prefix="htwolcpre2"
+
+%{
+ // This program is free software: you can redistribute it and/or modify
+ // it under the terms of the GNU General Public License as published by
+ // the Free Software Foundation, version 3 of the Licence.
+ //
+ // This program is distributed in the hope that it will be useful,
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ // GNU General Public License for more details.
+ //
+ // You should have received a copy of the GNU General Public License
+ // along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+ // Autotools stuff
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+
+ // The parser. Supplies flex with its symbol tables.
+#ifdef YACC_USE_PARSER_H_EXTENSION
+ #include "htwolcpre2-parser.h"
+#else
+ #include "htwolcpre2-parser.hh"
+#endif
+#include "../HfstExceptionDefs.h"
+
+extern int htwolcpre2error(const char*);
+
+#undef YY_FATAL_ERROR
+#define YY_FATAL_ERROR(msg) htwolcpre2error(msg);
+
+ // input_defs.h declares the inputHandler, which is
+ // an object that overrides flex' default input reading.
+ //#include "io_src/input_defs.h"
+#include "io_src/InputReader.h"
+extern InputReader htwolcpre2_input_reader;
+#undef YY_INPUT
+#define YY_INPUT( buf, result, max_size ) { \
+char c = htwolcpre2_input_reader.input(); \
+if (c == 0) { \
+result = YY_NULL; \
+} \
+ else { \
+buf[0] = c; \
+result = 1; \
+} \
+}
+
+ // non_alphabet_symbol_queue is used to store the grammar symbols which are
+ // not located in the Alphabet section of the grammar.
+ #include "HfstTwolcDefs.h"
+ extern HandyDeque<std::string> htwolcpre2_non_alphabet_symbol_queue;
+
+ // alphabet_symbol_queue is used to store the symbols in the Alphabet section
+ // of the grammar.
+ extern HandyDeque<std::string> htwolcpre2_alphabet_symbol_queue;
+
+ // Tells whether the Alphabet section ended. It ends when the first
+ // __HFST_TWOLC_; is seen.
+ bool alphabet_ended = false;
+%}
+
+%%
+
+__HFST_TWOLC_Alphabet {
+ // Alphabet declaration.
+ htwolcpre2_alphabet_symbol_queue.push_back("__HFST_TWOLC_Alphabet");
+ return ALPHABET_DECLARATION;
+}
+
+__HFST_TWOLC_Diacritics {
+ // Diacritics declaration.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_Diacritics");
+ return DIACRITICS_DECLARATION;
+}
+__HFST_TWOLC_Definitions {
+ // Definitions declaration.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_Definitions");
+ return DEFINITION_DECLARATION;
+}
+__HFST_TWOLC_Sets {
+ // Sets declaration.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_Sets");
+ return SETS_DECLARATION;
+}
+__HFST_TWOLC_Rules {
+ // Rules declaration.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_Rules");
+ return RULES_DECLARATION;
+}
+__HFST_TWOLC_except {
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_except");
+ return EXCEPT;
+}
+
+__HFST_TWOLC_KILL_SYMBOL {
+ // Signifies a syntax error in the first compilation phase.
+ // Just die quietly, since syntax error msgs have been issued by
+ // the first compilation phase.
+ throw HfstException();
+}
+[ ] { /* space: ignore */ }
+__HFST_TWOLC_RULE_NAME=\"[^\"]+\" {
+ // Rule name.
+ htwolcpre2_non_alphabet_symbol_queue.push_back(htwolcpre2text);
+ return RULE_NAME;
+}
+__HFST_TWOLC_[*] {
+ // Kleene star operator.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_*");
+ return STAR;
+}
+__HFST_TWOLC_[+] {
+ // Kleene plus operator.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_+");
+ return PLUS;
+}
+__HFST_TWOLC_[/] {
+ // Freely insert operator.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_/");
+ return FREELY_INSERT;
+}
+__HFST_TWOLC_[~] {
+ // Complement operator.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_~");
+ return COMPLEMENT;
+}
+__HFST_TWOLC_[\\] {
+ // Term complement operator.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_\\");
+ return TERM_COMPLEMENT;
+}
+__HFST_TWOLC_[$][.] {
+ // Containment once operator.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_$.");
+ return CONTAINMENT_ONCE;
+}
+__HFST_TWOLC_[$] {
+ // Containment at least once operator.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_$");
+ return CONTAINMENT;
+}
+__HFST_TWOLC_[?] {
+ // Any symbol.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_?");
+ return QUESTION_MARK;
+}
+__HFST_TWOLC_[0] {
+ if (alphabet_ended)
+ { htwolcpre2_non_alphabet_symbol_queue.push_back(htwolcpre2text); }
+ else
+ { htwolcpre2_alphabet_symbol_queue.push_back(htwolcpre2text); }
+ return SYMBOL;
+}
+__HFST_TWOLC_[|] {
+ // Or.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_|");
+ return UNION;
+}
+__HFST_TWOLC_[&] {
+ // And.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_&");
+ return INTERSECTION;
+}
+__HFST_TWOLC_[\^] {
+ // Power operator.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_^");
+ return POWER;
+}
+__HFST_TWOLC_[\-] {
+ // Difference operator.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_-");
+ return DIFFERENCE;
+}
+__HFST_TWOLC_NUMBER=[0-9]+,[0-9]+ {
+ // Number.
+ htwolcpre2_non_alphabet_symbol_queue.push_back(htwolcpre2text);
+ return NUMBER;
+}
+__HFST_TWOLC_NUMBER=[0-9]+ {
+ // Number.
+ htwolcpre2_non_alphabet_symbol_queue.push_back(htwolcpre2text);
+ return NUMBER;
+}
+__HFST_TWOLC_[.][#][.] {
+ // Word boundary.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_.#.");
+ return SYMBOL;
+}
+__HFST_TWOLC_\[ {
+ // Beginning of a bracketed regex.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_[");
+ return LEFT_SQUARE_BRACKET;
+}
+__HFST_TWOLC_\] {
+ // End of a bracketed regex.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_]");
+ return RIGHT_SQUARE_BRACKET;
+}
+__HFST_TWOLC_\[\[ {
+ // Beginning of a bracketed regex.
+ //
+ // For some bizarre reason [[ can't occur in the string-literal here...
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_[" "[");
+ return RE_LEFT_SQUARE_BRACKET;
+}
+__HFST_TWOLC_\]\] {
+ // End of a bracketed regex.
+ //
+ // For some bizarre reason ]] can't occur in the string-literal here...
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_]" "]");
+ return RE_RIGHT_SQUARE_BRACKET;
+}
+__HFST_TWOLC_\( {
+ // Beginning of an optional bracketed regex.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_(");
+ return LEFT_PARENTHESIS;
+}
+__HFST_TWOLC_\) {
+ // End of an optional bracketed regex.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_)");
+ return RIGHT_PARENTHESIS;
+}
+__HFST_TWOLC_[/][<][=] {
+ // Restriction rule operator.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_/<=");
+ return LEFT_RESTRICTION_ARROW;
+}
+__HFST_TWOLC_[<][=] {
+ // Left rule operator.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_<=");
+ return LEFT_ARROW;
+}
+__HFST_TWOLC_[=][>] {
+ // Right rule operator.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_=>");
+ return RIGHT_ARROW;
+}
+__HFST_TWOLC_[<][=][>] {
+ // Equivalence rule operator.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_<=>");
+ return LEFT_RIGHT_ARROW;
+}
+__HFST_TWOLC_[/][<][=][=] {
+ // Restriction rule operator.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_/<==");
+ return RE_LEFT_RESTRICTION_ARROW;
+}
+__HFST_TWOLC_[<][=][=] {
+ // Left rule operator.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_<==");
+ return RE_LEFT_ARROW;
+}
+__HFST_TWOLC_[=][=][>] {
+ // Right rule operator.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_==>");
+ return RE_RIGHT_ARROW;
+}
+__HFST_TWOLC_[<][=][=][>] {
+ // Equivalence rule operator.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_<==>");
+ return RE_LEFT_RIGHT_ARROW;
+}
+__HFST_TWOLC_[:] {
+ if (alphabet_ended)
+ { htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_:"); }
+ else
+ { htwolcpre2_alphabet_symbol_queue.push_back("__HFST_TWOLC_:"); }
+ // Pair separator in expressions like "[a:]".
+ return PAIR_SEPARATOR;
+}
+__HFST_TWOLC_[;] {
+ // End-of-line symbol.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_;");
+ // The first __HFST_TWOLC_; ends the alphabet.
+ alphabet_ended = true;
+ return SEMI_COLON;
+}
+__HFST_TWOLC_[=] {
+ // Equals sign used when defining sets and making definitions.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC_=");
+ return EQUALS;
+}
+__HFST_TWOLC__ {
+ // Center-marker in a rule context.
+ htwolcpre2_non_alphabet_symbol_queue.push_back("__HFST_TWOLC__");
+ return CENTER_MARKER;
+}
+__HFST_TWOLC_DIE {
+ // If this symbol is seen, pass it on and exit quietly.
+ //std::cout << "__HFST_TWOLC_DIE";
+ throw HfstException();
+}
+__HFST_TWOLC_SET_NAME=[^ ]+ {
+ htwolcpre2_non_alphabet_symbol_queue.push_back(htwolcpre2text);
+ return SET_NAME;
+}
+__HFST_TWOLC_DEFINITION_NAME=[^ ]+ {
+ htwolcpre2_non_alphabet_symbol_queue.push_back(htwolcpre2text);
+ return DEFINITION_NAME;
+}
+[^ ]+ {
+ if (alphabet_ended)
+ { htwolcpre2_non_alphabet_symbol_queue.push_back(htwolcpre2text); }
+ else
+ { htwolcpre2_alphabet_symbol_queue.push_back(htwolcpre2text); }
+ return SYMBOL;
+}
+
+%%
diff --git a/tools/src/hfst-twolc/src/htwolcpre2.yy b/libhfst/src/parsers/htwolcpre2-parser.yy
similarity index 69%
rename from tools/src/hfst-twolc/src/htwolcpre2.yy
rename to libhfst/src/parsers/htwolcpre2-parser.yy
index 11201ac..7710f6f 100644
--- a/tools/src/hfst-twolc/src/htwolcpre2.yy
+++ b/libhfst/src/parsers/htwolcpre2-parser.yy
@@ -24,51 +24,54 @@
#include <fstream>
#include <cstdlib>
#include "io_src/InputReader.h"
-#include "commandline_src/CommandLine.h"
-#include "grammar_defs.h"
#include "HfstTwolcDefs.h"
-#include "common_globals.h"
-
- extern int yylineno;
- extern char * yytext;
- extern int yylineno;
- extern char * yytext;
- extern bool rules_start;
- void yyerror(const char * text );
- void semantic_error(const char * text);
- void warn(const char * warning );
- int yylex();
- int yyparse();
- void complete_alphabet(void);
+#include "grammar_defs.h"
+#include "../HfstExceptionDefs.h"
+
+ extern int htwolcpre2lineno;
+ extern char * htwolcpre2text;
+ extern int htwolcpre2lineno;
+ extern char * htwolcpre2text;
+ void htwolcpre2error(const char * text );
+ int htwolcpre2lex();
+ int htwolcpre2parse();
#define YYERROR_VERBOSE 1
- size_t line_number = 1;
+ size_t htwolcpre2_line_number = 1;
// For reading input one byte at a time.
- InputReader input_reader(line_number);
-
- // non_alphabet_symbol_queue is used to store the grammar symbols which are
- // not located in the Alphabet section of the grammar.
- HandyDeque<std::string> non_alphabet_symbol_queue;
-
- // alphabet_symbol_queue is used to store the symbols in the Alphabet section
- // of the grammar.
- HandyDeque<std::string> alphabet_symbol_queue;
+ InputReader htwolcpre2_input_reader(htwolcpre2_line_number);
+
+namespace hfst { namespace twolcpre2 {
+ void set_input(std::istream & istr)
+ {
+ htwolcpre2_input_reader.set_input(istr);
+ }
+ int parse()
+ {
+ return htwolcpre2parse();
+ }
+ void set_warning_stream(std::ostream & ostr)
+ {
+ htwolcpre2_input_reader.set_warning_stream(ostr);
+ }
+ void set_error_stream(std::ostream & ostr)
+ {
+ htwolcpre2_input_reader.set_error_stream(ostr);
+ }
+
+}}
- // alphabet_symbol_queue is used to store the symbols in the Alphabet section
- // of the grammar after it has been completed with all symbol pairs in the
- // grammar.
- HandyDeque<std::string> total_alphabet_symbol_queue;
%}
+%name-prefix "htwolcpre2"
-
-%union
+%union
{ int symbol_number; };
- /*
+ /*
All unary operators have stronger precedence than binary ones.
*/
@@ -80,19 +83,19 @@
/* Unary operators ordered by precedence from lowest to highest. */
%right <symbol_number> STAR PLUS
-%left <symbol_number> CONTAINMENT CONTAINMENT_ONCE TERM_COMPLEMENT COMPLEMENT
+%left <symbol_number> CONTAINMENT CONTAINMENT_ONCE TERM_COMPLEMENT COMPLEMENT
%right <symbol_number> POWER
/* "[", "]", "(" and ")". */
-%right <symbol_number> RIGHT_SQUARE_BRACKET RIGHT_PARENTHESIS
+%right <symbol_number> RIGHT_SQUARE_BRACKET RIGHT_PARENTHESIS
%left <symbol_number> LEFT_SQUARE_BRACKET LEFT_PARENTHESIS
/* Twolc rule operators */
-%token <symbol_number> LEFT_RESTRICTION_ARROW LEFT_ARROW RIGHT_ARROW
+%token <symbol_number> LEFT_RESTRICTION_ARROW LEFT_ARROW RIGHT_ARROW
%token <symbol_number> LEFT_RIGHT_ARROW
/* Twolc regular expression rule operators */
-%token <symbol_number> RE_LEFT_RESTRICTION_ARROW RE_LEFT_ARROW RE_RIGHT_ARROW
+%token <symbol_number> RE_LEFT_RESTRICTION_ARROW RE_LEFT_ARROW RE_RIGHT_ARROW
%token <symbol_number> RE_LEFT_RIGHT_ARROW
/* Twolc regular expression rule center brackets. */
@@ -100,7 +103,7 @@
%left <symbol_number> RE_LEFT_SQUARE_BRACKET
/* Basic tokens. */
-%token <symbol_number> ALPHABET_DECLARATION DIACRITICS_DECLARATION
+%token <symbol_number> ALPHABET_DECLARATION DIACRITICS_DECLARATION
%token <symbol_number> SETS_DECLARATION DEFINITION_DECLARATION
%token <symbol_number> RULES_DECLARATION PAIR_SEPARATOR SYMBOL SEMI_COLON
%token <symbol_number> SET_NAME DEFINITION_NAME EQUALS CENTER_MARKER
@@ -110,13 +113,13 @@
ALL: GRAMMAR {}
;
-GRAMMAR: ALPHABET GRAMMAR1
+GRAMMAR: ALPHABET GRAMMAR1
| GRAMMAR1
-GRAMMAR1: DIACRITICS GRAMMAR2
+GRAMMAR1: DIACRITICS GRAMMAR2
| GRAMMAR2
-GRAMMAR2: SETS GRAMMAR3
+GRAMMAR2: SETS GRAMMAR3
| GRAMMAR3
GRAMMAR3: DEFINITIONS GRAMMAR4
@@ -129,9 +132,9 @@ RULES:RULES_DECLARATION RULE_LIST
RULE_LIST: /* empty */
| RULE_LIST RULE
-RULE: RULE_NAME_DECL RULE_CENTER RULE_OPERATOR RULE_CONTEXTS
+RULE: RULE_NAME_DECL RULE_CENTER RULE_OPERATOR RULE_CONTEXTS
NEGATIVE_RULE_CONTEXTS
-| RULE_NAME_DECL RE_RULE_CENTER RE_RULE_OPERATOR RULE_CONTEXTS
+| RULE_NAME_DECL RE_RULE_CENTER RE_RULE_OPERATOR RULE_CONTEXTS
NEGATIVE_RULE_CONTEXTS
RULE_NAME_DECL: RULE_NAME
@@ -164,7 +167,7 @@ RE_RULE_OPERATOR: RE_LEFT_ARROW
RULE_CONTEXTS: /* empty */
| RULE_CONTEXTS RULE_CONTEXT
-NEGATIVE_RULE_CONTEXTS: /* empty */
+NEGATIVE_RULE_CONTEXTS: /* empty */
| EXCEPT RULE_CONTEXTS
RULE_CONTEXT: REGULAR_EXPRESSION CENTER_MARKER REGULAR_EXPRESSION
@@ -174,7 +177,7 @@ ALPHABET: ALPHABET_DECLARATION ALPHABET_PAIR_LIST SEMI_COLON_LIST
DIACRITICS: DIACRITICS_DECLARATION SYMBOL_LIST SEMI_COLON_LIST
-SETS: SETS_DECLARATION SET_LIST
+SETS: SETS_DECLARATION SET_LIST
DEFINITIONS: DEFINITION_DECLARATION DEFINITION_LIST
@@ -203,8 +206,8 @@ RE: PAIR
| LEFT_SQUARE_BRACKET REGULAR_EXPRESSION RIGHT_SQUARE_BRACKET
| LEFT_PARENTHESIS REGULAR_EXPRESSION RIGHT_PARENTHESIS
-SET_LIST: /* empty */
-| SET_LIST SET_DEFINITION
+SET_LIST: /* empty */
+| SET_LIST SET_DEFINITION
SYMBOL_LIST: /* empty */
| SYMBOL_LIST SYMBOL
@@ -229,6 +232,47 @@ SEMI_COLON_LIST: SEMI_COLON
%%
+// Print warning.
+void htwolcpre2warn(const char * warning)
+{ htwolcpre2_input_reader.warn(warning); }
+
+// Print error messge and throw an exception.
+void htwolcpre2error(const char * text)
+{
+ //std::cerr << text << std::endl;
+ htwolcpre2_input_reader.error(text);
+ (void)text;
+ throw HfstException();
+}
+
+void htwolcpre2_semantic_error(const char * text)
+{ htwolcpre2_input_reader.error(text); }
+
+// non_alphabet_symbol_queue is used to store the grammar symbols which are
+// not located in the Alphabet section of the grammar.
+HandyDeque<std::string> htwolcpre2_non_alphabet_symbol_queue;
+
+// alphabet_symbol_queue is used to store the symbols in the Alphabet section
+// of the grammar.
+HandyDeque<std::string> htwolcpre2_alphabet_symbol_queue;
+
+// alphabet_symbol_queue is used to store the symbols in the Alphabet section
+// of the grammar after it has been completed with all symbol pairs in the
+// grammar.
+HandyDeque<std::string> total_alphabet_symbol_queue;
+
+namespace hfst { namespace twolcpre2 {
+const HandyDeque<std::string> & get_total_alphabet_symbol_queue()
+{
+ return total_alphabet_symbol_queue;
+}
+
+const HandyDeque<std::string> & get_non_alphabet_symbol_queue()
+{
+ return htwolcpre2_non_alphabet_symbol_queue;
+}
+}}
+
void insert_alphabet_pairs(const HandyDeque<std::string> &symbol_queue,
HandySet<SymbolPair> &symbol_pair_set)
{
@@ -242,9 +286,9 @@ void insert_alphabet_pairs(const HandyDeque<std::string> &symbol_queue,
*it == "__HFST_TWOLC_#" ||
*it == "__HFST_TWOLC_SPACE" ||
*it == "__HFST_TWOLC_TAB" ||
- it->find("__HFST_TWOLC_") == std::string::npos)
+ it->find("__HFST_TWOLC_") == std::string::npos)
&&
- *(it+1) == "__HFST_TWOLC_:"
+ *(it+1) == "__HFST_TWOLC_:"
&&
(*(it+2) == "__HFST_TWOLC_0" ||
*(it+2) == "__HFST_TWOLC_.#." ||
@@ -252,7 +296,7 @@ void insert_alphabet_pairs(const HandyDeque<std::string> &symbol_queue,
*(it+2) == "__HFST_TWOLC_SPACE" ||
*(it+2) == "__HFST_TWOLC_TAB" ||
(it+2)->find("__HFST_TWOLC_") == std::string::npos))
- {
+ {
std::string input_symbol = *it == "__HFST_TWOLC_#" ? "#" : *it;
++(++it);
std::string output_symbol = *it == "__HFST_TWOLC_#" ? "#" : *it;
@@ -263,12 +307,13 @@ void insert_alphabet_pairs(const HandyDeque<std::string> &symbol_queue,
}
// Add all pairs in the grammar, which are missing from the Alphabet section,
-// into the Alphabet section.
+// into the Alphabet section.
+namespace hfst { namespace twolcpre2 {
void complete_alphabet(void)
{
HandySet<SymbolPair> symbol_pair_set;
- insert_alphabet_pairs(alphabet_symbol_queue,symbol_pair_set);
- insert_alphabet_pairs(non_alphabet_symbol_queue,symbol_pair_set);
+ insert_alphabet_pairs(htwolcpre2_alphabet_symbol_queue,symbol_pair_set);
+ insert_alphabet_pairs(htwolcpre2_non_alphabet_symbol_queue,symbol_pair_set);
total_alphabet_symbol_queue.push_back("__HFST_TWOLC_Alphabet");
for(HandySet<SymbolPair>::const_iterator it = symbol_pair_set.begin();
@@ -280,39 +325,4 @@ void complete_alphabet(void)
total_alphabet_symbol_queue.push_back(it->second);
}
}
-
-// Print warning.
-void warn(const char * warning)
-{ input_reader.warn(warning); }
-
-// Print error messge and exit 1.
-void yyerror(const char * text)
-{
- std::cerr << text << std::endl;
- (void)text;
- exit(1);
-}
-
-void semantic_error(const char * text)
-{ input_reader.error(text); }
-
-int main(int argc, char * argv[])
-{
-#ifdef WINDOWS
- _setmode(0, _O_BINARY);
- _setmode(1, _O_BINARY);
-#endif
-
- CommandLine command_line(argc,argv);
- if (command_line.help || command_line.usage || command_line.version)
- { exit(0); }
- //yydebug = 1;
- input_reader.set_input(std::cin);
- int exit_code = yyparse();
- complete_alphabet();
- std::cout << total_alphabet_symbol_queue << " ";
- std::cout << non_alphabet_symbol_queue;
- return exit_code;
-}
-
-
+}}
diff --git a/tools/src/hfst-twolc/src/scanner3.ll b/libhfst/src/parsers/htwolcpre3-lexer.ll
similarity index 71%
rename from tools/src/hfst-twolc/src/scanner3.ll
rename to libhfst/src/parsers/htwolcpre3-lexer.ll
index cfeeca9..fd8ad41 100644
--- a/tools/src/hfst-twolc/src/scanner3.ll
+++ b/libhfst/src/parsers/htwolcpre3-lexer.ll
@@ -1,4 +1,4 @@
-%option 8Bit batch yylineno noyywrap nounput
+%option 8Bit batch yylineno noyywrap nounput prefix="htwolcpre3"
%{
// This program is free software: you can redistribute it and/or modify
@@ -20,16 +20,35 @@
// input_defs.h declares the inputHandler, which is
// an object that overrides flex' default input reading.
-#include "io_src/input_defs.h"
+//#include "io_src/input_defs.h"
+
+extern int htwolcpre3error(const char*);
+
+#undef YY_FATAL_ERROR
+#define YY_FATAL_ERROR(msg) htwolcpre3error(msg);
+
+#include "io_src/InputReader.h"
+extern InputReader htwolcpre3_input_reader;
+#undef YY_INPUT
+#define YY_INPUT( buf, result, max_size ) { \
+char c = htwolcpre3_input_reader.input(); \
+if (c == 0) { \
+result = YY_NULL; \
+} \
+ else { \
+buf[0] = c; \
+result = 1; \
+} \
+}
#include "string_src/string_manipulation.h"
#include "rule_src/TwolCGrammar.h"
#ifdef YACC_USE_PARSER_H_EXTENSION
- #include "htwolcpre3.h"
+ #include "htwolcpre3-parser.h"
#else
- #include "htwolcpre3.hh"
+ #include "htwolcpre3-parser.hh"
#endif
/*
@@ -46,19 +65,19 @@
%%
__HFST_TWOLC_Alphabet { return ALPHABET_DECLARATION; }
-__HFST_TWOLC_Diacritics { return DIACRITICS_DECLARATION; }
+__HFST_TWOLC_Diacritics { return DIACRITICS_DECLARATION; }
__HFST_TWOLC_Definitions { return DEFINITION_DECLARATION; }
__HFST_TWOLC_Sets { return SETS_DECLARATION; }
__HFST_TWOLC_Rules { return RULES_DECLARATION; }
__HFST_TWOLC_DIE {
// If this symbol is seen, exit quietly.
- exit(1);
+ throw HfstException();
}
[ \t\n] { /* space: ignore */ }
-__HFST_TWOLC_RULE_NAME=\"[^\"]+\" {
- yylval.value = string_copy(yytext);
- return RULE_NAME;
+__HFST_TWOLC_RULE_NAME=\"[^\"]+\" {
+ htwolcpre3lval.value = string_copy(htwolcpre3text);
+ return RULE_NAME;
}
__HFST_TWOLC_except { return EXCEPT; }
__HFST_TWOLC_[*] { return STAR; }
@@ -69,21 +88,21 @@ __HFST_TWOLC_[\\] { return TERM_COMPLEMENT; }
__HFST_TWOLC_[$][.] { return CONTAINMENT_ONCE; }
__HFST_TWOLC_[$] { return CONTAINMENT; }
__HFST_TWOLC_[?] { return QUESTION_MARK; }
-__HFST_TWOLC_[0] {
- yylval.value = string_copy(yytext);
+__HFST_TWOLC_[0] {
+ htwolcpre3lval.value = string_copy(htwolcpre3text);
return SYMBOL;
}
__HFST_TWOLC_[|] { return UNION; }
__HFST_TWOLC_[&] { return INTERSECTION; }
__HFST_TWOLC_[\^] { return POWER; }
__HFST_TWOLC_[\-] { return DIFFERENCE; }
-__HFST_TWOLC_NUMBER=[0-9]+ {
- yylval.value = string_copy(yytext);
- return NUMBER;
+__HFST_TWOLC_NUMBER=[0-9]+ {
+ htwolcpre3lval.value = string_copy(htwolcpre3text);
+ return NUMBER;
}
-__HFST_TWOLC_NUMBER=[0-9]+,[0-9]+ {
- yylval.value = string_copy(yytext);
- return NUMBER_RANGE;
+__HFST_TWOLC_NUMBER=[0-9]+,[0-9]+ {
+ htwolcpre3lval.value = string_copy(htwolcpre3text);
+ return NUMBER_RANGE;
}
__HFST_TWOLC_\[ { return LEFT_SQUARE_BRACKET; }
__HFST_TWOLC_\] { return RIGHT_SQUARE_BRACKET; }
@@ -105,15 +124,15 @@ __HFST_TWOLC_[=] { return EQUALS; }
__HFST_TWOLC__ { return CENTER_MARKER; }
__HFST_TWOLC_SET_NAME=[^ ]+ {
- yylval.value = string_copy(yytext);
+ htwolcpre3lval.value = string_copy(htwolcpre3text);
return SYMBOL;
}
__HFST_TWOLC_DEFINITION_NAME=[^ ]+ {
- yylval.value = string_copy(yytext);
+ htwolcpre3lval.value = string_copy(htwolcpre3text);
return DEFINITION_NAME;
}
[^ ]+ {
- yylval.value = string_copy(yytext);
+ htwolcpre3lval.value = string_copy(htwolcpre3text);
return SYMBOL;
}
diff --git a/tools/src/hfst-twolc/src/htwolcpre3.yy b/libhfst/src/parsers/htwolcpre3-parser.yy
similarity index 76%
rename from tools/src/hfst-twolc/src/htwolcpre3.yy
rename to libhfst/src/parsers/htwolcpre3-parser.yy
index 1b0d437..24413cd 100644
--- a/tools/src/hfst-twolc/src/htwolcpre3.yy
+++ b/libhfst/src/parsers/htwolcpre3-parser.yy
@@ -29,7 +29,6 @@
#include "rule_src/OtherSymbolTransducer.h"
#include "alphabet_src/Alphabet.h"
#include "string_src/string_manipulation.h"
-#include "commandline_src/CommandLine.h"
#include <cstdio>
#include "common_globals.h"
@@ -37,26 +36,61 @@
#define YYDEBUG 1
#endif
- extern int yylineno;
- extern char * yytext;
- extern int yylineno;
- extern char * yytext;
- extern bool rules_start;
- void yyerror(const char * text );
- void semantic_error(const char * text);
- void warn(const char * warning );
- void message(const std::string &m);
- int yylex();
- int yyparse();
+ extern int htwolcpre3lineno;
+ extern char * htwolcpre3text;
+ extern int htwolcpre3lineno;
+ extern char * htwolcpre3text;
+ void htwolcpre3error(const char * text );
+ void htwolcpre3_semantic_error(const char * text);
+ void htwolcpre3warn(const char * warning );
+ void htwolcpre3message(const std::string &m);
+ int htwolcpre3lex();
+ int htwolcpre3parse();
- bool silent = false;
- bool verbose = false;
+ bool silent_ = false;
+ bool verbose_ = false;
+
+namespace hfst {
+ namespace twolcpre3 {
+ void set_silent(bool val)
+ {
+ silent_ = val;
+ }
+ void set_verbose(bool val)
+ {
+ verbose_ = val;
+ }
+ int parse()
+ {
+ return htwolcpre3parse();
+ }
+ void message(const std::string &m)
+ {
+ return htwolcpre3message(m);
+ }
+}}
#define YYERROR_VERBOSE 1
// For reading input one byte at a time.
- size_t line_number = 1;
- InputReader input_reader(line_number);
+ size_t htwolcpre3_line_number = 1;
+ InputReader htwolcpre3_input_reader(htwolcpre3_line_number);
+
+namespace hfst { namespace twolcpre3 {
+ void set_input(std::istream & istr)
+ {
+ htwolcpre3_input_reader.set_input(istr);
+ }
+ void set_warning_stream(std::ostream & ostr)
+ {
+ htwolcpre3_input_reader.set_warning_stream(ostr);
+ }
+ void set_error_stream(std::ostream & ostr)
+ {
+ htwolcpre3_input_reader.set_error_stream(ostr);
+ }
+
+}}
#ifdef HAVE_XFSM
#define Alphabet TwolCAlphabet
@@ -68,31 +102,43 @@
// A map storing named regular expressions (i.e. definitions).
HandyMap<std::string,OtherSymbolTransducer> definition_map;
- // The grammar, which compiles rules, resolves conflicts and stores
+ // The grammar, which compiles rules, resolves conflicts and stores
// rule transducers.
// TwolCGrammar grammar(true,true);
TwolCGrammar * grammar;
+namespace hfst {
+ namespace twolcpre3 {
+ void set_grammar(TwolCGrammar * grammar_)
+ {
+ grammar = grammar_;
+ }
+ TwolCGrammar * get_grammar()
+ {
+ return grammar;
+ }
+}}
+
unsigned int get_number(const std::string &);
unsigned int get_second_number(const std::string &s);
std::string get_name(const std::string &s);
%}
+%name-prefix "htwolcpre3"
-
-%union
-{
+%union
+{
int symbol_number;
OtherSymbolTransducer * regular_expression;
- char * value;
+ char * value;
SymbolRange * symbol_range;
SymbolPairVector * symbol_pair_range;
op::OPERATOR rule_operator;
};
- /*
+ /*
All unary operators have stronger precedence than binary ones.
*/
@@ -104,19 +150,19 @@
/* Unary operators ordered by precedence from lowest to highest. */
%right <symbol_number> STAR PLUS
-%left <symbol_number> CONTAINMENT CONTAINMENT_ONCE TERM_COMPLEMENT COMPLEMENT
+%left <symbol_number> CONTAINMENT CONTAINMENT_ONCE TERM_COMPLEMENT COMPLEMENT
%right <symbol_number> POWER
/* "[", "]", "(", ")". */
-%right <symbol_number> RIGHT_SQUARE_BRACKET RIGHT_PARENTHESIS
+%right <symbol_number> RIGHT_SQUARE_BRACKET RIGHT_PARENTHESIS
%left <symbol_number> LEFT_SQUARE_BRACKET LEFT_PARENTHESIS
/* Twolc rule operators */
-%token <symbol_number> LEFT_RESTRICTION_ARROW LEFT_ARROW RIGHT_ARROW
+%token <symbol_number> LEFT_RESTRICTION_ARROW LEFT_ARROW RIGHT_ARROW
%token <symbol_number> LEFT_RIGHT_ARROW
/* Twolc regular expression rule operators */
-%token <symbol_number> RE_LEFT_RESTRICTION_ARROW RE_LEFT_ARROW RE_RIGHT_ARROW
+%token <symbol_number> RE_LEFT_RESTRICTION_ARROW RE_LEFT_ARROW RE_RIGHT_ARROW
%token <symbol_number> RE_LEFT_RIGHT_ARROW
/* Twolc regular expression rule center brackets. */
@@ -124,7 +170,7 @@
%left <symbol_number> RE_LEFT_SQUARE_BRACKET
/* Basic tokens. */
-%token <symbol_number> ALPHABET_DECLARATION DIACRITICS_DECLARATION
+%token <symbol_number> ALPHABET_DECLARATION DIACRITICS_DECLARATION
%token <symbol_number> SETS_DECLARATION DEFINITION_DECLARATION
%token <symbol_number> RULES_DECLARATION COLON SEMI_COLON
%token <symbol_number> EQUALS CENTER_MARKER QUESTION_MARK EXCEPT
@@ -133,7 +179,7 @@
%type<regular_expression> PAIR REGULAR_EXPRESSION RE_LIST RE RULE_CONTEXT
%type<regular_expression> RE_RULE_CENTER
%type<regular_expression> RULE_CONTEXTS NEGATIVE_RULE_CONTEXTS
-%type<symbol_range> SYMBOL_LIST
+%type<symbol_range> SYMBOL_LIST
%type<symbol_pair_range> CENTER_PAIR CENTER_LIST RULE_CENTER
%type<rule_operator> RULE_OPERATOR RE_RULE_OPERATOR
%type<value> CENTER_SYMBOL PAIR_SYMBOL
@@ -143,13 +189,13 @@
ALL: GRAMMAR {}
;
-GRAMMAR: ALPHABET GRAMMAR1
+GRAMMAR: ALPHABET GRAMMAR1
| GRAMMAR1
-GRAMMAR1: DIACRITICS GRAMMAR2
+GRAMMAR1: DIACRITICS GRAMMAR2
| GRAMMAR2
-GRAMMAR2: SETS GRAMMAR3
+GRAMMAR2: SETS GRAMMAR3
| GRAMMAR3
GRAMMAR3: DEFINITIONS GRAMMAR4
@@ -160,18 +206,18 @@ GRAMMAR4: RULES
RULES:RULES_HEADER RULE_LIST
RULES_HEADER:RULES_DECLARATION
-{ message("Reading rules and compiling their contexts and centers."); }
+{ htwolcpre3message("Reading rules and compiling their contexts and centers."); }
RULE_LIST: /* empty */
| RULE_LIST RULE
RULE: RULE_NAME RULE_CENTER RULE_OPERATOR RULE_CONTEXTS NEGATIVE_RULE_CONTEXTS
-{
+{
// Subtract negative contexts from positive contexts.
$4->apply(&HfstTransducer::subtract,*$5);
- if (verbose)
- { message(std::string("Processing: ")+ get_name($1)); }
+ if (verbose_)
+ { htwolcpre3message(std::string("Processing: ")+ get_name($1)); }
if ($2->size() == 1)
{ grammar->add_rule($1,(*$2)[0],$3,OtherSymbolTransducerVector(1,*$4)); }
@@ -182,7 +228,7 @@ RULE: RULE_NAME RULE_CENTER RULE_OPERATOR RULE_CONTEXTS NEGATIVE_RULE_CONTEXTS
delete $4;
delete $5;
}
-| RULE_NAME RE_RULE_CENTER RE_RULE_OPERATOR RULE_CONTEXTS
+| RULE_NAME RE_RULE_CENTER RE_RULE_OPERATOR RULE_CONTEXTS
NEGATIVE_RULE_CONTEXTS
{
// Subtract negative contexts from positive contexts.
@@ -212,15 +258,15 @@ RE_RULE_CENTER: RE_LEFT_SQUARE_BRACKET REGULAR_EXPRESSION RE_RIGHT_SQUARE_BRACKE
CENTER_LIST: CENTER_PAIR
{ $$ = $1; }
| CENTER_LIST UNION CENTER_PAIR
-{
+{
$$ = $1;
$$->push_back(*$3->begin());
delete $3;
}
CENTER_PAIR: CENTER_SYMBOL COLON CENTER_SYMBOL
-{
- $$ = alphabet.get_symbol_pair_vector(SymbolPair($1,$3));
+{
+ $$ = alphabet.get_symbol_pair_vector(SymbolPair($1,$3));
free($1); free($3);
}
@@ -250,7 +296,7 @@ RE_RULE_OPERATOR: RE_LEFT_ARROW
RULE_CONTEXTS: /* empty */
{ $$ = new OtherSymbolTransducer(); }
| RULE_CONTEXTS RULE_CONTEXT
-{
+{
$$ = &$1->apply(&HfstTransducer::disjunct,*$2);
delete $2;
}
@@ -272,27 +318,27 @@ ALPHABET: ALPHABET_HEADER ALPHABET_PAIR_LIST SEMI_COLON_LIST
{ alphabet.alphabet_done(); }
ALPHABET_HEADER:ALPHABET_DECLARATION
-{ message("Reading alphabet."); }
+{ htwolcpre3message("Reading alphabet."); }
DIACRITICS: DIACRITICS_HEADER SYMBOL_LIST SEMI_COLON_LIST
-{
+{
grammar->define_diacritics(*$2);
alphabet.define_diacritics(*$2);
delete $2;
}
DIACRITICS_HEADER:DIACRITICS_DECLARATION
-{ message("Reading diacritics."); }
+{ htwolcpre3message("Reading diacritics."); }
-SETS: SETS_HEADER SET_LIST
+SETS: SETS_HEADER SET_LIST
SETS_HEADER:SETS_DECLARATION
-{ message("Reading sets."); }
+{ htwolcpre3message("Reading sets."); }
DEFINITIONS: DEFINITION_HEADER DEFINITION_LIST
DEFINITION_HEADER:DEFINITION_DECLARATION
-{ message("Reading and compiling definitions."); }
+{ htwolcpre3message("Reading and compiling definitions."); }
DEFINITION_LIST: /* empty */
| DEFINITION_LIST DEFINITION
@@ -307,7 +353,7 @@ DEFINITION: DEFINITION_NAME EQUALS REGULAR_EXPRESSION SEMI_COLON_LIST
REGULAR_EXPRESSION: RE_LIST
{ $$ = $1; }
| REGULAR_EXPRESSION UNION RE_LIST
-{
+{
$$ = &$1->apply(&HfstTransducer::disjunct,*$3);
delete $3;
}
@@ -340,7 +386,7 @@ REGULAR_EXPRESSION: RE_LIST
RE_LIST: /* empty */
{ $$ = new OtherSymbolTransducer(HFST_EPSILON); }
| RE_LIST RE
-{
+{
$$ = &$1->apply(&HfstTransducer::concatenate,*$2);
delete $2;
}
@@ -348,12 +394,12 @@ RE_LIST: /* empty */
RE: PAIR
{ $$ = $1; }
| RE POWER NUMBER
-{
+{
$$ = &$1->apply(&HfstTransducer::repeat_n,get_number($3));
free($3);
}
| RE POWER NUMBER_RANGE
-{
+{
$$ = &$1->apply(&HfstTransducer::repeat_n_to_k,
get_number($3),get_second_number($3));
free($3);
@@ -375,8 +421,8 @@ RE: PAIR
| LEFT_PARENTHESIS REGULAR_EXPRESSION RIGHT_PARENTHESIS
{ $$ = &$2->apply(&HfstTransducer::optionalize); }
-SET_LIST: /* empty */
-| SET_LIST SET_DEFINITION
+SET_LIST: /* empty */
+| SET_LIST SET_DEFINITION
SYMBOL_LIST: /* empty */
{ $$ = new SymbolRange; }
@@ -398,32 +444,32 @@ ALPHABET_PAIR_LIST: /* empty */
| ALPHABET_PAIR_LIST ALPHABET_PAIR
PAIR: PAIR_SYMBOL COLON PAIR_SYMBOL
-{
- if (std::string("__HFST_TWOLC_0") == $1 &&
+{
+ if (std::string("__HFST_TWOLC_0") == $1 &&
std::string("__HFST_TWOLC_0") == $3)
{ $$ = new OtherSymbolTransducer(HFST_EPSILON); }
else if (std::string("__HFST_TWOLC_#") == $1)
- {
+ {
// __HFST_TWOLC_# corresponds to the #-symbol in grammars. It should
// be split into an absolute word boundary "__HFST_TWOLC_.#." and a
// relative word boundary "#". On the surface the relative word-boundary
// may correspond to whatever it did correspond to in the rule file.
// The absolute word boundary surface realization is treated the same as
// the surface realizations of other absolute word boundaries.
- OtherSymbolTransducer wb =
+ OtherSymbolTransducer wb =
alphabet.get_transducer(SymbolPair
("__HFST_TWOLC_.#.","__HFST_TWOLC_.#."));
- OtherSymbolTransducer alt_wb =
+ OtherSymbolTransducer alt_wb =
alphabet.get_transducer
(SymbolPair
- ("#",$3 == std::string("__HFST_TWOLC_#") ? "#" : $3));
+ ("#",$3 == std::string("__HFST_TWOLC_#") ? "#" : $3));
wb.apply(&HfstTransducer::disjunct,alt_wb);
$$ = new OtherSymbolTransducer(wb);
- }
+ }
else
{ $$ = new OtherSymbolTransducer
- (alphabet.get_transducer(SymbolPair($1,$3)));
+ (alphabet.get_transducer(SymbolPair($1,$3)));
if (alphabet.is_empty_pair(SymbolPair($1,$3)))
{
std::string error;
@@ -438,7 +484,7 @@ PAIR: PAIR_SYMBOL COLON PAIR_SYMBOL
std::string symbol1 = Rule::get_print_name($1);
std::string symbol2 = Rule::get_print_name($3);
- error = std::string("The pair set ") + symbol1 + ":" + symbol2 +
+ error = std::string("The pair set ") + symbol1 + ":" + symbol2 +
" is empty.";
}
error += std::string("\n\n") +
@@ -449,10 +495,10 @@ PAIR: PAIR_SYMBOL COLON PAIR_SYMBOL
"the context of a rule or result from substituting values for\n" +
"variables in a rule with variables.\n\n" +
- "Compilation is terminated because a rule context, definition\n" +
+ "Compilation is terminated because a rule context, definition\n" +
"or rule center becomes empty.\n\n";
- semantic_error(error.c_str());
+ htwolcpre3_semantic_error(error.c_str());
}
}
@@ -460,8 +506,8 @@ PAIR: PAIR_SYMBOL COLON PAIR_SYMBOL
free($3);
}
| DEFINITION_NAME COLON DEFINITION_NAME
-{
- $$ = new OtherSymbolTransducer(definition_map[$1]);
+{
+ $$ = new OtherSymbolTransducer(definition_map[$1]);
free($1);
free($3);
}
@@ -472,7 +518,7 @@ PAIR_SYMBOL: SYMBOL
{ $$ = string_copy("__HFST_TWOLC_?"); }
ALPHABET_PAIR: SYMBOL COLON SYMBOL
-{
+{
alphabet.define_alphabet_pair(SymbolPair($1,$3));
free($1);
free($3);
@@ -484,28 +530,30 @@ SEMI_COLON_LIST: SEMI_COLON
%%
// Print warning.
-void warn(const char * warning)
-{
+void htwolcpre3warn(const char * warning)
+{
#ifdef DEBUG_TWOLC_3_GRAMMAR
std::cerr << warning << std::endl;
#else
- (void)warning;
+ (void)warning;
#endif
+ htwolcpre3_input_reader.warn(warning);
}
-// Print error messge and exit 1.
-void yyerror(const char * text)
-{
+// Print error messge and throw an exception
+void htwolcpre3error(const char * text)
+{
(void)text;
- input_reader.error(text);
- exit(1);
+ htwolcpre3_input_reader.error(text);
+ throw HfstException();
}
// Print error messge and exit 1.
-void semantic_error(const char * text)
-{
- std::cerr << std::endl << "Error: " << text << std::endl;
- exit(1);
+void htwolcpre3_semantic_error(const char * text)
+{
+ htwolcpre3_input_reader.error(text);
+ //std::cerr << std::endl << "Error: " << text << std::endl;
+ throw HfstException();
}
unsigned int get_number(const std::string &s)
@@ -526,9 +574,9 @@ unsigned int get_second_number(const std::string &s)
return number;
}
-void message(const std::string &m)
+void htwolcpre3message(const std::string &m)
{
- if (! silent)
+ if (! silent_)
{ std::cerr << m << std::endl; }
}
@@ -543,66 +591,3 @@ std::string get_name(const std::string &s)
return ss;
}
-int main(int argc, char * argv[])
-{
-#ifdef WINDOWS
- _setmode(0, _O_BINARY);
- _setmode(1, _O_BINARY);
-#endif
-
-#ifdef DEBUG_TWOLC_3_GRAMMAR
- yydebug = 1;
-#endif
-
- try
- {
- CommandLine command_line(argc,argv);
- if (command_line.help || command_line.usage || command_line.version)
- { exit(0); }
- if (command_line.has_debug_file)
- { input_reader.set_input(command_line.set_input_file()); }
- else
- { input_reader.set_input(std::cin); }
-
- OtherSymbolTransducer::set_transducer_type(command_line.format);
- silent = command_line.be_quiet;
- verbose = command_line.be_verbose;
-
- TwolCGrammar twolc_grammar(command_line.be_quiet,
- command_line.be_verbose,
- command_line.resolve_left_conflicts,
- command_line.resolve_right_conflicts);
- grammar = &twolc_grammar;
- int exit_code = yyparse();
- if (exit_code != 0)
- { exit(exit_code); }
-
- message("Compiling and storing rules.");
- if (! command_line.has_output_file)
- {
- HfstOutputStream stdout_(command_line.format);
- grammar->compile_and_store(stdout_);
- }
- else
- {
- HfstOutputStream out
- (command_line.output_file_name,command_line.format);
- grammar->compile_and_store(out);
- }
- exit(0);
- }
- catch (const HfstException e)
- {
- std::cerr << "This is an hfst interface bug:" << std::endl
- << e() << std::endl;
- exit(1);
- }
- catch (const char * s)
- {
- std::cerr << "This is an a bug probably from sfst:" << std::endl
- << s << std::endl;
- exit(1);
- }
-}
-
-
diff --git a/tools/src/hfst-twolc/src/io_src/InputReader.cc b/libhfst/src/parsers/io_src/InputReader.cc
similarity index 80%
rename from tools/src/hfst-twolc/src/io_src/InputReader.cc
rename to libhfst/src/parsers/io_src/InputReader.cc
index 30ed070..ab20ac5 100644
--- a/tools/src/hfst-twolc/src/io_src/InputReader.cc
+++ b/libhfst/src/parsers/io_src/InputReader.cc
@@ -20,7 +20,9 @@ InputReader::InputReader(size_t &counter):
input_stream(NULL),
counter(counter),
buffer_size(500000),
- buffer_index(0)
+ buffer_index(0),
+ warning_stream(NULL),
+ error_stream(NULL)
{}
void InputReader::set_input(std::istream &file)
@@ -29,21 +31,37 @@ void InputReader::set_input(std::istream &file)
input_stream->getline(buffer,buffer_size);
}
+void InputReader::set_warning_stream(std::ostream & ostr)
+{
+ warning_stream = &ostr;
+}
+
+void InputReader::set_error_stream(std::ostream & ostr)
+{
+ error_stream = &ostr;
+}
+
void InputReader::warn(const std::string &warning)
{
- std::cerr << std::endl;
- std::cerr << warning << std::endl;
- std::cerr << "on line " << counter << ":" << std::endl;
- std::cerr << buffer << std::endl;
+ if (warning_stream != NULL)
+ {
+ *warning_stream << std::endl;
+ *warning_stream << warning << std::endl;
+ *warning_stream << "on line " << counter << ":" << std::endl;
+ *warning_stream << buffer << std::endl;
+ }
}
void InputReader::error(const std::string &err)
{
- std::cerr << std::endl;
- std::cerr << err << std::endl;
- std::cerr << "on line " << counter << ":" << std::endl;
- std::cerr << buffer << std::endl;
- std::cerr << "Aborted." << std::endl << std::endl;
+ if (error_stream != NULL)
+ {
+ *error_stream << std::endl;
+ *error_stream << err << std::endl;
+ *error_stream << "on line " << counter << ":" << std::endl;
+ *error_stream << buffer << std::endl;
+ *error_stream << "Aborted." << std::endl << std::endl;
+ }
}
char InputReader::input(void)
@@ -72,7 +90,7 @@ int main(void)
size_t counter = 1;
InputReader ir1(counter);
try
- {
+ {
ir1.input();
assert(false);
}
diff --git a/tools/src/hfst-twolc/src/io_src/InputReader.h b/libhfst/src/parsers/io_src/InputReader.h
similarity index 87%
rename from tools/src/hfst-twolc/src/io_src/InputReader.h
rename to libhfst/src/parsers/io_src/InputReader.h
index 04c20ca..37cb25f 100644
--- a/tools/src/hfst-twolc/src/io_src/InputReader.h
+++ b/libhfst/src/parsers/io_src/InputReader.h
@@ -1,11 +1,11 @@
//! @file InputReader.h
//!
//! @author Miikka Silfverberg
-//!
+//!
//! @note This file is based on the tutorial "Better error handling using
//! Flex and Bison", by Christian Hagen. The
//! <a href=
-//! "http://www.ibm.com/developerworks/opensource/library/l-flexbison.html">
+//! "http://www.ibm.com/developerworks/opensource/library/l-flexbison.html">
//! tutorial</a>.
//!
//! @brief Class for Flex and Bison input reading.
@@ -41,7 +41,7 @@ class InputNotSet
//! @brief class For reading input from an @a std::istream one character at a
//! time.
//!
-//! The class is also used for displaying error messages and warnings that
+//! The class is also used for displaying error messages and warnings that
//! point to the position of the error on the input line.
class InputReader
{
@@ -51,17 +51,25 @@ private:
char buffer[500000];
size_t buffer_size;
size_t buffer_index;
+ std::ostream * warning_stream;
+ std::ostream * error_stream;
public:
//! @brief Initialize a reader with line counter @a counter.
//!
- //! @param counter would usually be a reference to @a yylineno which is
+ //! @param counter would usually be a reference to @a yylineno which is
//! maintained by Bison.
InputReader(size_t &counter);
//! @brief Set the input stream from which the twolc-grmmar is read.
void set_input(std::istream &file);
+ //! @brief Set the output stream where \a warn prints its input.
+ void set_warning_stream(std::ostream & os);
+
+ //! @brief Set the output stream where \a error prints its input.
+ void set_error_stream(std::ostream & os);
+
//! @brief Display the warning message @a warning.
void warn(const std::string &warning);
diff --git a/tools/src/hfst-twolc/src/io_src/input_defs.h b/libhfst/src/parsers/io_src/input_defs.h
similarity index 99%
rename from tools/src/hfst-twolc/src/io_src/input_defs.h
rename to libhfst/src/parsers/io_src/input_defs.h
index c4cc785..cc01008 100644
--- a/tools/src/hfst-twolc/src/io_src/input_defs.h
+++ b/libhfst/src/parsers/io_src/input_defs.h
@@ -42,6 +42,6 @@ result = YY_NULL; \
buf[0] = c; \
result = 1; \
} \
-}
+}
#endif // INPUT_DEFS_H
diff --git a/libhfst/src/parsers/lexc-lexer.ll b/libhfst/src/parsers/lexc-lexer.ll
index 9ad611f..f69f764 100644
--- a/libhfst/src/parsers/lexc-lexer.ll
+++ b/libhfst/src/parsers/lexc-lexer.ll
@@ -1,13 +1,13 @@
%option 8Bit batch nounput noyywrap prefix="hlexc"
%{
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
//! @file lexc-lexer.cc
@@ -27,10 +27,16 @@
#endif
#include "lexc-utils.h"
+#include "HfstDataTypes.h"
#include <assert.h>
+
extern void hlexcerror(const char *text);
+
+#undef YY_FATAL_ERROR
+#define YY_FATAL_ERROR(msg) hlexcerror(msg);
+
%}
/* c.f. Unicode Standard 5.1 D92 Table 3-7 */
@@ -64,7 +70,7 @@ XREUNRESTRICTED [\x21-\x7e\n]{-}[- <>%!:;@0~""\\&?$+*/_{}\]\[-]
/* RegExp. stuff */
XREALPHA {XREUNRESTRICTED}|{U8H}|{EC}
-XREOPERATOR [~\\&\-/\t |+*$_\]\[{}()0?:""]|"@\""|"$."|"$?"|"./."|"<>"|".#."|"."[riul]
+XREOPERATOR [~\\&\-/\t |+*$_\]\[{}()0?:""]|"@\""|"$."|"$?"|"./."|"<>"|"^>"|"^<"|".#."|"."[riul]
XREQUOTSTRING "\""([^""\n;]|"%;"|"\\\"")*"\""
XRECHAR {XREOPERATOR}|{XREALPHA}|{XREQUOTSTRING}
XRETOKEN {XRECHAR}+
@@ -248,7 +254,7 @@ LWSP [\r\n\t ]
}
-<ENDED>. {
+<ENDED>. {
hfst::lexc::token_update_positions(hlexctext);
}
diff --git a/libhfst/src/parsers/lexc-parser.yy b/libhfst/src/parsers/lexc-parser.yy
index 75f494b..767f246 100644
--- a/libhfst/src/parsers/lexc-parser.yy
+++ b/libhfst/src/parsers/lexc-parser.yy
@@ -1,11 +1,11 @@
%{
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
//! @file lexc-parser.yy
@@ -37,7 +37,7 @@ static
void
handle_multichar(const string& multichar)
{
- //in multichars @ZERO@ should right away be replaced with "0"
+ //in multichars @ZERO@ should right away be replaced with "0"
string str = multichar;
string zero = "@ZERO@";
size_t start_pos = str.find(zero);
@@ -67,12 +67,13 @@ static
bool
handle_lexicon_name(const string& lexiconName)
{
- try
+ try
{
hfst::lexc::lexc_->setCurrentLexiconName(lexiconName);
}
catch(const char * msg)
{
+ (void)msg;
return false;
}
return true;
@@ -138,8 +139,9 @@ handle_string_pair_entry(const string& upper, const string& lower,
try {
hfst::lexc::lexc_->addStringPairEntry(upper, lower, cont, weight);
} catch(const char * msg) {
+ (void)msg;
return false;
- }
+ }
}
else
{
@@ -152,6 +154,7 @@ handle_string_pair_entry(const string& upper, const string& lower,
try {
hfst::lexc::lexc_->addStringPairEntry(upper_, lower_, cont, weight);
} catch(const char * msg) {
+ (void)msg;
return false;
}
}
@@ -174,7 +177,7 @@ static
void
hlexcwarn(const char* text)
{
- if (! hfst::lexc::lexc_->isQuiet())
+ if (! hfst::lexc::lexc_->isQuiet())
{ hfst::lexc::error_at_current_token(0, 0, text); }
}
@@ -200,7 +203,7 @@ handle_end()
// just cos I use the llloc struct
%locations
-%union
+%union
{
char* name;
int number;
@@ -208,7 +211,7 @@ handle_end()
%token <number> ERROR MULTICHARS_START
DEFINITIONS_START END_START NOFLAGS_START
-%token <name> LEXICON_START LEXICON_START_WRONG_CASE
+%token <name> LEXICON_START LEXICON_START_WRONG_CASE
LEXICON_NAME ULSTRING ENTRY_GLOSS
MULTICHAR_SYMBOL
DEFINITION_NAME DEFINITION_EXPRESSION
@@ -225,7 +228,7 @@ MULTICHAR_PART: MULTICHAR_SYMBOLS2 MULTICHAR_SYMBOL_LIST
|
;
-MULTICHAR_SYMBOLS2: MULTICHARS_START
+MULTICHAR_SYMBOLS2: MULTICHARS_START
;
MULTICHAR_SYMBOL_LIST: MULTICHAR_SYMBOL_LIST MULTICHAR_SYMBOL2
@@ -242,7 +245,7 @@ NOFLAGS_PART: NOFLAG_LEXICONS2 NOFLAG_LEXICON_LIST
|
;
-NOFLAG_LEXICONS2: NOFLAGS_START
+NOFLAG_LEXICONS2: NOFLAGS_START
;
NOFLAG_LEXICON_LIST: NOFLAG_LEXICON_LIST NOFLAG_LEXICON2
@@ -259,7 +262,7 @@ DEFINITIONS_PART: DEFINITIONS_START2 DEFINITION_LIST
|
;
-DEFINITIONS_START2: DEFINITIONS_START
+DEFINITIONS_START2: DEFINITIONS_START
;
DEFINITION_LIST: DEFINITION_LIST DEFINITION_LINE
@@ -287,7 +290,7 @@ LEXICON2: LEXICON_START {
{ hlexcerror("Sublexicon defined more than once."); YYABORT; }
}
| LEXICON_START_WRONG_CASE {
- if (hfst::lexc::lexc_->areWarningsTreatedAsErrors())
+ if (hfst::lexc::lexc_->areWarningsTreatedAsErrors())
{ hlexcerror("Keyword 'Lexicon' used instead of 'LEXICON'. [--Werror]"); YYABORT; }
else
{ hlexcwarn("Titlecase Lexicon parsed as LEXICON"); }
@@ -393,7 +396,7 @@ LEXICON_LINE: ULSTRING LEXICON_NAME ';' {
}
;
-END_PART: END_START {
+END_PART: END_START {
handle_end();
}
|
diff --git a/libhfst/src/parsers/lexc-utils.cc b/libhfst/src/parsers/lexc-utils.cc
index 5429a97..766b7f5 100644
--- a/libhfst/src/parsers/lexc-utils.cc
+++ b/libhfst/src/parsers/lexc-utils.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
//! @file lexc-utils.cc
@@ -196,14 +196,14 @@ void
token_update_positions(const char *token)
{
size_t token_length = strlen(token);
- int newlines = count_newlines(token);
+ int newlines = hfst::size_t_to_int(count_newlines(token));
hlexclloc.first_line = hlexclloc.last_line;
hlexclloc.last_line = (hlexclloc.first_line + newlines);
// FIXME: columns equal bytes not characters
hlexclloc.first_column = hlexclloc.last_column + 1;
if (0 == newlines)
{
- hlexclloc.last_column = (hlexclloc.first_column + token_length);
+ hlexclloc.last_column = (hlexclloc.first_column + hfst::size_t_to_int(token_length));
}
else
{
@@ -219,14 +219,14 @@ strdup_token_positions()
// N.B. reason for this error format is automagic support by vim/emacs/jedit
// must be “filename:lineno:colno-lineno:colno: stuff�
// c.f. http://www.gnu.org/prep/standards/standards.html#Errors
- char* filenames_lines_cols = (char*)malloc(sizeof(char) *
+ char* filenames_lines_cols = (char*)malloc(sizeof(char) *
(strlen(hlexcfilename) + 100));
// linenumbers and columns
- if( (hlexclloc.first_line == hlexclloc.last_line) &&
+ if( (hlexclloc.first_line == hlexclloc.last_line) &&
(hlexclloc.first_column == (hlexclloc.last_column - 1)) )
{
// TRANSLATORS: filename, line and column
- (void)sprintf(filenames_lines_cols, "%s:%d.%d",
+ (void)sprintf(filenames_lines_cols, "%s:%d.%d",
hlexcfilename, hlexclloc.first_line, hlexclloc.first_column);
}
else if ( hlexclloc.first_line == hlexclloc.last_line )
@@ -280,7 +280,7 @@ strip_percents(const char* s, bool do_zeros)
if (do_zeros)
{
// worst case scenario is string of 0's...
- rv = (char*)malloc(sizeof(char) *
+ rv = (char*)malloc(sizeof(char) *
strlen(s) * strlen("@0@")
+ 1);
}
@@ -298,7 +298,7 @@ strip_percents(const char* s, bool do_zeros)
{
if (in_at)
{
- if (*p == '@')
+ if (*c == '@')
{
in_at = false;
}
@@ -420,14 +420,14 @@ strdup_nonconst_part(const char* token, const char* prefix,
varpart_len = strlen(token) - prefix_len - suffix_len;
assert(varpart_len <= token_len);
- if (prefix == NULL)
+ if (prefix == NULL)
{ assert(strncmp(token, "", prefix_len) == 0); }
- else
+ else
{ assert(strncmp(token, prefix, prefix_len) == 0); }
- if (suffix == NULL)
+ if (suffix == NULL)
{ assert(strncmp(token + prefix_len + varpart_len, "", suffix_len) == 0); }
- else
+ else
{ assert(strncmp(token + prefix_len + varpart_len, suffix, suffix_len) == 0); }
(void)memcpy(token_part, token + prefix_len, varpart_len);
@@ -473,7 +473,7 @@ pair<vector<string>, vector<string> > find_med_alingment(const vector<string> &s
}
for(unsigned int i = 1; i <= len2; ++i)
- {
+ {
d[0][i] = insertion * i;
dir[0][i] = INSERT;
}
@@ -493,9 +493,9 @@ pair<vector<string>, vector<string> > find_med_alingment(const vector<string> &s
d[i][j] = sub;
dir[i][j] = SUBSTITUTE;
}
- // It's important to prioritize "del" when it has the same value as "ins"" because we want the first
+ // It's important to prioritize "del" when it has the same value as "ins"" because we want the first
// string to have zeroes before the second one.
- // For example, if we have two strings: abc and xyz, we would prefer the output 000abc:xyz000
+ // For example, if we have two strings: abc and xyz, we would prefer the output 000abc:xyz000
// over abc000:000xyz, because we need to invert the transducer to use it for the analysis and in lookup
// we want zeroes as late as possible on the upper side.
// Anyway, the order of this two following blocks determines the order of zeroes on upper/lower side
@@ -516,7 +516,7 @@ pair<vector<string>, vector<string> > find_med_alingment(const vector<string> &s
vector<string> medcwordout;
int x, y, i ;
- for ( x = s1.size(), y = s2.size(), i = 0; (x > 0) || (y > 0); i++)
+ for ( x = hfst::size_t_to_int(s1.size()), y = hfst::size_t_to_int(s2.size()), i = 0; (x > 0) || (y > 0); i++)
{
int dirValue = dir[x][y];
@@ -549,5 +549,5 @@ pair<vector<string>, vector<string> > find_med_alingment(const vector<string> &s
}
-} }
+} }
diff --git a/libhfst/src/parsers/lexc-utils.h b/libhfst/src/parsers/lexc-utils.h
index 5c5cd37..2ab49ee 100644
--- a/libhfst/src/parsers/lexc-utils.h
+++ b/libhfst/src/parsers/lexc-utils.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
//! @file lexc-utils.h
@@ -47,7 +47,7 @@ const char REG_EX_END[] = "_$";
//! @brief Strips lexc style percent escaping from a string.
//!
-//! E.g. like stripslashes() in PHP.
+//! E.g. like stripslashes() in PHP.
std::string& stripPercents(std::string& s);
//! @brief Percent encode critical characters in raw string for lexc.
@@ -102,7 +102,7 @@ char* strstrip(const char* s);
//! @brief extracts the variable substring part from token.
//! Omits constant string prefix, suffix and optionally strips spaces.
-char* strdup_nonconst_part(const char* token,
+char* strdup_nonconst_part(const char* token,
const char* prefix,
const char* suffix,
bool strip);
diff --git a/libhfst/src/parsers/pmatch_lex.ll b/libhfst/src/parsers/pmatch_lex.ll
index 33c8b6d..dfd864f 100644
--- a/libhfst/src/parsers/pmatch_lex.ll
+++ b/libhfst/src/parsers/pmatch_lex.ll
@@ -1,13 +1,13 @@
%option 8Bit batch yylineno noyywrap nounput prefix="pmatch"
%{
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include <string.h>
@@ -26,8 +26,10 @@
#undef YY_INPUT
#define YY_INPUT(buf, retval, maxlen) (retval = hfst::pmatch::getinput(buf, maxlen))
-extern
-int pmatcherror(char *text);
+extern int pmatcherror(const char *text);
+
+#undef YY_FATAL_ERROR
+#define YY_FATAL_ERROR(msg) pmatcherror(msg);
%}
@@ -72,6 +74,7 @@ UNICODE_ESCAPE ("\\u"{HEXCHAR}{HEXCHAR}{HEXCHAR}{HEXCHAR})|("\\U00"{HEXCHAR}{HEX
[Dd]"efine" { return DEFINE; }
"DefFun" { return DEFINE; }
"regex" { return REGEX; }
+"set" { return SET_VARIABLE; }
"list" { return DEFINED_LIST; }
"Lit(" { return LIT_LEFT; }
"Ins(" { return INS_LEFT; }
@@ -94,11 +97,11 @@ UNICODE_ESCAPE ("\\u"{HEXCHAR}{HEXCHAR}{HEXCHAR}{HEXCHAR})|("\\U00"{HEXCHAR}{HEX
".t(" { return TAG_LEFT; }
"Lst(" { return LST_LEFT; }
"Exc(" { return EXC_LEFT; }
+"Like(" { return LIKE_LEFT; }
"Interpolate(" { return INTERPOLATE_LEFT; }
"Sigma(" { return SIGMA_LEFT; }
"Counter(" { return COUNTER_LEFT; }
-"DownCase(" { return TOUPPER_LEFT; }
-"Define(" { return DEFINE_LEFT; }
+[Dd]"efine(" { return DEFINE_LEFT; }
"DefIns" { return DEFINS; }
"Alpha" { return ALPHA; }
@@ -108,6 +111,44 @@ UNICODE_ESCAPE ("\\u"{HEXCHAR}{HEXCHAR}{HEXCHAR}{HEXCHAR})|("\\U00"{HEXCHAR}{HEX
"Punct" { return PUNCT; }
"Whitespace" { return WHITESPACE; }
+"count-patterns" {
+ pmatchlval.label = strcpy((char *) malloc(strlen("count-patterns") + 1), "count-patterns");
+ return VARIABLE_NAME;
+}
+"delete-patterns" {
+ pmatchlval.label = strcpy((char *) malloc(strlen("delete-patterns") + 1), "delete-patterns");
+ return VARIABLE_NAME;
+}
+"extract-patterns" {
+ pmatchlval.label = strcpy((char *) malloc(strlen("extract-patterns") + 1), "extract-patterns");
+ return VARIABLE_NAME;
+}
+"locate-patterns" {
+ pmatchlval.label = strcpy((char *) malloc(strlen("locate-patterns") + 1), "locate-patterns");
+ return VARIABLE_NAME;
+}
+"mark-patterns" {
+ pmatchlval.label = strcpy((char *) malloc(strlen("mark-patterns") + 1), "mark-patterns");
+ return VARIABLE_NAME;
+}
+"need-separators" {
+ pmatchlval.label = strcpy((char *) malloc(strlen("need-separators") + 1), "need-separators");
+ return VARIABLE_NAME;
+}
+"max-context-length" {
+ pmatchlval.label = strcpy((char *) malloc(strlen("max-context_length") + 1), "max-context-length");
+ return VARIABLE_NAME;
+}
+"max-recursion" {
+ pmatchlval.label = strcpy((char *) malloc(strlen("max-recursion") + 1), "max-recursion");
+ return VARIABLE_NAME;
+}
+
+"vector-similarity-projection-factor" {
+ pmatchlval.label = strcpy((char *) malloc(strlen("vector-similarity-projection-factor") + 1), "vector-similarity-projection-factor");
+ return VARIABLE_NAME;
+}
+
"~" { return COMPLEMENT; }
"\\" { return TERM_COMPLEMENT; }
"&" { return INTERSECTION; }
@@ -231,6 +272,11 @@ UNICODE_ESCAPE ("\\u"{HEXCHAR}{HEXCHAR}{HEXCHAR}{HEXCHAR})|("\\U00"{HEXCHAR}{HEX
return READ_RE;
}
+"@vec\""[^""]+"\"" {
+ pmatchlval.label = hfst::pmatch::get_escaped_delimited(pmatchtext, '"');
+ return READ_VEC;
+}
+
"\""(({UNICODE_ESCAPE}|{U8C})"-"({UNICODE_ESCAPE}|{U8C}))+"\"" {
pmatchlval.pmatchObject = hfst::pmatch::parse_range(pmatchtext);
return CHARACTER_RANGE;
@@ -295,7 +341,7 @@ UNICODE_ESCAPE ("\\u"{HEXCHAR}{HEXCHAR}{HEXCHAR}{HEXCHAR})|("\\U00"{HEXCHAR}{HEX
("!")[^\n]*$ { /* ignore comments */ }
-. {
+. {
return LEXER_ERROR;
}
diff --git a/libhfst/src/parsers/pmatch_parse.yy b/libhfst/src/parsers/pmatch_parse.yy
index 2fc251e..6cb7c43 100644
--- a/libhfst/src/parsers/pmatch_parse.yy
+++ b/libhfst/src/parsers/pmatch_parse.yy
@@ -1,11 +1,11 @@
%{
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#define YYDEBUG 0
@@ -50,13 +50,13 @@
hfst::pmatch::PmatchObjectPair * restrictionContext;
hfst::pmatch::MappingPairVector * restrictionContexts;
hfst::xeroxRules::ReplaceType replType;
- hfst::xeroxRules::ReplaceArrow replaceArrow;
+ hfst::xeroxRules::ReplaceArrow replaceArrow;
}
%start PMATCH
%type <pmatchDefinition> DEFINITION
%type <pmatchObject> EXPRESSION1 EXPRESSION2 EXPRESSION3 EXPRESSION4 EXPRESSION5 EXPRESSION6 EXPRESSION7 EXPRESSION8 EXPRESSION9 EXPRESSION10 EXPRESSION11 EXPRESSION12 EXPRESSION13
-%type <label> SYMBOL QUOTED_LITERAL CURLY_LITERAL SYMBOL_WITH_LEFT_PAREN CONCATENATED_STRING_LIST
+%type <label> SYMBOL QUOTED_LITERAL CURLY_LITERAL SYMBOL_WITH_LEFT_PAREN CONCATENATED_STRING_LIST VARIABLE_NAME
%type <string_vector> ARGLIST
%type <pmatchObject_vector> FUNCALL_ARGLIST
@@ -69,7 +69,7 @@
%type <restrictionContext> RESTR_CONTEXT
%type <restrictionContexts> RESTR_CONTEXTS
%type <replType> CONTEXT_MARK
-%type <pmatchObject> INSERTION FUNCALL EXPLODE IMPLODE ENDTAG READ_FROM CONTEXT_CONDITION PMATCH_CONTEXT PMATCH_OR_CONTEXT PMATCH_AND_CONTEXT
+%type <pmatchObject> INSERTION FUNCALL EXPLODE IMPLODE ENDTAG LIKE READ_FROM CONTEXT_CONDITION PMATCH_CONTEXT PMATCH_OR_CONTEXT PMATCH_AND_CONTEXT
PMATCH_RIGHT_CONTEXT PMATCH_LEFT_CONTEXT PMATCH_NEGATIVE_RIGHT_CONTEXT PMATCH_NEGATIVE_LEFT_CONTEXT
%type <pmatchObject_vector> PMATCH_CONTEXTS
@@ -82,7 +82,7 @@ PMATCH_RIGHT_CONTEXT PMATCH_LEFT_CONTEXT PMATCH_NEGATIVE_RIGHT_CONTEXT PMATCH_NE
%right LEFT_ARROW RIGHT_ARROW LEFT_RIGHT_ARROW LEFT_RESTRICTION // LEFT_RESTRICTION not implemented
%left REPLACE_RIGHT REPLACE_LEFT OPTIONAL_REPLACE_RIGHT OPTIONAL_REPLACE_LEFT
REPLACE_LEFT_RIGHT OPTIONAL_REPLACE_LEFT_RIGHT RTL_LONGEST_MATCH RTL_SHORTEST_MATCH
-LTR_LONGEST_MATCH LTR_SHORTEST_MATCH
+LTR_LONGEST_MATCH LTR_SHORTEST_MATCH
%right REPLACE_CONTEXT_UU REPLACE_CONTEXT_LU REPLACE_CONTEXT_UL REPLACE_CONTEXT_LL
%left UNION MINUS UPPER_MINUS LOWER_MINUS UPPER_PRIORITY_UNION LOWER_PRIORITY_UNION
%left IGNORING IGNORE_INTERNALLY LEFT_QUOTIENT
@@ -90,7 +90,7 @@ LTR_LONGEST_MATCH LTR_SHORTEST_MATCH
%nonassoc SUBSTITUTE_LEFT TERM_COMPLEMENT COMPLEMENT CONTAINMENT CONTAINMENT_ONCE CONTAINMENT_OPT
%nonassoc STAR PLUS REVERSE INVERT UPPER_PROJECT LOWER_PROJECT
-%nonassoc <label> READ_BIN READ_TEXT READ_SPACED READ_PROLOG READ_RE READ_LEXC
+%nonassoc <label> READ_BIN READ_TEXT READ_SPACED READ_PROLOG READ_RE READ_VEC READ_LEXC
%nonassoc <values> CATENATE_N_TO_K
%nonassoc <value> CATENATE_N CATENATE_N_PLUS CATENATE_N_MINUS
@@ -98,10 +98,11 @@ LTR_LONGEST_MATCH LTR_SHORTEST_MATCH
%token PAIR_SEPARATOR PAIR_SEPARATOR_SOLE PAIR_SEPARATOR_WO_RIGHT PAIR_SEPARATOR_WO_LEFT
%token EPSILON_TOKEN ANY_TOKEN BOUNDARY_MARKER
%token LEXER_ERROR SYMBOL SYMBOL_WITH_LEFT_PAREN QUOTED_LITERAL CURLY_LITERAL
-%token ALPHA LOWERALPHA UPPERALPHA NUM PUNCT WHITESPACE
+%token ALPHA LOWERALPHA UPPERALPHA NUM PUNCT WHITESPACE VARIABLE_NAME
// MAP_LEFT
-%right DEFINE LIT_LEFT INS_LEFT REGEX DEFINS DEFINED_LIST CAP_LEFT OPTCAP_LEFT OPT_TOLOWER_LEFT TOLOWER_LEFT
-OPT_TOUPPER_LEFT TOUPPER_LEFT ANY_CASE_LEFT IMPLODE_LEFT EXPLODE_LEFT DEFINE_LEFT ENDTAG_LEFT LC_LEFT RC_LEFT NLC_LEFT NRC_LEFT OR_LEFT AND_LEFT
+%right DEFINE SET_VARIABLE
+LIT_LEFT INS_LEFT REGEX DEFINS DEFINED_LIST CAP_LEFT OPTCAP_LEFT OPT_TOLOWER_LEFT TOLOWER_LEFT
+OPT_TOUPPER_LEFT TOUPPER_LEFT ANY_CASE_LEFT IMPLODE_LEFT EXPLODE_LEFT DEFINE_LEFT ENDTAG_LEFT LIKE_LEFT LC_LEFT RC_LEFT NLC_LEFT NRC_LEFT OR_LEFT AND_LEFT
TAG_LEFT LST_LEFT EXC_LEFT INTERPOLATE_LEFT SIGMA_LEFT COUNTER_LEFT
%%
@@ -122,7 +123,21 @@ PMATCH: //empty
delete definitions[$2->first];
}
definitions.insert(*$2);
- };
+ }
+| PMATCH SET_VARIABLE VARIABLE_NAME SYMBOL {
+ hfst::pmatch::variables[$3] = $4;
+ free($3); free($4);
+ } |
+ PMATCH SET_VARIABLE VARIABLE_NAME EPSILON_TOKEN {
+ // the symbol can be 0, and that pretty much has to be reserved for
+ // epsilon, so we detect that possibility here
+ hfst::pmatch::variables[$3] = "0";
+ free($3);
+ } | PMATCH READ_VEC {
+ std::string filepath = hfst::pmatch::path_from_filename($2);
+ free($2);
+ hfst::pmatch::read_vec(filepath);
+ };
DEFINITION: DEFINE SYMBOL EXPRESSION1 {
$$ = new std::pair<std::string, PmatchObject*>($2, $3);
@@ -154,9 +169,11 @@ DEFINITION: DEFINE SYMBOL EXPRESSION1 {
};
ARGLIST:
-SYMBOL COMMA ARGLIST { $$ = $3; $$->push_back(std::string($1));
- } | SYMBOL { $$ = new std::vector<std::string>(1, std::string($1)); }
-| { $$ = new std::vector<std::string>(); };
+SYMBOL COMMA ARGLIST { $$ = $3; $$->push_back(std::string($1)); free($1); } |
+QUOTED_LITERAL COMMA ARGLIST { $$ = $3; $$->push_back(std::string($1)); free($1); } |
+SYMBOL { $$ = new std::vector<std::string>(1, std::string($1)); free($1); } |
+QUOTED_LITERAL { $$ = new std::vector<std::string>(1, std::string($1)); free($1); } |
+{ $$ = new std::vector<std::string>(); };
EXPRESSION1: EXPRESSION2 END_OF_WEIGHTED_EXPRESSION {
$1->weight += $2;
@@ -355,7 +372,7 @@ TERM_COMPLEMENT EXPRESSION12 { $$ = new PmatchUnaryOperation(TermComplement, $2)
EXPRESSION12: EXPRESSION13 { } |
LEFT_BRACKET EXPRESSION2 RIGHT_BRACKET { $$ = $2; } |
-EXPRESSION12 PAIR_SEPARATOR EXPRESSION13 { $$ = new PmatchBinaryOperation(CrossProduct, $1, $3); } |
+EXPRESSION12 PAIR_SEPARATOR EXPRESSION12 { $$ = new PmatchBinaryOperation(CrossProduct, $1, $3); } |
LEFT_PARENTHESIS EXPRESSION2 RIGHT_PARENTHESIS { $$ = new PmatchUnaryOperation(Optionalize, $2); } |
EXPRESSION12 WEIGHT { $$ = $1; $$->weight += $2; } |
LEFT_BRACKET EXPRESSION2 RIGHT_BRACKET TAG_LEFT SYMBOL RIGHT_PARENTHESIS {
@@ -385,6 +402,7 @@ IMPLODE { } |
FUNCALL { } |
//MAP { } |
INSERTION { } |
+LIKE { } |
ALPHA { $$ = new PmatchAcceptor(Alpha); } |
LOWERALPHA { $$ = new PmatchAcceptor(LowercaseAlpha); } |
UPPERALPHA { $$ = new PmatchAcceptor(UppercaseAlpha); } |
@@ -398,6 +416,84 @@ TOUPPER_LEFT EXPRESSION2 RIGHT_PARENTHESIS { $$ = new PmatchUnaryOperation(ToUpp
OPT_TOLOWER_LEFT EXPRESSION2 RIGHT_PARENTHESIS { $$ = new PmatchUnaryOperation(OptToLower, $2); } |
OPT_TOUPPER_LEFT EXPRESSION2 RIGHT_PARENTHESIS { $$ = new PmatchUnaryOperation(OptToUpper, $2); } |
ANY_CASE_LEFT EXPRESSION2 RIGHT_PARENTHESIS { $$ = new PmatchUnaryOperation(AnyCase, $2); } |
+
+CAP_LEFT EXPRESSION2 COMMA SYMBOL RIGHT_PARENTHESIS {
+ if (strcmp($4, "U") == 0) {
+ $$ = new PmatchUnaryOperation(CapUpper, $2);
+ } else if (strcmp($4, "L") == 0) {
+ $$ = new PmatchUnaryOperation(CapLower, $2);
+ } else {
+ pmatcherror("Side argument to casing function not understood\n");
+ $$ = new PmatchUnaryOperation(Cap, $2);
+ }
+ free($4);
+} |
+OPTCAP_LEFT EXPRESSION2 COMMA SYMBOL RIGHT_PARENTHESIS {
+ if (strcmp($4, "U") == 0) {
+ $$ = new PmatchUnaryOperation(OptCapUpper, $2);
+ } else if (strcmp($4, "L") == 0) {
+ $$ = new PmatchUnaryOperation(OptCapLower, $2);
+ } else {
+ pmatcherror("Side argument to casing function not understood\n");
+ $$ = new PmatchUnaryOperation(OptCap, $2);
+ }
+ free($4);
+} |
+TOLOWER_LEFT EXPRESSION2 COMMA SYMBOL RIGHT_PARENTHESIS {
+ if (strcmp($4, "U") == 0) {
+ $$ = new PmatchUnaryOperation(ToLowerUpper, $2);
+ } else if (strcmp($4, "L") == 0) {
+ $$ = new PmatchUnaryOperation(ToLowerLower, $2);
+ } else {
+ pmatcherror("Side argument to casing function not understood\n");
+ $$ = new PmatchUnaryOperation(ToLower, $2);
+ }
+ free($4);
+} |
+TOUPPER_LEFT EXPRESSION2 COMMA SYMBOL RIGHT_PARENTHESIS {
+ if (strcmp($4, "U") == 0) {
+ $$ = new PmatchUnaryOperation(ToUpperUpper, $2);
+ } else if (strcmp($4, "L") == 0) {
+ $$ = new PmatchUnaryOperation(ToUpperLower, $2);
+ } else {
+ pmatcherror("Side argument to casing function not understood\n");
+ $$ = new PmatchUnaryOperation(ToUpper, $2);
+ }
+ free($4);
+} |
+OPT_TOLOWER_LEFT EXPRESSION2 COMMA SYMBOL RIGHT_PARENTHESIS {
+ if (strcmp($4, "U") == 0) {
+ $$ = new PmatchUnaryOperation(OptToLowerUpper, $2);
+ } else if (strcmp($4, "L") == 0) {
+ $$ = new PmatchUnaryOperation(OptToLowerLower, $2);
+ } else {
+ pmatcherror("Side argument to casing function not understood\n");
+ $$ = new PmatchUnaryOperation(OptToLower, $2);
+ }
+ free($4);
+} |
+OPT_TOUPPER_LEFT EXPRESSION2 COMMA SYMBOL RIGHT_PARENTHESIS {
+ if (strcmp($4, "U") == 0) {
+ $$ = new PmatchUnaryOperation(OptToUpperUpper, $2);
+ } else if (strcmp($4, "L") == 0) {
+ $$ = new PmatchUnaryOperation(OptToUpperLower, $2);
+ } else {
+ pmatcherror("Side argument to casing function not understood\n");
+ $$ = new PmatchUnaryOperation(OptToUpper, $2);
+ }
+ free($4);
+} |
+ANY_CASE_LEFT EXPRESSION2 COMMA SYMBOL RIGHT_PARENTHESIS {
+ if (strcmp($4, "U") == 0) {
+ $$ = new PmatchUnaryOperation(AnyCaseUpper, $2);
+ } else if (strcmp($4, "L") == 0) {
+ $$ = new PmatchUnaryOperation(AnyCaseLower, $2);
+ } else {
+ pmatcherror("Side argument to casing function not understood\n");
+ $$ = new PmatchUnaryOperation(AnyCase, $2);
+ }
+ free($4);
+} |
DEFINE_LEFT EXPRESSION2 RIGHT_PARENTHESIS { $$ = new PmatchUnaryOperation(AddDelimiters, $2); } |
READ_FROM { } |
CHARACTER_RANGE { $$ = $1; } |
@@ -407,7 +503,13 @@ INTERPOLATE_LEFT FUNCALL_ARGLIST RIGHT_PARENTHESIS { $$ = new PmatchBuiltinFunct
SIGMA_LEFT EXPRESSION2 RIGHT_PARENTHESIS { $$ = new PmatchUnaryOperation(MakeSigma, $2); } |
COUNTER_LEFT SYMBOL RIGHT_PARENTHESIS { $$ = hfst::pmatch::make_counter($2); free($2); } |
ENDTAG { $$ = $1; hfst::pmatch::need_delimiters = true; } |
-CONTEXT_CONDITION { $$ = $1; hfst::pmatch::need_delimiters = true; } |
+CONTEXT_CONDITION {
+ $$ = $1;
+ // We will wrap the current definition with entry and exit guards
+ hfst::pmatch::need_delimiters = true;
+ // Switch off the automatic separator-seeking context condition
+ hfst::pmatch::variables["need-separators"] = "off";
+} |
SYMBOL {
std::string sym($1);
free($1);
@@ -415,6 +517,7 @@ SYMBOL {
$$ = new PmatchEmpty;
} else {
$$ = new PmatchSymbol(sym);
+ used_definitions.insert(sym);
}
};
@@ -481,6 +584,29 @@ INSERTION: INS_LEFT SYMBOL RIGHT_PARENTHESIS {
free($2);
};
+LIKE: LIKE_LEFT ARGLIST RIGHT_PARENTHESIS {
+ if ($2->size() == 0) {
+ $$ = hfst::pmatch::compile_like_arc("");
+ } else if ($2->size() == 1) {
+ $$ = hfst::pmatch::compile_like_arc($2->operator[](0));
+ } else {
+ $$ = hfst::pmatch::compile_like_arc($2->operator[](0),
+ $2->operator[](1));
+ }
+ delete($2);
+} |
+LIKE_LEFT ARGLIST RIGHT_PARENTHESIS CATENATE_N {
+ if ($2->size() == 0) {
+ $$ = hfst::pmatch::compile_like_arc("");
+ } else if ($2->size() == 1) {
+ $$ = hfst::pmatch::compile_like_arc($2->operator[](0), "", $4);
+ } else {
+ $$ = hfst::pmatch::compile_like_arc($2->operator[](0),
+ $2->operator[](1), $4);
+ }
+ delete($2);
+}
+
ENDTAG: ENDTAG_LEFT SYMBOL RIGHT_PARENTHESIS {
$$ = hfst::pmatch::make_end_tag($2);
free($2);
@@ -490,49 +616,36 @@ ENDTAG: ENDTAG_LEFT SYMBOL RIGHT_PARENTHESIS {
};
READ_FROM: READ_BIN {
- HfstTransducer * read;
+ std::string filepath = hfst::pmatch::path_from_filename($1);
+ free($1);
+ HfstTransducer * read = NULL;
try {
- hfst::HfstInputStream instream($1);
+ hfst::HfstInputStream instream(filepath);
read = new HfstTransducer(instream);
instream.close();
} catch(HfstException) {
std::string ermsg =
std::string("Couldn't read transducer from ") +
- std::string($1);
- free($1);
+ filepath;
pmatcherror(ermsg.c_str());
}
if (read->get_type() != hfst::pmatch::format) {
read->convert(hfst::pmatch::format);
}
$$ = new PmatchTransducerContainer(read);
- free($1);
} | READ_TEXT {
- $$ = new PmatchTransducerContainer(hfst::pmatch::read_text($1));
+ std::string filepath = hfst::pmatch::path_from_filename($1);
free($1);
+ $$ = new PmatchTransducerContainer(hfst::pmatch::read_text(filepath));
} | READ_SPACED {
- FILE * f = NULL;
- f = fopen($1, "r");
- if (f == NULL) {
- pmatcherror("File cannot be opened.\n");
- } else {
- HfstTokenizer tok;
- HfstBasicTransducer tmp;
- char line [1000];
- while( fgets(line, 1000, f) != NULL )
- {
- hfst::pmatch::strip_newline(line);
- StringPairVector spv = HfstTokenizer::tokenize_space_separated(line);
- tmp.disjunct(spv, 0);
- }
- fclose(f);
- HfstTransducer * t = new HfstTransducer(tmp, hfst::pmatch::format);
- t->minimize();
- $$ = new PmatchTransducerContainer(t);
- }
+ std::string filepath = hfst::pmatch::path_from_filename($1);
+ free($1);
+ $$ = new PmatchTransducerContainer(hfst::pmatch::read_spaced_text(filepath));
} | READ_PROLOG {
+ std::string filepath = hfst::pmatch::path_from_filename($1);
+ free($1);
FILE * f = NULL;
- f = fopen($1, "r");
+ f = hfst::hfst_fopen(filepath.c_str(), "r");
if (f == NULL) {
pmatcherror("File cannot be opened.\n");
} else {
@@ -551,12 +664,15 @@ READ_FROM: READ_BIN {
}
}
} | READ_LEXC {
- $$ = new PmatchTransducerContainer(hfst::HfstTransducer::read_lexc_ptr($1, format, hfst::pmatch::verbose));
+ std::string filepath = hfst::pmatch::path_from_filename($1);
free($1);
+ $$ = new PmatchTransducerContainer(hfst::HfstTransducer::read_lexc_ptr(filepath, format, hfst::pmatch::verbose));
} | READ_RE {
+ std::string filepath = hfst::pmatch::path_from_filename($1);
+ free($1);
std::string regex;
std::string tmp;
- std::ifstream regexfile($1);
+ std::ifstream regexfile(filepath);
if (regexfile.is_open()) {
while (getline(regexfile, tmp)) {
regex.append(tmp);
@@ -564,7 +680,7 @@ READ_FROM: READ_BIN {
}
if (regex.size() == 0) {
std::stringstream err;
- err << "Failed to read regex from " << $1 << ".\n";
+ err << "Failed to read regex from " << filepath << ".\n";
pmatcherror(err.str().c_str());
}
hfst::xre::XreCompiler xre_compiler;
@@ -605,7 +721,7 @@ PMATCH_AND_CONTEXT: AND_LEFT PMATCH_CONTEXTS RIGHT_PARENTHESIS
$$ = NULL;
for (std::vector<PmatchObject *>::reverse_iterator it = $2->rbegin();
it != $2->rend(); ++it) {
- if ($$ = NULL) {
+ if ($$ == NULL) {
$$ = *it;
} else {
PmatchObject * tmp = $$;
@@ -625,38 +741,23 @@ PMATCH_CONTEXT COMMA PMATCH_CONTEXTS {
};
PMATCH_RIGHT_CONTEXT: RC_LEFT EXPRESSION2 RIGHT_PARENTHESIS {
- $$ = new PmatchBinaryOperation(Concatenate, make_rc_entry(),
- new PmatchBinaryOperation(
- Concatenate, $2, make_rc_exit()));
+ $2->mark_context_children();
+ $$ = new PmatchUnaryOperation(RC, $2);
};
PMATCH_NEGATIVE_RIGHT_CONTEXT: NRC_LEFT EXPRESSION2 RIGHT_PARENTHESIS {
- $$ = new PmatchBinaryOperation(
- Concatenate, make_minimization_guard(),
- new PmatchBinaryOperation(
- Disjunct, make_passthrough(),
- new PmatchBinaryOperation(
- Concatenate, make_nrc_entry(),
- new PmatchBinaryOperation(Concatenate, $2, make_nrc_exit()))));
+ $2->mark_context_children();
+ $$ = new PmatchUnaryOperation(NRC, $2);
};
PMATCH_LEFT_CONTEXT: LC_LEFT EXPRESSION2 RIGHT_PARENTHESIS {
- $$ = new PmatchBinaryOperation(
- Concatenate, make_lc_entry(),
- new PmatchBinaryOperation(
- Concatenate, new PmatchUnaryOperation(
- Reverse, $2), make_lc_exit()));
+ $2->mark_context_children();
+ $$ = new PmatchUnaryOperation(LC, $2);
};
PMATCH_NEGATIVE_LEFT_CONTEXT: NLC_LEFT EXPRESSION2 RIGHT_PARENTHESIS {
- $$ = new PmatchBinaryOperation(
- Concatenate, make_minimization_guard(),
- new PmatchBinaryOperation(
- Disjunct, make_passthrough(),
- new PmatchBinaryOperation(
- Concatenate, make_nlc_entry(),
- new PmatchBinaryOperation(Concatenate, new PmatchUnaryOperation(
- Reverse, $2), make_nlc_exit()))));
+ $2->mark_context_children();
+ $$ = new PmatchUnaryOperation(NLC, $2);
};
@@ -677,7 +778,7 @@ PMATCH_NEGATIVE_LEFT_CONTEXT: NLC_LEFT EXPRESSION2 RIGHT_PARENTHESIS {
// caller_args[callee_args[i]] = new HfstTransducer(it->at(i), tok, hfst::pmatch::format);
// }
// $$->disjunct(*hfst::pmatch::functions[$2].evaluate(caller_args));
-// // Clean up the string transducers we allocated each time
+// // Clean up the string transducers we allocated each time
// for (std::map<std::string, HfstTransducer *>::iterator it = caller_args.begin();
// it != caller_args.end(); ++it) {
// delete it->second;
diff --git a/libhfst/src/parsers/pmatch_utils.cc b/libhfst/src/parsers/pmatch_utils.cc
index 084fece..36d780d 100644
--- a/libhfst/src/parsers/pmatch_utils.cc
+++ b/libhfst/src/parsers/pmatch_utils.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
/**
@@ -18,8 +18,10 @@
#include <cstring>
#include <sstream>
-#include "pmatch_utils.h"
#include "HfstTransducer.h"
+#include "HfstExceptionDefs.h"
+
+#include "pmatch_utils.h"
//#include "tools/src/HfstUtf8.h"
#include "implementations/optimized-lookup/pmatch.h"
@@ -34,21 +36,31 @@ extern int pmatchnerrs;
int
pmatcherror(const char *msg)
{
- fprintf(stderr, "*** pmatch parsing failed: %s\n", msg);
+ std::string parsedata;
if (strlen(hfst::pmatch::data) < 60)
{
- fprintf(stderr, "*** parsing %s [line %d, near %s]\n",
- hfst::pmatch::data, pmatchlineno, pmatchtext);
+ parsedata = hfst::pmatch::data;
}
else
{
- char buf[200];
- memcpy(buf, hfst::pmatch::data, 200 - 1);
- buf[200 - 1] = '\0';
- fprintf(stderr, "*** parsing %s [line %d, near %s]...\n",
- buf, pmatchlineno, pmatchtext);
- }
- exit(0);
+ parsedata = std::string(hfst::pmatch::data, 59) + "... [truncated]";
+ }
+ std::string errmsg = "pmatch parsing failed: ";
+ errmsg.append(msg);
+ errmsg.append("\n*** parsing ");
+ errmsg.append(parsedata);
+ errmsg.append(" at line ");
+ std::ostringstream ss;
+ ss << pmatchlineno;
+ errmsg.append(ss.str());
+ errmsg.append(" near ");
+ errmsg.append(pmatchtext);
+ errmsg.append("\n");
+
+ // TODO: clean the potentially large amounts of data we're leaking in case
+ // the caller isn't the command line utility that exits after this
+
+ HFST_THROW_MESSAGE(HfstException, errmsg);
}
void pmatchwarning(const char *msg)
@@ -58,28 +70,31 @@ void pmatchwarning(const char *msg)
}
}
-namespace hfst
-{
-namespace pmatch
+namespace hfst
+{
+namespace pmatch
{
char* data;
std::map<std::string, hfst::pmatch::PmatchObject*> definitions;
+std::map<std::string, std::string> variables;
std::vector<std::map<std::string, PmatchObject*> > call_stack;
std::map<std::string, PmatchObject*> def_insed_expressions;
std::set<std::string> inserted_names;
std::set<std::string> unsatisfied_insertions;
std::set<std::string> used_definitions;
std::set<std::string> function_names;
+std::vector<WordVector> word_vectors;
char* startptr;
hfst::ImplementationType format;
size_t len;
bool verbose;
bool flatten;
+std::string includedir;
clock_t timer;
-clock_t tmp_timer;
int minimization_guard_count;
bool need_delimiters;
+WordVecFloat vector_similarity_projection_factor;
std::map<std::string, hfst::HfstTransducer> named_transducers;
PmatchUtilityTransducers* utils=NULL;
@@ -92,9 +107,9 @@ void warn(std::string warning)
PmatchUtilityTransducers*
get_utils()
{
- if (utils == NULL)
+ if (utils == NULL)
{
- utils = new PmatchUtilityTransducers();
+ utils = new PmatchUtilityTransducers();
}
return utils;
}
@@ -129,11 +144,6 @@ bool symbol_in_local_context(std::string & sym)
return call_stack.back().count(sym) != 0;
}
-bool should_use_cache(void)
-{
- return call_stack.size() == 0;
-}
-
PmatchObject * symbol_from_global_context(std::string & sym)
{
if (symbol_in_global_context(sym)) {
@@ -179,8 +189,8 @@ int
getinput(char *buf, int maxlen)
{
int retval = 0;
- if ( maxlen > len ) {
- maxlen = len;
+ if ( maxlen > (int)len ) {
+ maxlen = hfst::size_t_to_int(len);
}
memcpy(buf, data, maxlen);
data += maxlen;
@@ -294,6 +304,246 @@ HfstTransducer * add_pmatch_delimiters(HfstTransducer * regex)
PmatchTransducerContainer * make_end_tag(std::string tag)
{ return epsilon_to_symbol_container("@PMATCH_ENDTAG_" + tag + "@"); }
+struct DotProductWithWordVectorComparison {
+ WordVector compare_with_this;
+ DotProductWithWordVectorComparison(WordVector word): compare_with_this(word) {}
+ // if the vectors are normalized, dot product is == cosine similarity
+ bool operator()(WordVector left, WordVector right) {
+ WordVecFloat left_accumulator = 0;
+ WordVecFloat right_accumulator = 0;
+ for (size_t i = 0; i < compare_with_this.vector.size(); ++i) {
+ left_accumulator += compare_with_this.vector[i]*left.vector[i];
+ right_accumulator += compare_with_this.vector[i]*right.vector[i];
+ }
+ return left_accumulator > right_accumulator;
+ }
+};
+
+struct CosineSimilarityWithWordVectorComparison {
+ WordVector compare_with_this;
+ CosineSimilarityWithWordVectorComparison(WordVector word): compare_with_this(word) {}
+ bool operator()(WordVector left, WordVector right) {
+ WordVecFloat left_accumulator = 0.0;
+ WordVecFloat right_accumulator = 0.0;
+ for (size_t i = 0; i < compare_with_this.vector.size(); ++i) {
+ left_accumulator += compare_with_this.vector[i]*left.vector[i];
+ right_accumulator += compare_with_this.vector[i]*right.vector[i];
+ }
+ left_accumulator /= left.norm;
+ right_accumulator /= right.norm;
+ return left_accumulator > right_accumulator;
+ }
+};
+
+std::vector<WordVecFloat> get_projected_vector(std::vector<WordVecFloat> vec,
+ std::vector<WordVecFloat> plane_vec,
+ WordVecFloat translation_term)
+{
+ return pointwise_plus(vec,
+ pointwise_multiplication(
+ (((translation_term - dot_product(vec, plane_vec)) / square_sum(plane_vec))
+ * vector_similarity_projection_factor),
+ plane_vec));
+}
+
+struct CosineSimilarityProjectedToPlaneComparison {
+ std::vector<WordVecFloat> plane_vec;
+ std::vector<WordVecFloat> comparison_point;
+ WordVecFloat translation_term;
+ WordVecFloat plane_vec_square_sum;
+ CosineSimilarityProjectedToPlaneComparison(
+ std::vector<WordVecFloat> plane_vec_, std::vector<WordVecFloat> comparison_point_, WordVecFloat translation_term_):
+ plane_vec(plane_vec_), comparison_point(comparison_point_), translation_term(translation_term_)
+ {
+ plane_vec_square_sum = square_sum(plane_vec_);
+ }
+ bool operator()(WordVector left, WordVector right) {
+ /*
+ * First, given a plane "plane_vec = translation term" and a point,
+ * find the multiple of plane_vec which produces a vector going
+ * from point to the nearest point in the plane.
+ */
+ WordVecFloat left_scaler = (translation_term - dot_product(left.vector, plane_vec)) / plane_vec_square_sum;
+ WordVecFloat right_scaler = (translation_term - dot_product(right.vector, plane_vec)) / plane_vec_square_sum;
+ left_scaler *= vector_similarity_projection_factor;
+ right_scaler *= vector_similarity_projection_factor;
+ std::vector<WordVecFloat> new_left = pointwise_plus(left.vector, pointwise_multiplication(left_scaler, plane_vec));
+ std::vector<WordVecFloat> new_right = pointwise_plus(right.vector, pointwise_multiplication(right_scaler, plane_vec));
+ // Then calculate cosine similarity
+ WordVecFloat left_accumulator = 0.0;
+ WordVecFloat right_accumulator = 0.0;
+ for (size_t i = 0; i < comparison_point.size(); ++i) {
+ left_accumulator += comparison_point[i]*new_left[i];
+ right_accumulator += comparison_point[i]*new_right[i];
+ }
+ left_accumulator /= norm(new_left);
+ right_accumulator /= norm(new_right);
+ return left_accumulator > right_accumulator;
+ }
+};
+
+template<typename T> std::vector<T> pointwise_minus(std::vector<T> l,
+ std::vector<T> r)
+{
+ std::vector<T> ret(l.size(), 0);
+ for(size_t i = 0; i < l.size(); ++i) {
+ ret[i] = l[i] - r[i];
+ }
+ return ret;
+}
+
+template<typename T> std::vector<T> pointwise_plus(std::vector<T> l,
+ std::vector<T> r)
+{
+ std::vector<T> ret(l.size(), 0);
+ for(size_t i = 0; i < l.size(); ++i) {
+ ret[i] = l[i] + r[i];
+ }
+ return ret;
+}
+
+template<typename T> std::vector<T> pointwise_multiplication(T l,
+ std::vector<T> r)
+{
+ std::vector<T> ret(r.size(), 0);
+ for(size_t i = 0; i < r.size(); ++i) {
+ ret[i] = l * r[i];
+ }
+ return ret;
+}
+
+template<typename T> T dot_product(std::vector<T> l,
+ std::vector<T> r)
+{
+ T ret = 0;
+ for(size_t i = 0; i < l.size(); ++i) {
+ ret += l[i] * r[i];
+ }
+ return ret;
+}
+
+template<typename T> T square_sum(std::vector<T> v)
+{
+ T ret = 0;
+ for(size_t i = 0; i < v.size(); ++i) {
+ ret += v[i] * v[i];
+ }
+ return ret;
+}
+
+template<typename T> T norm(std::vector<T> v)
+{
+ return sqrt(square_sum(v));
+}
+
+WordVecFloat cosine_distance(WordVector left, WordVector right)
+{
+ // Sometimes very nearby vectors combined with rounding error will produce
+ // a slightly negative distance, so make sure to return at least 0.0
+ WordVecFloat retval = 1.0 - dot_product(left.vector, right.vector) /
+ (left.norm * right.norm);
+ if (retval < 0.0) {
+ return 0.0;
+ }
+ return retval;
+}
+
+WordVecFloat cosine_distance(std::vector<WordVecFloat> left, std::vector<WordVecFloat> right)
+{
+ WordVecFloat retval = 1.0 - dot_product(left, right) / (norm(left) * norm(right));
+ if (retval < 0.0) {
+ return 0.0;
+ }
+ return retval;
+}
+
+PmatchObject * compile_like_arc(std::string word1, std::string word2,
+ unsigned int nwords)
+{
+ WordVector this_word1;
+ WordVector this_word2;
+ for (std::vector<WordVector>::iterator it = word_vectors.begin();
+ (it != word_vectors.end() &&
+ (this_word1.word == "" || this_word2.word == "")); ++it) {
+ if (word1 == it->word) {
+ this_word1 = *it;
+ }
+ if (word2 == it->word) {
+ this_word2 = *it;
+ }
+ }
+ if (this_word1.word == "" && this_word2.word == "") {
+ // got no matches
+ PmatchString * word1_o = new PmatchString(word1);
+ PmatchString * word2_o = new PmatchString(word2);
+ word1_o->multichar = true; word2_o->multichar = true;
+ return new PmatchBinaryOperation(Disjunct, word1_o, word2_o);
+ }
+
+ if (this_word1.word == "" || this_word2.word == "") {
+ // just one match
+ WordVector this_word = (this_word1.word == "" ? this_word2 : this_word1);
+ CosineSimilarityWithWordVectorComparison comparison_object(this_word);
+ std::sort(word_vectors.begin(), word_vectors.end(), comparison_object);
+ HfstTokenizer tok;
+ HfstTransducer * retval = new HfstTransducer(format);
+ for (size_t i = 0; i < word_vectors.size() && i <= nwords; ++i) {
+ HfstTransducer tmp(word_vectors[i].word, tok, format);
+ tmp.set_final_weights(cosine_distance(word_vectors[i], this_word));
+ retval->disjunct(tmp);
+ }
+ return new PmatchTransducerContainer(retval);
+ }
+
+ // the general case
+ if(variables["vector-similarity-projection-factor"] != "1.0") {
+ vector_similarity_projection_factor =
+ strtod(variables["vector-similarity-projection-factor"].c_str(), NULL);
+ }
+ /*
+ * When there are two vectors A and B, we compute the vector A - B that
+ * goes from one to the other, and define a hyperplane orthogonal to that
+ * vector that intersects the vector at the midpoint between the
+ * two. We then add to all vectors a multiple of A - B to move them closer
+ * to the plane, reducing the distance that is due to the difference
+ * between A and B. (This is like projecting the space to the hyperplane
+ * if we go all the way to the plane)
+ *
+ * The hyperplane is defined by the equation |B - A| = d, where d is a
+ * translation term. |B - A| = 0 would be the set of vectors orthogonal to
+ * |B - A|. We set d so that the distance from the hyperplane to A is
+ * half of the norm of |B - A|.
+ */
+
+ std::vector<WordVecFloat> B_minus_A = pointwise_minus(
+ this_word1.vector, this_word2.vector);
+ std::vector<WordVecFloat> halfway_point = pointwise_plus(
+ this_word2.vector, pointwise_multiplication(
+ static_cast<WordVecFloat>(0.5), B_minus_A));
+ WordVecFloat hyperplane_translation_term = dot_product(B_minus_A, this_word1.vector)
+ - square_sum(B_minus_A) * 0.5;
+ CosineSimilarityProjectedToPlaneComparison comparison_object(
+ B_minus_A, halfway_point, hyperplane_translation_term);
+ std::sort(word_vectors.begin(), word_vectors.end(), comparison_object);
+
+ HfstTokenizer tok;
+ HfstTransducer * retval = new HfstTransducer(format);
+ for (size_t i = 0; i < word_vectors.size() && i <= nwords; ++i) {
+ HfstTransducer tmp(word_vectors[i].word, tok, format);
+ std::vector<WordVecFloat> projected_i = get_projected_vector(
+ word_vectors[i].vector, B_minus_A, hyperplane_translation_term);
+ tmp.set_final_weights(cosine_distance(projected_i, halfway_point));
+ retval->disjunct(tmp);
+ for (size_t j = i + 1; j < word_vectors.size() && j <= nwords; ++j) {
+ HfstTransducer tmp2(word_vectors[i].word + "_cos_" + word_vectors[j].word, tok, format);
+ tmp2.set_final_weights(cosine_distance(projected_i,
+ get_projected_vector(word_vectors[j].vector, B_minus_A, hyperplane_translation_term)));
+ retval->disjunct(tmp2);
+ }
+ }
+ return new PmatchTransducerContainer(retval);
+}
+
PmatchTransducerContainer * make_counter(std::string name)
{ return epsilon_to_symbol_container("@PMATCH_COUNTER_" + name + "@"); }
@@ -611,7 +861,7 @@ std::string codepoint_to_utf8(unsigned int codepoint)
buf[1] = 128 + codepoint / 4096 % 64;
buf[2] = 128 + codepoint / 64 % 64;
buf[3] = 128 + codepoint % 64;
- buf[4] = '\0';
+ buf[4] = '\0';
} else {
u_parse_err = true;
}
@@ -627,7 +877,6 @@ PmatchTransducerContainer * parse_range(const char * s)
char * quoted = get_delimited(s, '"');
char * orig_quoted = quoted;
char ** c = & quoted;
- unsigned char bytes_in_char;
HfstTransducer * retval = new HfstTransducer(format);
while (**c != '\0') {
unsigned int codepoint1 = 0;
@@ -695,7 +944,7 @@ get_weight(const char *s)
{
double rv = -3.1415;
const char* weightstart = s;
- while ((*weightstart != '\0') &&
+ while ((*weightstart != '\0') &&
((*weightstart == ' ') || (*weightstart == '\t') ||
(*weightstart == ';')))
{
@@ -710,6 +959,17 @@ get_weight(const char *s)
void init_globals(void)
{
definitions.clear();
+ variables.clear();
+ variables["count-patterns"] = "off";
+ variables["delete-patterns"] = "off";
+ variables["extract-patterns"] = "off";
+ variables["locate-patterns"] = "off";
+ variables["mark-patterns"] = "on";
+ variables["max-context-length"] = "254";
+ variables["max-recursion"] = "5000";
+ variables["need-separators"] = "on";
+ variables["vector-similarity-projection-factor"] = "1.0";
+ call_stack.clear();
def_insed_expressions.clear();
inserted_names.clear();
unsatisfied_insertions.clear();
@@ -717,11 +977,13 @@ void init_globals(void)
function_names.clear();
zero_minimization_guard();
need_delimiters = false;
+ pmatchnerrs = 0;
}
std::map<std::string, HfstTransducer*>
compile(const string& pmatch, map<string,HfstTransducer*>& defs,
- ImplementationType impl, bool be_verbose, bool do_flatten)
+ ImplementationType impl, bool be_verbose, bool do_flatten,
+ std::string includedir_)
{
// lock here?
init_globals();
@@ -730,6 +992,8 @@ compile(const string& pmatch, map<string,HfstTransducer*>& defs,
len = strlen(data);
verbose = be_verbose;
flatten = do_flatten;
+ includedir = includedir_;
+ vector_similarity_projection_factor = 1.0;
for (map<string, HfstTransducer*>::iterator it = defs.begin();
it != defs.end(); ++it) {
definitions[it->first] = new PmatchTransducerContainer(it->second);
@@ -783,6 +1047,7 @@ compile(const string& pmatch, map<string,HfstTransducer*>& defs,
if (defs_it->first.compare("TOP") == 0 ||
inserted_names.count(defs_it->first) != 0) {
HfstTransducer * tmp = defs_it->second->evaluate();
+ tmp->minimize();
dummy.harmonize(*tmp);
retval[defs_it->first] = tmp;
}
@@ -803,6 +1068,7 @@ compile(const string& pmatch, map<string,HfstTransducer*>& defs,
std::cerr << "Pmatch compilation warning: regex or TOP was undefined, using ";
std::cerr << definitions.begin()->first << " as root\n";
hfst::HfstTransducer * tmp = definitions.begin()->second->evaluate();
+ tmp->minimize();
retval.insert(std::pair<std::string, hfst::HfstTransducer*>("TOP", tmp));
} else {
hfst::HfstTransducer * tmp = definitions["TOP"]->evaluate();
@@ -816,18 +1082,51 @@ compile(const string& pmatch, map<string,HfstTransducer*>& defs,
hfst::pmatch::timer = clock();
std::cerr << "compiled and harmonized in " << duration << " seconds\n";
}
+ if (variables["need-separators"] == "on") {
+ HfstTransducer not_whitespace(hfst::internal_identity, format);
+ not_whitespace.subtract(*(get_utils()->latin1_whitespace_acceptor));
+ HfstTransducer anything(hfst::internal_identity, format);
+ anything.repeat_star();
+ HfstTransducer begins_and_ends_with_non_whitespace(not_whitespace);
+ begins_and_ends_with_non_whitespace.concatenate(anything);
+ begins_and_ends_with_non_whitespace.concatenate(not_whitespace);
+ begins_and_ends_with_non_whitespace.compose(*(retval["TOP"]));
+ HfstTransducer empty(format);
+ if (begins_and_ends_with_non_whitespace.compare(empty) == false) {
+ HfstTransducer whitespace_punct_context(*(get_utils()->latin1_whitespace_acceptor));
+ whitespace_punct_context.disjunct(*(get_utils()->latin1_punct_acceptor));
+ whitespace_punct_context.disjunct(HfstTransducer("@BOUNDARY@", format));
+ HfstTransducer * top_with_boundaries = new HfstTransducer(hfst::internal_epsilon, LC_ENTRY_SYMBOL, format);
+ top_with_boundaries->concatenate(whitespace_punct_context);
+ top_with_boundaries->concatenate(HfstTransducer(hfst::internal_epsilon, LC_EXIT_SYMBOL, format));
+ HfstTransducer RC(hfst::internal_epsilon, RC_ENTRY_SYMBOL, format);
+ RC.concatenate(whitespace_punct_context);
+ RC.concatenate(HfstTransducer(hfst::internal_epsilon, RC_EXIT_SYMBOL, format));
+ top_with_boundaries->concatenate(*(retval["TOP"]));
+ top_with_boundaries->concatenate(RC);
+ delete retval["TOP"];
+ retval["TOP"] = add_pmatch_delimiters(top_with_boundaries);
+ }
+ }
+ for(std::map<std::string, std::string>::iterator it = variables.begin();
+ it != variables.end(); ++it) {
+ retval["TOP"]->set_property(it->first, it->second);
+ }
return retval;
}
void print_size_info(HfstTransducer * net)
{
+ if (!hfst::pmatch::verbose) {
+ return;
+ }
HfstBasicTransducer tmp(*net);
size_t states = 0;
size_t arcs = 0;
for(HfstBasicTransducer::const_iterator state_it = tmp.begin();
state_it != tmp.end(); ++state_it) {
++states;
- for(HfstBasicTransducer::HfstTransitions::const_iterator tr_it =
+ for(hfst::implementations::HfstBasicTransitions::const_iterator tr_it =
state_it->begin(); tr_it != state_it->end(); ++tr_it) {
++arcs;
}
@@ -836,7 +1135,8 @@ void print_size_info(HfstTransducer * net)
" states and " << arcs << " arcs" << std::endl;
}
-HfstTransducer * read_text(char * filename, ImplementationType type)
+HfstTransducer * read_text(std::string filename, ImplementationType type,
+ bool spaced_text)
{
std::ifstream infile;
std::string line;
@@ -852,8 +1152,12 @@ HfstTransducer * read_text(char * filename, ImplementationType type)
std::getline(infile, line);
if(!line.empty()) {
++n;
- StringPairVector spv = tok.tokenize(line);
- retval->disjunct(spv);
+ if (spaced_text) {
+ StringPairVector spv = tok.tokenize_space_separated(line);
+ } else {
+ StringPairVector spv = tok.tokenize(line);
+ retval->disjunct(spv);
+ }
}
}
}
@@ -861,6 +1165,87 @@ HfstTransducer * read_text(char * filename, ImplementationType type)
return retval;
}
+HfstTransducer * read_spaced_text(std::string filename, ImplementationType type)
+{ return read_text(filename, type, true); }
+
+std::string path_from_filename(char * filename)
+{
+ std::string retval(filename);
+ if (includedir.size() > 0 && retval.size() > 0) {
+ // includedir won't be > 0 under Windows until this mechanism is ported
+ if (retval[0] != '/') {
+ // not an absolute dir
+ retval.insert(0, includedir);
+ }
+ }
+ return retval;
+}
+
+void read_vec(std::string filename)
+{
+ if (word_vectors.size() != 0) {
+ word_vectors.clear();
+ std::cerr << "pmatch: vector model file " << filename
+ << " overrides earlier one\n";
+ }
+ std::ifstream infile;
+ std::string line;
+ size_t linenumber = 0;
+ char separator = '\t';
+ infile.open(filename);
+ if(!infile.good()) {
+ std::cerr << "pmatch: could not open vector file " << filename <<
+ " for reading\n";
+ return;
+ } else {
+ while(infile.good()) {
+ std::getline(infile, line);
+ ++linenumber;
+ if (linenumber == 1) { continue; } // first line is metadata
+ if (line.empty()) { continue; }
+ size_t pos = line.find(separator);
+ if (pos == std::string::npos) {
+ separator = ' ';
+ pos = line.find(separator);
+ if (pos == std::string::npos) {
+ std::cerr << "pmatch warning: vector file " << filename <<
+ " doesn't appear to be tab- or space-separated\n (reading line " << linenumber << ")\n";
+ break;
+ }
+ }
+ std::string word = line.substr(0, pos);
+ std::vector<WordVecFloat> components;
+ size_t nextpos;
+ while (std::string::npos != (nextpos = line.find(separator, pos + 1))) {
+ components.push_back(strtod(line.substr(pos + 1, nextpos - pos).c_str(), NULL));
+ pos = nextpos;
+ }
+ // there can be one more from pos to the newline if there isn't a
+ // separator at the end
+ if (line.back() != separator) {
+ components.push_back(strtof(line.substr(pos + 1).c_str(), NULL));
+ }
+ if (word_vectors.size() != 0 && word_vectors[0].vector.size() != components.size()) {
+ std::cerr << "pmatch warning: vector file " << filename <<
+ " appears malformed\n (reading line " << linenumber << ")\n";
+ continue;
+ }
+ WordVector wv;
+ wv.word = word;
+ wv.vector = components;
+ wv.norm = norm(components);
+ word_vectors.push_back(wv);
+ }
+ }
+ infile.close();
+ if (verbose) {
+ if (word_vectors.size() == 0) {
+ std::cerr << "Tried to read word vector file, empty result\n";
+ }
+ std::cerr << "Read " << word_vectors.size() << " vectors of dimensionality " << word_vectors[0].vector.size() << std::endl;
+ }
+}
+
std::vector<std::vector<std::string> > read_args(char * filename, unsigned int argcount)
{
std::ifstream infile;
@@ -882,7 +1267,7 @@ std::vector<std::vector<std::string> > read_args(char * filename, unsigned int a
int nextpos = -1;
do {
curpos = nextpos + 1;
- nextpos = line.find_first_of(" ", curpos);
+ nextpos = hfst::size_t_to_int(line.find_first_of(" ", curpos));
current_tokens.push_back(line.substr(curpos, nextpos - curpos));
} while (nextpos != std::string::npos);
if (current_tokens.size() != argcount) {
@@ -978,8 +1363,8 @@ HfstTransducer * PmatchUtilityTransducers::make_latin1_numeral_acceptor(Implemen
++it) {
retval->disjunct(HfstTransducer(std::string(1, *it), type));
}
+ //retval->minimize(); ?
return retval;
- retval->minimize();
}
HfstTransducer * PmatchUtilityTransducers::make_latin1_punct_acceptor(ImplementationType type)
@@ -1022,100 +1407,135 @@ HfstTransducer * PmatchUtilityTransducers::make_lowerfy(ImplementationType type)
return retval;
}
-HfstTransducer * PmatchUtilityTransducers::cap(HfstTransducer & t)
+HfstTransducer * PmatchUtilityTransducers::cap(HfstTransducer & t, Side side, bool optional)
{
- HfstTokenizer tok;
+ HfstTransducer * retval = NULL;
HfstTransducer cap(*capify);
+ HfstTransducer decap(cap);
+ decap.invert();
HfstTransducer anything(HfstTransducer::identity_pair(t.get_type()));
- HfstTransducer anything_but_whitespace(anything);
- anything_but_whitespace.subtract(*latin1_whitespace_acceptor);
- HfstTransducer anything_but_lowercase(anything);
- anything_but_lowercase.subtract(*latin1_lowercase_acceptor);
- cap.disjunct(anything_but_lowercase);
- HfstTransducer cap_one_word(cap);
- cap_one_word.concatenate(anything_but_whitespace.repeat_star());
- /* If we consider to cross the word boundary */
- HfstTransducer more_words(*latin1_whitespace_acceptor);
- more_words.concatenate(cap_one_word);
- more_words.repeat_star();
- HfstTransducer * retval = new HfstTransducer(t);
- retval->compose(cap_one_word.concatenate(more_words));
- retval->output_project();
- retval->minimize();
- return retval;
-}
-
-HfstTransducer * PmatchUtilityTransducers::optcap(HfstTransducer & t)
-{
- HfstTokenizer tok;
- HfstTransducer optcap(*capify);
- HfstTransducer anything(HfstTransducer::identity_pair(t.get_type()));
- HfstTransducer anything_but_whitespace(anything);
- anything_but_whitespace.subtract(*latin1_whitespace_acceptor);
- optcap.disjunct(anything);
- HfstTransducer optcap_one_word(optcap);
- optcap_one_word.concatenate(anything_but_whitespace.repeat_star());
- /* If we consider to cross the word boundary */
- HfstTransducer more_words(*latin1_whitespace_acceptor);
- more_words.concatenate(optcap_one_word);
- more_words.repeat_star();
- HfstTransducer * retval = new HfstTransducer(t);
- retval->compose(optcap_one_word.concatenate(more_words));
- retval->output_project();
- retval->minimize();
- return retval;
-}
-
-HfstTransducer * PmatchUtilityTransducers::tolower(HfstTransducer & t)
-{
- HfstTokenizer tok;
- HfstTransducer lowercase(*lowerfy);
- HfstTransducer any_but_upper(hfst::internal_identity, hfst::pmatch::format);
- any_but_upper.subtract(*latin1_uppercase_acceptor);
- lowercase.disjunct(any_but_upper);
- HfstTransducer * retval = new HfstTransducer(t);
- retval->compose(lowercase.repeat_star());
- retval->output_project();
- retval->minimize();
- return retval;
-}
-
-HfstTransducer * PmatchUtilityTransducers::toupper(HfstTransducer & t)
-{
- HfstTokenizer tok;
- HfstTransducer uppercase(*capify);
- HfstTransducer any_but_lower(hfst::internal_identity, hfst::pmatch::format);
- any_but_lower.subtract(*latin1_lowercase_acceptor);
- uppercase.disjunct(any_but_lower);
- HfstTransducer * retval = new HfstTransducer(t);
- retval->compose(uppercase.repeat_star());
- retval->output_project();
+ HfstTransducer anything_but_whitespace_star(anything);
+ anything_but_whitespace_star.subtract(*latin1_whitespace_acceptor);
+ anything_but_whitespace_star.repeat_star();
+ if (optional == false) {
+ // don't let lowercased first letters through
+ anything.subtract(*latin1_lowercase_acceptor);
+ }
+ // As in the regexp
+ // [[[["A":"a" [[\" "]* (" " "A":"a")]* ] .o. [{ab ad}:{ef eh}].u]] .o.
+ // [{ab ad}:{ef eh}] ] .o. [[{ab ad}:{ef eh}].l] .o.
+ // ["e":"E" [[\" "]+ (" " "e":"E")]*]
+ if (side == Lower) {
+ retval = new HfstTransducer(t);
+ cap.disjunct(anything);
+ // Cap is the first letter to either capitalize or accept if it's not a
+ // lowercase letter
+ HfstTransducer continuation(anything_but_whitespace_star);
+ // continuation is the rest of the first word
+ HfstTransducer more_caps(*latin1_whitespace_acceptor);
+ // more_caps is more words to capitalize
+ more_caps.concatenate(cap);
+ more_caps.optionalize();
+ continuation.concatenate(more_caps);
+ continuation.repeat_star();
+ cap.concatenate(continuation);
+ retval->compose(cap);
+ } else if (side == Upper) {
+ decap.disjunct(anything);
+ HfstTransducer continuation(anything_but_whitespace_star);
+ HfstTransducer more_decaps(*latin1_whitespace_acceptor);
+ more_decaps.concatenate(decap);
+ more_decaps.optionalize();
+ continuation.concatenate(more_decaps);
+ continuation.repeat_star();
+ retval = new HfstTransducer(decap);
+ retval->concatenate(continuation);
+ retval->compose(t);
+ } else { // both
+ decap.disjunct(anything);
+ HfstTransducer continuation(anything_but_whitespace_star);
+ HfstTransducer more_decaps(*latin1_whitespace_acceptor);
+ more_decaps.concatenate(decap);
+ more_decaps.optionalize();
+ continuation.concatenate(more_decaps);
+ continuation.repeat_star();
+ retval = new HfstTransducer(decap);
+ retval->concatenate(continuation);
+ retval->compose(t);
+ HfstTransducer continuation2(anything_but_whitespace_star);
+ HfstTransducer more_caps(*latin1_whitespace_acceptor);
+ cap.disjunct(anything);
+ more_caps.concatenate(cap);
+ more_caps.optionalize();
+ continuation2.concatenate(more_caps);
+ continuation2.repeat_star();
+ cap.concatenate(continuation2);
+ retval->compose(cap);
+ }
retval->minimize();
return retval;
}
-HfstTransducer * PmatchUtilityTransducers::opt_tolower(HfstTransducer & t)
+HfstTransducer * PmatchUtilityTransducers::tolower(HfstTransducer & t, Side side, bool optional)
{
- HfstTokenizer tok;
- HfstTransducer lowercase(*lowerfy);
HfstTransducer anything(hfst::internal_identity, hfst::pmatch::format);
- lowercase.disjunct(anything);
- HfstTransducer * retval = new HfstTransducer(t);
- retval->compose(lowercase.repeat_star());
- retval->output_project();
+ if (optional == false) {
+ anything.subtract(*latin1_uppercase_acceptor);
+ }
+ HfstTransducer * retval = NULL;
+ if (side == Lower) {
+ HfstTransducer lowercase(*lowerfy);
+ lowercase.disjunct(anything);
+ lowercase.repeat_star();
+ retval = new HfstTransducer(t);
+ retval->compose(lowercase);
+ } else if (side == Upper) {
+ retval = new HfstTransducer(*capify);
+ retval->disjunct(anything);
+ retval->repeat_star();
+ retval->compose(t);
+ } else { // both
+ retval = new HfstTransducer(*capify);
+ retval->disjunct(anything);
+ retval->repeat_star();
+ retval->compose(t);
+ HfstTransducer lowercase(*lowerfy);
+ lowercase.disjunct(anything);
+ lowercase.repeat_star();
+ retval->compose(lowercase);
+ }
retval->minimize();
return retval;
}
-HfstTransducer * PmatchUtilityTransducers::opt_toupper(HfstTransducer & t)
+HfstTransducer * PmatchUtilityTransducers::toupper(HfstTransducer & t, Side side, bool optional)
{
- HfstTokenizer tok;
- HfstTransducer uppercase(*capify);
HfstTransducer anything(hfst::internal_identity, hfst::pmatch::format);
- uppercase.disjunct(anything);
- HfstTransducer * retval = new HfstTransducer(t);
- retval->compose(uppercase.repeat_star());
- retval->output_project();
+ if (optional == false) {
+ anything.subtract(*latin1_lowercase_acceptor);
+ }
+ HfstTransducer * retval = NULL;
+ if (side == Lower) {
+ HfstTransducer uppercase(*capify);
+ uppercase.disjunct(anything);
+ uppercase.repeat_star();
+ retval = new HfstTransducer(t);
+ retval->compose(uppercase);
+ } else if (side == Upper) {
+ retval = new HfstTransducer(*lowerfy);
+ retval->disjunct(anything);
+ retval->repeat_star();
+ retval->compose(t);
+ } else { // both
+ retval = new HfstTransducer(*lowerfy);
+ retval->disjunct(anything);
+ retval->repeat_star();
+ retval->compose(t);
+ HfstTransducer uppercase(*capify);
+ uppercase.disjunct(anything);
+ uppercase.repeat_star();
+ retval->compose(uppercase);
+ }
retval->minimize();
return retval;
}
@@ -1126,6 +1546,7 @@ PmatchObject::PmatchObject(void)
weight = 0.0;
line_defined = pmatchlineno;
cache = (HfstTransducer*) (NULL);
+ parent_is_context = false;
}
HfstTransducer * PmatchObject::evaluate(std::vector<PmatchObject *> args)
@@ -1141,8 +1562,9 @@ HfstTransducer * PmatchObject::evaluate(std::vector<PmatchObject *> args)
} else {
start_timing();
HfstTransducer * retval = evaluate();
+ retval->minimize();
report_time();
- return new HfstTransducer(*retval);
+ return retval;
}
} else {
std::stringstream errstring;
@@ -1154,7 +1576,7 @@ HfstTransducer * PmatchObject::evaluate(std::vector<PmatchObject *> args)
HfstTransducer * PmatchSymbol::evaluate(PmatchEvalType eval_type)
{
start_timing();
- HfstTransducer * retval;
+ HfstTransducer * retval = NULL;
if (symbol_in_local_context(sym)) {
retval = symbol_from_local_context(sym)->evaluate();
} else if (symbol_in_global_context(sym)) {
@@ -1171,7 +1593,8 @@ HfstTransducer * PmatchSymbol::evaluate(PmatchEvalType eval_type)
}
retval = new HfstTransducer(sym, format);
}
- retval->set_final_weights(weight, true);
+ retval->set_final_weights(hfst::double_to_float(weight), true);
+ retval->minimize();
report_time();
return retval;
}
@@ -1188,18 +1611,21 @@ HfstTransducer * PmatchString::evaluate(PmatchEvalType eval_type) {
} else {
tmp = new HfstTransducer(string, format);
}
- tmp->set_final_weights(weight, true);
- if (cache == NULL && should_use_cache()) {
+ tmp->set_final_weights(hfst::double_to_float(weight), true);
+ if (cache == NULL && should_use_cache() == true) {
cache = tmp;
+ cache->minimize();
+ report_time();
+ return new HfstTransducer(*cache);
}
- report_time();
- return new HfstTransducer(*tmp);
+ report_time();
+ return tmp;
}
HfstTransducer * PmatchFunction::evaluate(std::vector<PmatchObject *> funargs)
{
if (verbose) {
- tmp_timer = clock();
+ my_timer = clock();
}
if (funargs.size() != args.size()) {
std::stringstream errstring;
@@ -1210,15 +1636,15 @@ HfstTransducer * PmatchFunction::evaluate(std::vector<PmatchObject *> funargs)
if (call_stack.size() != 0) {
local_env = call_stack.back();
};
- for (int i = 0; i < args.size(); ++i) {
+ for (int i = 0; i < (int)args.size(); ++i) {
local_env[args[i]] = funargs[i];
}
call_stack.push_back(local_env);
HfstTransducer * retval = root->evaluate();
- retval->set_final_weights(weight, true);
+ retval->set_final_weights(hfst::double_to_float(weight), true);
call_stack.pop_back();
if (verbose) {
- double duration = (clock() - tmp_timer) /
+ double duration = (clock() - my_timer) /
(double) CLOCKS_PER_SEC;
std::cerr << "Call to " << name << " evaluated in " << duration << " seconds\n";
}
@@ -1234,7 +1660,7 @@ HfstTransducer * PmatchFunction::evaluate(PmatchEvalType eval_type)
HfstTransducer * PmatchBuiltinFunction::evaluate(PmatchEvalType eval_type)
{
start_timing();
- HfstTransducer * retval;
+ HfstTransducer * retval = NULL;
if (type == Interpolate) {
if (args->size() < 3) {
std::stringstream errstring;
@@ -1253,7 +1679,7 @@ HfstTransducer * PmatchBuiltinFunction::evaluate(PmatchEvalType eval_type)
}
delete interpolator;
}
- retval->set_final_weights(weight, true);
+ retval->set_final_weights(hfst::double_to_float(weight), true);
report_time();
return retval;
}
@@ -1275,12 +1701,15 @@ HfstTransducer * PmatchNumericOperation::evaluate(PmatchEvalType eval_type)
} else if (op == RepeatNToK) {
tmp->repeat_n_to_k(values[0], values[1]);
}
- tmp->set_final_weights(weight, true);
- report_time();
- if (cache == NULL && should_use_cache()) {
+ tmp->set_final_weights(hfst::double_to_float(weight), true);
+ if (cache == NULL && should_use_cache() == true) {
cache = tmp;
+ cache->minimize();
+ report_time();
+ return new HfstTransducer(*cache);
}
- return new HfstTransducer(*tmp);
+ report_time();
+ return tmp;
}
HfstTransducer * PmatchUnaryOperation::evaluate(PmatchEvalType eval_type)
@@ -1288,7 +1717,7 @@ HfstTransducer * PmatchUnaryOperation::evaluate(PmatchEvalType eval_type)
if (cache != NULL && should_use_cache()) {
return new HfstTransducer(*cache);
}
- HfstTransducer * retval;
+ HfstTransducer * retval = NULL;
start_timing();
retval = root->evaluate();
if (op == AddDelimiters) {
@@ -1365,7 +1794,7 @@ HfstTransducer * PmatchUnaryOperation::evaluate(PmatchEvalType eval_type)
delete retval;
retval = tmp;
} else if (op == OptCap) {
- HfstTransducer * tmp = get_utils()->optcap(*retval);
+ HfstTransducer * tmp = get_utils()->cap(*retval, Both, true);
delete retval;
retval = tmp;
} else if (op == ToLower) {
@@ -1377,17 +1806,79 @@ HfstTransducer * PmatchUnaryOperation::evaluate(PmatchEvalType eval_type)
delete retval;
retval = tmp;
} else if (op == OptToLower) {
- HfstTransducer * tmp = get_utils()->opt_tolower(*retval);
+ HfstTransducer * tmp = get_utils()->tolower(*retval, Both, true);
tmp->disjunct(*retval);
delete retval;
retval = tmp;
} else if (op == OptToUpper) {
- HfstTransducer * tmp = get_utils()->opt_toupper(*retval);
+ HfstTransducer * tmp = get_utils()->toupper(*retval, Both, true);
delete retval;
retval = tmp;
} else if (op == AnyCase) {
- HfstTransducer * toupper = get_utils()->opt_toupper(*retval);
- HfstTransducer * tolower = get_utils()->opt_tolower(*retval);
+ HfstTransducer * toupper = get_utils()->toupper(*retval, Both, true);
+ HfstTransducer * tolower = get_utils()->tolower(*retval, Both, true);
+ retval->disjunct(*toupper);
+ retval->disjunct(*tolower);
+ delete toupper; delete tolower;
+ } else if (op == CapUpper) {
+ HfstTransducer * tmp = get_utils()->cap(*retval, Upper);
+ delete retval;
+ retval = tmp;
+ } else if (op == OptCapUpper) {
+ HfstTransducer * tmp = get_utils()->cap(*retval, Upper, true);
+ delete retval;
+ retval = tmp;
+ } else if (op == ToLowerUpper) {
+ HfstTransducer * tmp = get_utils()->tolower(*retval, Upper);
+ delete retval;
+ retval = tmp;
+ } else if (op == ToUpperUpper) {
+ HfstTransducer * tmp = get_utils()->toupper(*retval, Upper);
+ delete retval;
+ retval = tmp;
+ } else if (op == OptToLowerUpper) {
+ HfstTransducer * tmp = get_utils()->tolower(*retval, Upper, true);
+ tmp->disjunct(*retval);
+ delete retval;
+ retval = tmp;
+ } else if (op == OptToUpperUpper) {
+ HfstTransducer * tmp = get_utils()->toupper(*retval, Upper, true);
+ delete retval;
+ retval = tmp;
+ } else if (op == AnyCaseUpper) {
+ HfstTransducer * toupper = get_utils()->toupper(*retval, Upper, true);
+ HfstTransducer * tolower = get_utils()->tolower(*retval, Upper, true);
+ retval->disjunct(*toupper);
+ retval->disjunct(*tolower);
+ delete toupper; delete tolower;
+ } else if (op == CapLower) {
+ HfstTransducer * tmp = get_utils()->cap(*retval, Lower);
+ delete retval;
+ retval = tmp;
+ } else if (op == OptCapLower) {
+ HfstTransducer * tmp = get_utils()->cap(*retval, Lower, true);
+ delete retval;
+ retval = tmp;
+ } else if (op == ToLowerLower) {
+ HfstTransducer * tmp = get_utils()->tolower(*retval, Lower);
+ delete retval;
+ retval = tmp;
+ } else if (op == ToUpperLower) {
+ HfstTransducer * tmp = get_utils()->toupper(*retval, Lower);
+ delete retval;
+ retval = tmp;
+ } else if (op == OptToLowerLower) {
+ HfstTransducer * tmp = get_utils()->tolower(*retval, Lower, true);
+ tmp->disjunct(*retval);
+ delete retval;
+ retval = tmp;
+ } else if (op == OptToUpperLower) {
+ HfstTransducer * tmp = get_utils()->toupper(*retval, Lower, true);
+ delete retval;
+ retval = tmp;
+ } else if (op == AnyCaseLower) {
+ HfstTransducer * toupper = get_utils()->toupper(*retval, Lower, true);
+ HfstTransducer * tolower = get_utils()->tolower(*retval, Lower, true);
retval->disjunct(*toupper);
retval->disjunct(*tolower);
delete toupper; delete tolower;
@@ -1407,13 +1898,66 @@ HfstTransducer * PmatchUnaryOperation::evaluate(PmatchEvalType eval_type)
delete retval;
retval = tmp;
}
+ } else if (op == LC) {
+ if (!parent_is_context) {
+ retval->reverse();
+ HfstTransducer * tmp = new HfstTransducer(hfst::internal_epsilon, LC_ENTRY_SYMBOL, format);
+ tmp->concatenate(*retval);
+ HfstTransducer lc_exit(hfst::internal_epsilon, LC_EXIT_SYMBOL, format);
+ tmp->concatenate(lc_exit);
+ delete retval;
+ retval = tmp;
+ }
+ } else if (op == NLC) {
+ if (!parent_is_context) {
+ retval->reverse();
+ PmatchTransducerContainer * tmp = make_minimization_guard();
+ HfstTransducer * head = tmp->evaluate(); delete tmp;
+ HfstTransducer passthrough(hfst::internal_epsilon, PASSTHROUGH_SYMBOL, format);
+ HfstTransducer nlc_entry(hfst::internal_epsilon, NLC_ENTRY_SYMBOL, format);
+ HfstTransducer nlc_exit(hfst::internal_epsilon, NLC_EXIT_SYMBOL, format);
+ nlc_entry.concatenate(*retval);
+ nlc_entry.concatenate(nlc_exit);
+ nlc_entry.disjunct(passthrough);
+ head->concatenate(nlc_entry);
+ delete retval;
+ retval = head;
+ }
+ } else if (op == RC) {
+ if (!parent_is_context) {
+ HfstTransducer * tmp = new HfstTransducer(hfst::internal_epsilon, RC_ENTRY_SYMBOL, format);
+ tmp->concatenate(*retval);
+ HfstTransducer rc_exit(hfst::internal_epsilon, RC_EXIT_SYMBOL, format);
+ tmp->concatenate(rc_exit);
+ delete retval;
+ retval = tmp;
+ }
+ } else if (op == NRC) {
+ if (!parent_is_context) {
+ PmatchTransducerContainer * tmp = make_minimization_guard();
+ HfstTransducer * head = tmp->evaluate(); delete tmp;
+ HfstTransducer passthrough(hfst::internal_epsilon, PASSTHROUGH_SYMBOL, format);
+ HfstTransducer nlc_entry(hfst::internal_epsilon, NLC_ENTRY_SYMBOL, format);
+ HfstTransducer nlc_exit(hfst::internal_epsilon, NLC_EXIT_SYMBOL, format);
+ nlc_entry.concatenate(*retval);
+ nlc_entry.concatenate(nlc_exit);
+ nlc_entry.disjunct(passthrough);
+ head->concatenate(nlc_entry);
+ delete retval;
+ retval = head;
+ }
}
- retval->set_final_weights(weight, true);
- report_time();
- if (cache == NULL && should_use_cache()) {
+
+ retval->set_final_weights(hfst::double_to_float(weight), true);
+ if (cache == NULL && should_use_cache() == true) {
cache = retval;
+ cache->minimize();
+ report_time();
+ print_size_info(cache);
+ return new HfstTransducer(*cache);
}
- return new HfstTransducer(*retval);
+ report_time();
+ return retval;
}
HfstTransducer * PmatchBinaryOperation::evaluate(PmatchEvalType eval_type)
@@ -1422,7 +1966,7 @@ HfstTransducer * PmatchBinaryOperation::evaluate(PmatchEvalType eval_type)
return new HfstTransducer(*cache);
}
start_timing();
- HfstTransducer * retval;
+ HfstTransducer * retval = NULL;
HfstTransducer * lhs = left->evaluate();
HfstTransducer * rhs = right->evaluate();
if (op == Concatenate) {
@@ -1456,6 +2000,7 @@ HfstTransducer * PmatchBinaryOperation::evaluate(PmatchEvalType eval_type)
try {
lhs->shuffle(*rhs);
} catch (const TransducersAreNotAutomataException & e) {
+ (void)e;
pmatchwarning("tried to shuffle with non-automaton transducers,\n"
" shuffling with their input projection instead.");
lhs->input_project();
@@ -1487,18 +2032,23 @@ HfstTransducer * PmatchBinaryOperation::evaluate(PmatchEvalType eval_type)
tmp = hfst::xre::merge_first_to_second(lhs, rhs);
}
catch (const TransducersAreNotAutomataException & e) {
+ (void)e;
pmatcherror("Error: transducers must be automata in merge operation.");
}
delete lhs; lhs = tmp;
}
delete rhs;
- lhs->set_final_weights(weight, true);
- report_time();
+ lhs->set_final_weights(hfst::double_to_float(weight), true);
retval = lhs;
- if (cache == NULL && should_use_cache()) {
+ if (cache == NULL && should_use_cache() == true) {
cache = retval;
+ cache->minimize();
+ print_size_info(cache);
+ report_time();
+ return new HfstTransducer(*cache);
}
- return new HfstTransducer(*retval);
+ report_time();
+ return retval;
}
StringPair PmatchBinaryOperation::as_string_pair(void)
@@ -1517,7 +2067,7 @@ HfstTransducer * PmatchTernaryOperation::evaluate(PmatchEvalType eval_type)
return new HfstTransducer(*cache);
}
start_timing();
- HfstTransducer * retval;
+ HfstTransducer * retval = NULL;
if (op == Substitute) {
retval = left->evaluate();
StringPair middle_pair = middle->as_string_pair();
@@ -1530,18 +2080,21 @@ HfstTransducer * PmatchTernaryOperation::evaluate(PmatchEvalType eval_type)
delete tmp;
}
}
- retval->set_final_weights(weight, true);
- report_time();
- if (cache == NULL && should_use_cache()) {
+ retval->set_final_weights(hfst::double_to_float(weight), true);
+ if (cache == NULL && should_use_cache() == true) {
cache = retval;
+ cache->minimize();
+ report_time();
+ return new HfstTransducer(*cache);
}
- return new HfstTransducer(*retval);
+ report_time();
+ return retval;
}
HfstTransducer * PmatchAcceptor::evaluate(PmatchEvalType eval_type)
{
start_timing();
- HfstTransducer * retval;
+ HfstTransducer * retval = NULL;
switch(set) {
case Alpha:
retval = new HfstTransducer(* get_utils()->latin1_alpha_acceptor);
@@ -1561,7 +2114,7 @@ HfstTransducer * PmatchAcceptor::evaluate(PmatchEvalType eval_type)
case Whitespace:
retval = new HfstTransducer(* get_utils()->latin1_whitespace_acceptor);
}
- retval->set_final_weights(weight, true);
+ retval->set_final_weights(hfst::double_to_float(weight), true);
report_time();
return retval;
}
@@ -1572,7 +2125,7 @@ HfstTransducer * PmatchParallelRulesContainer::evaluate(PmatchEvalType eval_type
return new HfstTransducer(*cache);
}
start_timing();
- HfstTransducer * retval;
+ HfstTransducer * retval = NULL;
switch (arrow) {
case hfst::xeroxRules::E_REPLACE_RIGHT:
retval = new HfstTransducer(replace(make_mappings(), false));
@@ -1603,12 +2156,14 @@ HfstTransducer * PmatchParallelRulesContainer::evaluate(PmatchEvalType eval_type
pmatcherror("Unrecognized arrow type");
return (HfstTransducer *) NULL;
}
- retval->set_final_weights(weight, true);
+ retval->set_final_weights(hfst::double_to_float(weight), true);
report_time();
- if (cache == NULL && should_use_cache()) {
+ if (cache == NULL && should_use_cache() == true) {
cache = retval;
+ cache->minimize();
+ return new HfstTransducer(*cache);
}
- return new HfstTransducer(*retval);
+ return retval;
}
std::vector<hfst::xeroxRules::Rule> PmatchParallelRulesContainer::make_mappings(void)
@@ -1628,7 +2183,7 @@ HfstTransducer * PmatchReplaceRuleContainer::evaluate(PmatchEvalType eval_type)
return new HfstTransducer(*cache);
}
start_timing();
- HfstTransducer * retval;
+ HfstTransducer * retval = NULL;
switch (arrow) {
case hfst::xeroxRules::E_REPLACE_RIGHT:
retval = new HfstTransducer(replace(make_mapping(), false));
@@ -1659,12 +2214,14 @@ HfstTransducer * PmatchReplaceRuleContainer::evaluate(PmatchEvalType eval_type)
pmatcherror("Unrecognized arrow");
return (HfstTransducer *) NULL;
}
- retval->set_final_weights(weight, true);
+ retval->set_final_weights(hfst::double_to_float(weight), true);
report_time();
- if (cache == NULL && should_use_cache()) {
+ if (cache == NULL && should_use_cache() == true) {
cache = retval;
+ cache->minimize();
+ return new HfstTransducer(*cache);
}
- return new HfstTransducer(*retval);
+ return retval;
}
hfst::xeroxRules::Rule PmatchReplaceRuleContainer::make_mapping(void)
@@ -1696,13 +2253,13 @@ hfst::xeroxRules::Rule PmatchReplaceRuleContainer::make_mapping(void)
HfstTransducer * PmatchQuestionMark::evaluate(PmatchEvalType eval_type)
{
start_timing();
- HfstTransducer * retval;
+ HfstTransducer * retval = NULL;
if (eval_type == Transducer) {
retval = new HfstTransducer(hfst::internal_identity, format);
} else {
retval = new HfstTransducer(hfst::internal_unknown, format);
}
- retval->set_final_weights(weight, true);
+ retval->set_final_weights(hfst::double_to_float(weight), true);
report_time();
return retval;
}
@@ -1713,7 +2270,7 @@ HfstTransducer * PmatchRestrictionContainer::evaluate(PmatchEvalType eval_type)
return new HfstTransducer(*cache);
}
start_timing();
- HfstTransducer * retval;
+ HfstTransducer * retval = NULL;
HfstTransducerPairVector pair_vector;
for (MappingPairVector::iterator it = contexts->begin();
it != contexts->end(); ++it) {
@@ -1726,12 +2283,14 @@ HfstTransducer * PmatchRestrictionContainer::evaluate(PmatchEvalType eval_type)
HfstTransducer * l = left->evaluate();
retval = new HfstTransducer(hfst::xeroxRules::restriction(*l, pair_vector));
delete l;
- retval->set_final_weights(weight, true);
+ retval->set_final_weights(hfst::double_to_float(weight), true);
report_time();
- if (cache == NULL && should_use_cache()) {
+ if (cache == NULL && should_use_cache() == true) {
cache = retval;
+ cache->minimize();
+ return new HfstTransducer(*cache);
}
- return new HfstTransducer(*retval);
+ return retval;
}
HfstTransducer * PmatchMarkupContainer::evaluate(PmatchEvalType eval_type) { pmatcherror("Should never happen\n"); throw 1; }
diff --git a/libhfst/src/parsers/pmatch_utils.h b/libhfst/src/parsers/pmatch_utils.h
index 4fe859a..abf245c 100644
--- a/libhfst/src/parsers/pmatch_utils.h
+++ b/libhfst/src/parsers/pmatch_utils.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
/**
@@ -21,6 +21,7 @@
#include <set>
#include <time.h>
#include <iomanip>
+#include <cmath>
#include "HfstTransducer.h"
#include "HfstXeroxRules.h"
#include "xre_utils.h"
@@ -34,24 +35,29 @@ struct PmatchObject;
struct PmatchTransducerContainer;
typedef std::pair<std::string, std::string> StringPair;
+typedef float WordVecFloat;
+struct WordVector;
extern char* data;
extern char* startptr;
extern size_t len;
extern std::map<std::string, PmatchObject*> definitions;
+extern std::map<std::string, std::string> variables;
extern std::vector<std::map<std::string, PmatchObject*> > call_stack;
extern std::map<std::string, PmatchObject*> def_insed_expressions;
extern std::set<std::string> inserted_names;
extern std::set<std::string> unsatisfied_insertions;
extern std::set<std::string> used_definitions;
extern std::set<std::string> function_names;
+extern std::vector<WordVector> word_vectors;
extern ImplementationType format;
extern bool verbose;
extern bool flatten;
+extern std::string includedir;
extern clock_t timer;
-extern clock_t tmp_timer;
extern int minimization_guard_count;
extern bool need_delimiters;
+extern WordVecFloat vector_similarity_projection_factor;
struct PmatchUtilityTransducers;
const std::string RC_ENTRY_SYMBOL = "@PMATCH_RC_ENTRY@";
@@ -67,13 +73,19 @@ const std::string BOUNDARY_SYMBOL = "@BOUNDARY@";
const std::string ENTRY_SYMBOL = "@PMATCH_ENTRY@";
const std::string EXIT_SYMBOL = "@PMATCH_EXIT@";
+// These are used as arguments for casing functions
+enum Side {
+ Both,
+ Upper,
+ Lower
+};
+
void add_to_pmatch_symbols(StringSet symbols);
void warn(std::string warning);
PmatchUtilityTransducers* get_utils();
void zero_minimization_guard(void);
bool symbol_in_global_context(std::string & sym);
bool symbol_in_local_context(std::string & sym);
-bool should_use_cache(void);
PmatchObject * symbol_from_global_context(std::string & sym);
PmatchObject * symbol_from_local_context(std::string & sym);
@@ -114,6 +126,19 @@ HfstTransducer * add_pmatch_delimiters(HfstTransducer * regex);
*/
PmatchTransducerContainer * epsilon_to_symbol_container(std::string s);
PmatchTransducerContainer * make_end_tag(std::string tag);
+template<typename T> std::vector<T> pointwise_minus(std::vector<T> l,
+ std::vector<T> r);
+template<typename T> std::vector<T> pointwise_plus(std::vector<T> l,
+ std::vector<T> r);
+template<typename T> std::vector<T> pointwise_multiplication(T,
+ std::vector<T> r);
+template<typename T> T dot_product(std::vector<T> l,
+ std::vector<T> r);
+template<typename T> T square_sum(std::vector<T> v);
+template<typename T> T norm(std::vector<T> v);
+WordVecFloat cosine_distance(WordVector left, WordVector right);
+PmatchObject * compile_like_arc(std::string word1, std::string word2 = "",
+ unsigned int nwords = 10);
PmatchTransducerContainer * make_counter(std::string name);
HfstTransducer * make_list(HfstTransducer * t,
ImplementationType f = format);
@@ -159,7 +184,8 @@ std::map<std::string, HfstTransducer*>
compile(const std::string& pmatch,
std::map<std::string,hfst::HfstTransducer*>& defs,
hfst::ImplementationType type,
- bool be_verbose, bool do_flatten);
+ bool be_verbose = false, bool do_flatten = false,
+ std::string includedir = "");
void print_size_info(HfstTransducer * net);
@@ -167,8 +193,31 @@ void print_size_info(HfstTransducer * net);
* @brief Given a text file, read it line by line and return an acceptor
* of a disjunction of the lines
*/
-HfstTransducer * read_text(char * filename,
- ImplementationType type = TROPICAL_OPENFST_TYPE);
+HfstTransducer * read_text(std::string filename,
+ ImplementationType type = TROPICAL_OPENFST_TYPE,
+ bool spaced_text = false);
+
+HfstTransducer * read_spaced_text(std::string filename,
+ ImplementationType type = TROPICAL_OPENFST_TYPE);
+
+/**
+ * @brief Concatenate include directory with filename to get a real path
+ * (unless the filename is already an absolute path)
+ */
+std::string path_from_filename(char * filename);
+
+struct WordVector
+{
+ std::string word;
+ std::vector<WordVecFloat> vector;
+ WordVecFloat norm;
+};
+
+/**
+ * @brief Given a list of words and their vector representations, parse it into
+ * hfst::pmatch::word_vectors
+ */
+void read_vec(std::string filename);
/**
* @brief Given a text file, read it line by line and return a tokenized
@@ -177,7 +226,7 @@ HfstTransducer * read_text(char * filename,
std::vector<std::vector<std::string> > read_args(char * filename, unsigned int argcount);
/** @brief Return a transducer that accepts a single string from an array of
- * char *.
+ * char *.
*/
/* First some magic templates for compile-time length checking */
@@ -300,7 +349,7 @@ struct PmatchUtilityTransducers
static HfstTransducer * make_combining_accent_acceptor(
ImplementationType type = TROPICAL_OPENFST_TYPE);
-/** @brief Return a transducer that accepts one arabic numeral character.
+/** @brief Return a transducer that accepts one arabic numeral character.
*/
static HfstTransducer * make_latin1_numeral_acceptor(
ImplementationType type = TROPICAL_OPENFST_TYPE);
@@ -324,12 +373,12 @@ struct PmatchUtilityTransducers
HfstTransducer * make_capify(
ImplementationType type = TROPICAL_OPENFST_TYPE);
- HfstTransducer * cap(HfstTransducer & t);
- HfstTransducer * optcap(HfstTransducer & t);
- HfstTransducer * tolower(HfstTransducer & t);
- HfstTransducer * toupper(HfstTransducer & t);
- HfstTransducer * opt_tolower(HfstTransducer & t);
- HfstTransducer * opt_toupper(HfstTransducer & t);
+ HfstTransducer * cap(HfstTransducer & t, Side side = Both,
+ bool optional = false);
+ HfstTransducer * tolower(HfstTransducer & t, Side side = Both,
+ bool optional = false);
+ HfstTransducer * toupper(HfstTransducer & t, Side side = Both,
+ bool optional = false );
};
struct PmatchObject;
@@ -344,24 +393,33 @@ struct PmatchObject {
std::string name; // optional, given if the object appears as a definition
double weight;
int line_defined;
+ clock_t my_timer;
HfstTransducer * cache;
+ bool parent_is_context;
PmatchObject(void);
void start_timing(void)
{
if (verbose && name != "") {
- tmp_timer = clock();
+ my_timer = clock();
}
}
void report_time(void)
{
if (verbose && name != "") {
- double duration = (clock() - tmp_timer) /
+ double duration = (clock() - my_timer) /
(double) CLOCKS_PER_SEC;
std::cerr << name << " compiled in " << duration << " seconds\n";
+
}
}
+ bool should_use_cache(void)
+ {
+ return name != "" && call_stack.size() == 0;
+ }
+
virtual HfstTransducer * evaluate(PmatchEvalType eval_type = Transducer) = 0;
virtual HfstTransducer * evaluate(std::vector<PmatchObject *> args);
+ virtual void mark_context_children(void) { parent_is_context = true; }
virtual std::string as_string(void) { return ""; }
virtual StringPair as_string_pair(void)
{ return StringPair("", ""); }
@@ -392,7 +450,6 @@ struct PmatchQuestionMark: public PmatchObject {
StringPair as_string_pair(void)
{ return StringPair(
hfst::internal_identity, hfst::internal_identity); }
-
};
enum PmatchUnaryOp {
@@ -416,9 +473,27 @@ enum PmatchUnaryOp {
OptToLower,
OptToUpper,
AnyCase,
+ CapUpper,
+ OptCapUpper,
+ ToLowerUpper,
+ ToUpperUpper,
+ OptToLowerUpper,
+ OptToUpperUpper,
+ AnyCaseUpper,
+ CapLower,
+ OptCapLower,
+ ToLowerLower,
+ ToUpperLower,
+ OptToLowerLower,
+ OptToUpperLower,
+ AnyCaseLower,
MakeSigma,
MakeList,
- MakeExcList
+ MakeExcList,
+ LC,
+ NLC,
+ RC,
+ NRC
};
enum PmatchBinaryOp {
@@ -472,6 +547,11 @@ struct PmatchNumericOperation: public PmatchObject{
PmatchNumericOperation(PmatchNumericOp _op, PmatchObject * _root):
op(_op), root(_root) {}
HfstTransducer * evaluate(PmatchEvalType eval_type = Transducer);
+ void mark_context_children(void)
+ {
+ parent_is_context = true;
+ root->mark_context_children();
+ }
};
struct PmatchUnaryOperation: public PmatchObject{
@@ -480,6 +560,11 @@ struct PmatchUnaryOperation: public PmatchObject{
PmatchUnaryOperation(PmatchUnaryOp _op, PmatchObject * _root):
op(_op), root(_root) {}
HfstTransducer * evaluate(PmatchEvalType eval_type = Transducer);
+ void mark_context_children(void)
+ {
+ parent_is_context = true;
+ root->mark_context_children();
+ }
};
struct PmatchBinaryOperation: public PmatchObject{
@@ -490,6 +575,12 @@ struct PmatchBinaryOperation: public PmatchObject{
op(_op), left(_left), right(_right) {}
HfstTransducer * evaluate(PmatchEvalType eval_type = Transducer);
StringPair as_string_pair(void);
+ void mark_context_children(void)
+ {
+ parent_is_context = true;
+ left->mark_context_children();
+ right->mark_context_children();
+ }
};
struct PmatchTernaryOperation: public PmatchObject{
@@ -500,6 +591,13 @@ struct PmatchTernaryOperation: public PmatchObject{
PmatchTernaryOperation(PmatchTernaryOp _op, PmatchObject * _left, PmatchObject * _middle, PmatchObject * _right):
op(_op), left(_left), middle(_middle), right(_right) {}
HfstTransducer * evaluate(PmatchEvalType eval_type = Transducer);
+ void mark_context_children(void)
+ {
+ parent_is_context = true;
+ left->mark_context_children();
+ middle->mark_context_children();
+ right->mark_context_children();
+ }
};
struct PmatchTransducerContainer: public PmatchObject{
@@ -512,7 +610,7 @@ struct PmatchTransducerContainer: public PmatchObject{
t->convert(format);
}
HfstTransducer * retval = new HfstTransducer(*t);
- retval->set_final_weights(weight, true);
+ retval->set_final_weights(hfst::double_to_float(weight), true);
return retval;
}
};
@@ -527,6 +625,7 @@ struct PmatchFunction: public PmatchObject {
HfstTransducer * evaluate(std::vector<PmatchObject *> funargs);
HfstTransducer * evaluate(PmatchEvalType eval_type = Transducer);
+
};
struct PmatchFuncall: public PmatchObject {
@@ -551,6 +650,14 @@ struct PmatchFuncall: public PmatchObject {
}
return retval;
}
+ void mark_context_children(void)
+ {
+ for (std::vector<PmatchObject *>::iterator it = args->begin();
+ it != args->end(); ++it) {
+ (*it)->mark_context_children();
+ }
+ parent_is_context = true;
+ }
};
struct PmatchBuiltinFunction: public PmatchObject {
@@ -558,8 +665,16 @@ struct PmatchBuiltinFunction: public PmatchObject {
PmatchBuiltin type;
PmatchBuiltinFunction(PmatchBuiltin _type,
std::vector<PmatchObject*>* argument_vector):
- type(_type), args(argument_vector) {}
+ args(argument_vector), type(_type) {}
HfstTransducer * evaluate(PmatchEvalType eval_type = Transducer);
+ void mark_context_children(void)
+ {
+ parent_is_context = true;
+ for (std::vector<PmatchObject *>::iterator it = args->begin();
+ it != args->end(); ++it) {
+ (*it)->mark_context_children();
+ }
+ }
};
using hfst::xeroxRules::ReplaceArrow;
@@ -574,6 +689,11 @@ struct PmatchRestrictionContainer: public PmatchObject
PmatchRestrictionContainer(PmatchObject * l, MappingPairVector * c):
left(l), contexts(c) { }
HfstTransducer * evaluate(PmatchEvalType eval_type = Transducer);
+ void mark_context_children(void)
+ {
+ parent_is_context = true;
+ left->mark_context_children();
+ }
};
struct PmatchMarkupContainer: public PmatchObject
@@ -583,6 +703,12 @@ struct PmatchMarkupContainer: public PmatchObject
PmatchMarkupContainer(PmatchObject * l, PmatchObject * r):
left(l), right(r) {}
HfstTransducer * evaluate(PmatchEvalType eval_type = Transducer);
+ void mark_context_children(void)
+ {
+ parent_is_context = true;
+ left->mark_context_children();
+ right->mark_context_children();
+ }
};
struct PmatchMappingPairsContainer: public PmatchObject
@@ -646,8 +772,8 @@ struct PmatchReplaceRuleContainer: public PmatchObject
arrow(pairs->arrow), mapping(pairs->mapping_pairs) {}
PmatchReplaceRuleContainer(PmatchMappingPairsContainer * pairs,
PmatchContextsContainer * contexts):
- arrow(pairs->arrow), mapping(pairs->mapping_pairs),
- context(contexts->context_pairs), type(contexts->type) {}
+ arrow(pairs->arrow), type(contexts->type),
+ mapping(pairs->mapping_pairs), context(contexts->context_pairs) {}
hfst::xeroxRules::Rule make_mapping(void);
HfstTransducer * evaluate(PmatchEvalType eval_type = Transducer);
};
diff --git a/tools/src/hfst-twolc/src/rule_src/ConflictResolvingLeftArrowRule.cc b/libhfst/src/parsers/rule_src/ConflictResolvingLeftArrowRule.cc
similarity index 99%
rename from tools/src/hfst-twolc/src/rule_src/ConflictResolvingLeftArrowRule.cc
rename to libhfst/src/parsers/rule_src/ConflictResolvingLeftArrowRule.cc
index 5e78fef..a20a693 100644
--- a/tools/src/hfst-twolc/src/rule_src/ConflictResolvingLeftArrowRule.cc
+++ b/libhfst/src/parsers/rule_src/ConflictResolvingLeftArrowRule.cc
@@ -58,8 +58,8 @@ OtherSymbolTransducer wbize(const OtherSymbolTransducer &t)
bool ConflictResolvingLeftArrowRule::conflicts_this
(const ConflictResolvingLeftArrowRule &another,StringVector &v)
-{
- return ! context.is_empty_intersection(wbize(another.context),v);
+{
+ return ! context.is_empty_intersection(wbize(another.context),v);
}
bool ConflictResolvingLeftArrowRule::resolvable_conflict
diff --git a/tools/src/hfst-twolc/src/rule_src/ConflictResolvingLeftArrowRule.h b/libhfst/src/parsers/rule_src/ConflictResolvingLeftArrowRule.h
similarity index 99%
rename from tools/src/hfst-twolc/src/rule_src/ConflictResolvingLeftArrowRule.h
rename to libhfst/src/parsers/rule_src/ConflictResolvingLeftArrowRule.h
index aa4a9ce..44a9dcb 100644
--- a/tools/src/hfst-twolc/src/rule_src/ConflictResolvingLeftArrowRule.h
+++ b/libhfst/src/parsers/rule_src/ConflictResolvingLeftArrowRule.h
@@ -41,19 +41,19 @@ class ConflictResolvingLeftArrowRule : public LeftArrowRule
//! @brief Return true if @a another conflicts @a this.
//!
//! Return true if the input symbols of the centers of @a another and
- //! @this are equal, the output symbols differ, and the contexts of @a
+ //! @this are equal, the output symbols differ, and the contexts of @a
//! another and @a this have a non-empty intersection.
//!
//! If a conflict exists, store the conflicting string in v.
bool conflicts_this(const ConflictResolvingLeftArrowRule &another,
StringVector &v);
- //! Return @a true, if the context of @a another is a sub language of
+ //! Return @a true, if the context of @a another is a sub language of
//! the context of @a this.
bool resolvable_conflict(const ConflictResolvingLeftArrowRule &another);
//! @brief Resolve the conflict between @a this and @a another by subtracting
- //! the context of @a another fromt the @a context of @a this.
+ //! the context of @a another fromt the @a context of @a this.
void resolve_conflict(const ConflictResolvingLeftArrowRule &another);
friend class LeftArrowRuleContainer;
diff --git a/tools/src/hfst-twolc/src/rule_src/ConflictResolvingRightArrowRule.cc b/libhfst/src/parsers/rule_src/ConflictResolvingRightArrowRule.cc
similarity index 98%
rename from tools/src/hfst-twolc/src/rule_src/ConflictResolvingRightArrowRule.cc
rename to libhfst/src/parsers/rule_src/ConflictResolvingRightArrowRule.cc
index e433c41..e6dcc56 100644
--- a/tools/src/hfst-twolc/src/rule_src/ConflictResolvingRightArrowRule.cc
+++ b/libhfst/src/parsers/rule_src/ConflictResolvingRightArrowRule.cc
@@ -30,16 +30,16 @@ ConflictResolvingRightArrowRule::ConflictResolvingRightArrowRule
bool ConflictResolvingRightArrowRule::conflicts_this
(ConflictResolvingRightArrowRule &another)
-{ return
+{ return
center_pair.first == another.center_pair.first &&
center_pair.second == another.center_pair.second; }
void ConflictResolvingRightArrowRule::resolve_conflict
(ConflictResolvingRightArrowRule &another)
-{
+{
context.
apply(&HfstTransducer::disjunct,another.context).
- apply(&HfstTransducer::minimize);
+ apply(&HfstTransducer::minimize);
name += " and " + another.name;
}
diff --git a/tools/src/hfst-twolc/src/rule_src/ConflictResolvingRightArrowRule.h b/libhfst/src/parsers/rule_src/ConflictResolvingRightArrowRule.h
similarity index 100%
rename from tools/src/hfst-twolc/src/rule_src/ConflictResolvingRightArrowRule.h
rename to libhfst/src/parsers/rule_src/ConflictResolvingRightArrowRule.h
diff --git a/tools/src/hfst-twolc/src/rule_src/LeftArrowRule.cc b/libhfst/src/parsers/rule_src/LeftArrowRule.cc
similarity index 93%
rename from tools/src/hfst-twolc/src/rule_src/LeftArrowRule.cc
rename to libhfst/src/parsers/rule_src/LeftArrowRule.cc
index 44c4a9c..b8256fa 100644
--- a/tools/src/hfst-twolc/src/rule_src/LeftArrowRule.cc
+++ b/libhfst/src/parsers/rule_src/LeftArrowRule.cc
@@ -25,7 +25,7 @@ LeftArrowRule::LeftArrowRule
OtherSymbolTransducer LeftArrowRule::compile(void)
{
- OtherSymbolTransducer abstract_center =
+ OtherSymbolTransducer abstract_center =
center.get_inverse_of_upper_projection();
context.
apply(&HfstTransducer::intersect,abstract_center);
@@ -46,23 +46,23 @@ int main(void)
{
bool have_openfst = false;
#if HAVE_OPENFST
- have_openfst = true;
+ have_openfst = true;
#endif // HAVE_OPENFST
bool have_sfst = false;
#if HAVE_SFST
- have_sfst = true;
+ have_sfst = true;
#endif // HAVE_SFST
bool have_foma = false;
#if HAVE_FOMA
- have_foma = true;
+ have_foma = true;
#endif // HAVE_FOMA
-ImplementationType transducer_type
-= have_openfst ? hfst::TROPICAL_OPENFST_TYPE :
+ImplementationType transducer_type
+= have_openfst ? hfst::TROPICAL_OPENFST_TYPE :
have_sfst ? hfst::SFST_TYPE :
- have_foma ? hfst::FOMA_TYPE :
+ have_foma ? hfst::FOMA_TYPE :
hfst::ERROR_TYPE;
OtherSymbolTransducer::set_transducer_type(transducer_type);
@@ -92,7 +92,7 @@ ImplementationType transducer_type
// ?* a:b <D> ?* <D> ?*
OtherSymbolTransducer context = unknown;
- context.
+ context.
apply(&HfstTransducer::concatenate,a_b_pair).
apply(&HfstTransducer::concatenate,diamond).
apply(&HfstTransducer::concatenate,unknown_optional).
diff --git a/tools/src/hfst-twolc/src/rule_src/LeftArrowRule.h b/libhfst/src/parsers/rule_src/LeftArrowRule.h
similarity index 100%
rename from tools/src/hfst-twolc/src/rule_src/LeftArrowRule.h
rename to libhfst/src/parsers/rule_src/LeftArrowRule.h
diff --git a/tools/src/hfst-twolc/src/rule_src/LeftArrowRuleContainer.cc b/libhfst/src/parsers/rule_src/LeftArrowRuleContainer.cc
similarity index 88%
rename from tools/src/hfst-twolc/src/rule_src/LeftArrowRuleContainer.cc
rename to libhfst/src/parsers/rule_src/LeftArrowRuleContainer.cc
index 49444c3..24ca3a3 100644
--- a/tools/src/hfst-twolc/src/rule_src/LeftArrowRuleContainer.cc
+++ b/libhfst/src/parsers/rule_src/LeftArrowRuleContainer.cc
@@ -28,7 +28,7 @@ void LeftArrowRuleContainer::set_report_left_arrow_conflicts(bool option)
void LeftArrowRuleContainer::add_rule_and_display_and_resolve_conflicts
(ConflictResolvingLeftArrowRule * rule,std::ostream &out)
{
- std::string input = rule->input_symbol;
+ std::string input = rule->input_symbol;
if (input_to_rule_map.has_key(input))
{
for (LeftArrowRuleVector::iterator it = input_to_rule_map[input].begin();
@@ -37,37 +37,37 @@ void LeftArrowRuleContainer::add_rule_and_display_and_resolve_conflicts
{
StringVector conflicting_context;
if ((*it)->conflicts_this(*rule,conflicting_context))
- {
+ {
if (report_left_arrow_conflicts)
{
- out << "There is a <=-rule conflict between "
- << Rule::get_print_name((*it)->name) << " and "
+ out << "There is a <=-rule conflict between "
+ << Rule::get_print_name((*it)->name) << " and "
<< Rule::get_print_name(rule->name) << "."
<< std::endl
<< "E.g. in context ";
bool diamond_seen = false;
- for (StringVector::const_iterator it =
+ for (StringVector::const_iterator it =
conflicting_context.begin();
it != conflicting_context.end();
++it)
- {
+ {
std::string symbol_pair = *it;
symbol_pair = replace_substr
(symbol_pair,TWOLC_EPSILON,"");
- if (symbol_pair ==
+ if (symbol_pair ==
"__HFST_TWOLC_DIAMOND:__HFST_TWOLC_DIAMOND")
- {
+ {
if (diamond_seen)
{ continue; }
symbol_pair = "_";
diamond_seen = true;
}
- else if
- (symbol_pair ==
+ else if
+ (symbol_pair ==
"@_TWOLC_IDENTITY_SYMBOL_@:@_TWOLC_IDENTITY_SYMBOL_@")
{ symbol_pair = "?"; }
- out << symbol_pair << " ";
+ out << symbol_pair << " ";
}
out << std::endl;
}
@@ -78,7 +78,7 @@ void LeftArrowRuleContainer::add_rule_and_display_and_resolve_conflicts
if (report_left_arrow_conflicts)
{
out << "Resolving the conflict by restricting the "
- << "context of "
+ << "context of "
<< Rule::get_print_name((*it)->name) << "."
<< std::endl;
}
@@ -97,14 +97,14 @@ void LeftArrowRuleContainer::add_rule_and_display_and_resolve_conflicts
else
{
if (report_left_arrow_conflicts)
- {
+ {
out << "WARNING! The conflict is unresolvable."
- << std::endl;
+ << std::endl;
}
}
}
if (report_left_arrow_conflicts)
- {
+ {
out << std::endl;
}
}
diff --git a/tools/src/hfst-twolc/src/rule_src/LeftArrowRuleContainer.h b/libhfst/src/parsers/rule_src/LeftArrowRuleContainer.h
similarity index 97%
rename from tools/src/hfst-twolc/src/rule_src/LeftArrowRuleContainer.h
rename to libhfst/src/parsers/rule_src/LeftArrowRuleContainer.h
index cda9ad7..dc6b294 100644
--- a/tools/src/hfst-twolc/src/rule_src/LeftArrowRuleContainer.h
+++ b/libhfst/src/parsers/rule_src/LeftArrowRuleContainer.h
@@ -16,7 +16,7 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
-#ifndef LEFTT_ARROW_RULE_CONTAINER_H_
+#ifndef LEFT_ARROW_RULE_CONTAINER_H_
#define LEFT_ARROW_RULE_CONTAINER_H_
#ifdef HAVE_CONFIG_H
diff --git a/tools/src/hfst-twolc/src/rule_src/LeftRestrictionArrowRule.cc b/libhfst/src/parsers/rule_src/LeftRestrictionArrowRule.cc
similarity index 94%
rename from tools/src/hfst-twolc/src/rule_src/LeftRestrictionArrowRule.cc
rename to libhfst/src/parsers/rule_src/LeftRestrictionArrowRule.cc
index 6f31b19..84bccbb 100644
--- a/tools/src/hfst-twolc/src/rule_src/LeftRestrictionArrowRule.cc
+++ b/libhfst/src/parsers/rule_src/LeftRestrictionArrowRule.cc
@@ -52,23 +52,23 @@ int main(void)
{
bool have_openfst = false;
#if HAVE_OPENFST
- have_openfst = true;
+ have_openfst = true;
#endif // HAVE_OPENFST
bool have_sfst = false;
#if HAVE_SFST
- have_sfst = true;
+ have_sfst = true;
#endif // HAVE_SFST
bool have_foma = false;
#if HAVE_FOMA
- have_foma = true;
+ have_foma = true;
#endif // HAVE_FOMA
-ImplementationType transducer_type
-= have_openfst ? hfst::TROPICAL_OPENFST_TYPE :
+ImplementationType transducer_type
+= have_openfst ? hfst::TROPICAL_OPENFST_TYPE :
have_sfst ? hfst::SFST_TYPE :
- have_foma ? hfst::FOMA_TYPE :
+ have_foma ? hfst::FOMA_TYPE :
hfst::ERROR_TYPE;
OtherSymbolTransducer::set_transducer_type(transducer_type);
@@ -95,7 +95,7 @@ ImplementationType transducer_type
apply(&HfstTransducer::concatenate,unknown);
OtherSymbolTransducer b_c_pair("b","c");
OtherSymbolTransducer context = unknown;
- context.
+ context.
apply(&HfstTransducer::concatenate,b_c_pair).
apply(&HfstTransducer::concatenate,diamond).
apply(&HfstTransducer::concatenate,unknown).
diff --git a/tools/src/hfst-twolc/src/rule_src/LeftRestrictionArrowRule.h b/libhfst/src/parsers/rule_src/LeftRestrictionArrowRule.h
similarity index 100%
rename from tools/src/hfst-twolc/src/rule_src/LeftRestrictionArrowRule.h
rename to libhfst/src/parsers/rule_src/LeftRestrictionArrowRule.h
diff --git a/tools/src/hfst-twolc/src/rule_src/OtherSymbolTransducer.cc b/libhfst/src/parsers/rule_src/OtherSymbolTransducer.cc
similarity index 89%
rename from tools/src/hfst-twolc/src/rule_src/OtherSymbolTransducer.cc
rename to libhfst/src/parsers/rule_src/OtherSymbolTransducer.cc
index fe37864..62210e2 100644
--- a/tools/src/hfst-twolc/src/rule_src/OtherSymbolTransducer.cc
+++ b/libhfst/src/parsers/rule_src/OtherSymbolTransducer.cc
@@ -17,6 +17,8 @@
#include "OtherSymbolTransducer.h"
+#ifndef TEST_OTHER_SYMBOL_TRANSDUCER
+
HandySet<std::string> OtherSymbolTransducer::input_symbols;
HandySet<std::string> OtherSymbolTransducer::output_symbols;
HandySet<std::string> OtherSymbolTransducer::diacritics;
@@ -48,13 +50,13 @@ void OtherSymbolTransducer::define_diacritics
OtherSymbolTransducer::diacritics.clear();
OtherSymbolTransducer::diacritics.insert
(diacritics.begin(),diacritics.end());
- for (HandySet<std::string>::iterator it =
+ for (HandySet<std::string>::iterator it =
OtherSymbolTransducer::diacritics.begin();
it != OtherSymbolTransducer::diacritics.end();
++it)
- {
- symbol_pairs.erase(SymbolPair(*it,*it));
- symbol_pairs.erase(SymbolPair(*it,TWOLC_EPSILON));
+ {
+ symbol_pairs.erase(SymbolPair(*it,*it));
+ symbol_pairs.erase(SymbolPair(*it,TWOLC_EPSILON));
input_symbols.erase(*it);
output_symbols.erase(*it);
}
@@ -75,7 +77,7 @@ OtherSymbolTransducer OtherSymbolTransducer::get_universal(void)
if (it->first == TWOLC_DIAMOND)
{ continue; }
fst.add_transition(0,HfstBasicTransition
- (target,it->first,it->second,0.0));
+ (target,it->first,it->second,0.0));
}
universal.transducer = HfstTransducer(fst,transducer_type);
return universal;
@@ -94,14 +96,14 @@ OtherSymbolTransducer::OtherSymbolTransducer
(const std::string &i_symbol,const std::string &o_symbol):
is_broken(false),
transducer(transducer_type)
-{
+{
std::string input_symbol = i_symbol;
std::string output_symbol = o_symbol;
input_symbol = (input_symbol == TWOLC_UNKNOWN ? HFST_UNKNOWN : input_symbol);
output_symbol = (output_symbol == TWOLC_UNKNOWN ? HFST_UNKNOWN : output_symbol);
- check_pair(input_symbol,output_symbol);
+ check_pair(input_symbol,output_symbol);
if (is_broken)
{ return; }
if (input_symbol == HFST_UNKNOWN && output_symbol == HFST_UNKNOWN)
@@ -114,7 +116,7 @@ OtherSymbolTransducer::OtherSymbolTransducer
if (input_symbol == HFST_UNKNOWN)
{
- for (HandySet<std::string>::const_iterator it =
+ for (HandySet<std::string>::const_iterator it =
input_symbols.begin();
it != input_symbols.end();
++it)
@@ -125,8 +127,8 @@ OtherSymbolTransducer::OtherSymbolTransducer
}
}
else if (output_symbol == HFST_UNKNOWN)
- {
- for (HandySet<std::string>::const_iterator it =
+ {
+ for (HandySet<std::string>::const_iterator it =
output_symbols.begin();
it != output_symbols.end();
++it)
@@ -146,7 +148,7 @@ OtherSymbolTransducer::OtherSymbolTransducer
OtherSymbolTransducer::OtherSymbolTransducer(const std::string &sym):
is_broken(false),
transducer(transducer_type)
-{
+{
std::string symbol = sym;
symbol = (symbol == TWOLC_UNKNOWN ? HFST_UNKNOWN : symbol);
@@ -178,11 +180,11 @@ OtherSymbolTransducer &OtherSymbolTransducer::harmonize_diacritics
(OtherSymbolTransducer &t)
{
HfstBasicTransducer basic(transducer);
- std::set<std::string> alphabet =
+ std::set<std::string> alphabet =
basic.get_alphabet();
HfstBasicTransducer basic_t(t.transducer);
- std::set<std::string> t_alphabet =
+ std::set<std::string> t_alphabet =
basic_t.get_alphabet();
HandySet<std::string> missing_diacritics;
@@ -203,7 +205,7 @@ OtherSymbolTransducer &OtherSymbolTransducer::harmonize_diacritics
it != basic.end();
++it)
{
- for (HfstBasicTransducer::HfstTransitions::const_iterator jt
+ for (hfst::implementations::HfstBasicTransitions::const_iterator jt
= it->begin();
jt != it->end();
++jt)
@@ -212,7 +214,7 @@ OtherSymbolTransducer &OtherSymbolTransducer::harmonize_diacritics
{
HfstState target = jt->get_target_state();
- for (HandySet<std::string>::iterator kt =
+ for (HandySet<std::string>::iterator kt =
missing_diacritics.begin();
kt != missing_diacritics.end();
++kt)
@@ -290,12 +292,12 @@ void OtherSymbolTransducer::check_pair(const std::string &input_symbol,
{ is_broken = false; }
// other:X is valid, iff X is an output symbol or 0.
else if (input_symbol == HFST_UNKNOWN)
- { is_broken = ! (output_symbol == TWOLC_EPSILON ||
- output_symbols.has_element(output_symbol)); }
+ { is_broken = ! (output_symbol == TWOLC_EPSILON ||
+ output_symbols.has_element(output_symbol)); }
// X:other is valid, iff X is an input symbol or 0.
else if (output_symbol == HFST_UNKNOWN)
- { is_broken = ! (input_symbol == TWOLC_EPSILON ||
- input_symbols.has_element(input_symbol)); }
+ { is_broken = ! (input_symbol == TWOLC_EPSILON ||
+ input_symbols.has_element(input_symbol)); }
// 0:X is valid, iff X is an output symbol.
else if (input_symbol == TWOLC_EPSILON)
{ is_broken = ! output_symbols.has_element(output_symbol); }
@@ -303,12 +305,12 @@ void OtherSymbolTransducer::check_pair(const std::string &input_symbol,
else if (output_symbol == TWOLC_EPSILON)
{ is_broken = ! input_symbols.has_element(input_symbol); }
// X:X is valid if X is a diacritic.
- else if (diacritics.has_element(input_symbol) /*and
+ else if (diacritics.has_element(input_symbol) /*and
input_symbol == output_symbol*/)
{ is_broken = false; }
- // X:Y is valid iff it has been declared in the alphabet.
+ // X:Y is valid iff it has been declared in the alphabet.
else
- { is_broken =
+ { is_broken =
! symbol_pairs.has_element(SymbolPair(input_symbol,output_symbol)); }
if (is_broken)
{ std::cerr << "Unknown pair: "
@@ -317,29 +319,29 @@ void OtherSymbolTransducer::check_pair(const std::string &input_symbol,
OtherSymbolTransducer &OtherSymbolTransducer::operator=
(const OtherSymbolTransducer &another)
-{
+{
if (&another == this)
{ return *this; }
is_broken = another.is_broken;
- transducer = another.transducer;
+ transducer = another.transducer;
return *this;
}
OtherSymbolTransducer &OtherSymbolTransducer::apply
(HfstTransducerZeroArgMember p)
-{
+{
if (symbol_pairs.empty())
{ throw EmptySymbolPairSet(); }
if (is_broken)
{ throw UndefinedSymbolPairsFound(); }
- transducer = CALL_MEMBER_FN(transducer,p)();
- transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
+ transducer = CALL_MEMBER_FN(transducer,p)();
+ transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
return *this;
}
OtherSymbolTransducer &OtherSymbolTransducer::apply
(const HfstTransducerOneArgMember p,const OtherSymbolTransducer &another)
-{
+{
if (symbol_pairs.empty())
{ throw EmptySymbolPairSet(); }
if (is_broken)
@@ -352,14 +354,14 @@ OtherSymbolTransducer &OtherSymbolTransducer::apply
harmonize_diacritics(another_copy);
another_copy.harmonize_diacritics(*this);
}
- transducer = CALL_MEMBER_FN(transducer,p)(another_copy.transducer);
- transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
+ transducer = CALL_MEMBER_FN(transducer,p)(another_copy.transducer);
+ transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
return *this;
}
OtherSymbolTransducer &OtherSymbolTransducer::apply
(const HfstTransducerBoolArgMember p,const OtherSymbolTransducer &another)
-{
+{
if (symbol_pairs.empty())
{ throw EmptySymbolPairSet(); }
if (is_broken)
@@ -372,15 +374,15 @@ OtherSymbolTransducer &OtherSymbolTransducer::apply
harmonize_diacritics(another_copy);
another_copy.harmonize_diacritics(*this);
}
- transducer = CALL_MEMBER_FN(transducer,p)(another_copy.transducer, true);
- transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
+ transducer = CALL_MEMBER_FN(transducer,p)(another_copy.transducer, true);
+ transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
return *this;
}
bool OtherSymbolTransducer::apply
(const HfstTransducerOneArgMemberBool p,const OtherSymbolTransducer &another)
const
-{
+{
if (symbol_pairs.empty())
{ throw EmptySymbolPairSet(); }
if (is_broken)
@@ -389,59 +391,59 @@ const
{ throw UndefinedSymbolPairsFound(); }
OtherSymbolTransducer copy(*this);
OtherSymbolTransducer another_copy(another);
- return CALL_MEMBER_FN(copy.transducer,p)(another_copy.transducer);
+ return CALL_MEMBER_FN(copy.transducer,p)(another_copy.transducer);
}
OtherSymbolTransducer &OtherSymbolTransducer::apply
(const HfstTransducerOneNumArgMember p,unsigned int num)
-{
+{
if (symbol_pairs.empty())
{ throw EmptySymbolPairSet(); }
if (is_broken)
{ throw UndefinedSymbolPairsFound(); }
- transducer = CALL_MEMBER_FN(transducer,p)(num);
- transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
+ transducer = CALL_MEMBER_FN(transducer,p)(num);
+ transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
return *this;
}
OtherSymbolTransducer &OtherSymbolTransducer::apply
(const HfstTransducerTwoNumArgMember p,unsigned int num1,unsigned int num2)
-{
+{
if (symbol_pairs.empty())
{ throw EmptySymbolPairSet(); }
if (is_broken)
{ throw UndefinedSymbolPairsFound(); }
- transducer = CALL_MEMBER_FN(transducer,p)(num1,num2);
- transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
+ transducer = CALL_MEMBER_FN(transducer,p)(num1,num2);
+ transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
return *this;
}
OtherSymbolTransducer &OtherSymbolTransducer::apply
(const HfstTransducerOneSymbolPairArgMember p,const SymbolPair &pair)
-{
+{
if (symbol_pairs.empty())
{ throw EmptySymbolPairSet(); }
if (is_broken)
{ throw UndefinedSymbolPairsFound(); }
- transducer = CALL_MEMBER_FN(transducer,p)(pair);
- transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
+ transducer = CALL_MEMBER_FN(transducer,p)(pair);
+ transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
return *this;
}
OtherSymbolTransducer &OtherSymbolTransducer::apply
(const HfstTransducerOneSymbolPairBoolArgMember p,const SymbolPair &pair, bool b)
-{
+{
if (symbol_pairs.empty())
{ throw EmptySymbolPairSet(); }
if (is_broken)
{ throw UndefinedSymbolPairsFound(); }
- transducer = CALL_MEMBER_FN(transducer,p)(pair,b);
- transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
+ transducer = CALL_MEMBER_FN(transducer,p)(pair,b);
+ transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
return *this;
}
OtherSymbolTransducer &OtherSymbolTransducer::apply
-(const HfstTransducerSubstMember p,const std::string &str1,
+(const HfstTransducerSubstMember p,const std::string &str1,
const std::string &str2, bool b1, bool b2)
{
if (symbol_pairs.empty())
@@ -449,12 +451,12 @@ OtherSymbolTransducer &OtherSymbolTransducer::apply
if (is_broken)
{ throw UndefinedSymbolPairsFound(); }
transducer = CALL_MEMBER_FN(transducer,p)(str1,str2,b1,b2);
- transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
+ transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
return *this;
}
OtherSymbolTransducer &OtherSymbolTransducer::apply
-(const HfstTransducerSubstPairMember p,const SymbolPair &p1,
+(const HfstTransducerSubstPairMember p,const SymbolPair &p1,
const SymbolPair &p2)
{
if (symbol_pairs.empty())
@@ -462,12 +464,12 @@ OtherSymbolTransducer &OtherSymbolTransducer::apply
if (is_broken)
{ throw UndefinedSymbolPairsFound(); }
transducer = CALL_MEMBER_FN(transducer,p)(p1,p2);
- transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
+ transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
return *this;
}
OtherSymbolTransducer &OtherSymbolTransducer::apply
-(const HfstTransducerSubstPairFstMember p,const SymbolPair &p1,
+(const HfstTransducerSubstPairFstMember p,const SymbolPair &p1,
const OtherSymbolTransducer &t, bool b)
{
if (symbol_pairs.empty())
@@ -477,7 +479,7 @@ OtherSymbolTransducer &OtherSymbolTransducer::apply
OtherSymbolTransducer t_copy(t);
transducer = CALL_MEMBER_FN(transducer,p)(p1,t_copy.transducer,b);
- transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
+ transducer = CALL_MEMBER_FN(transducer,&HfstTransducer::minimize)();
return *this;
}
@@ -489,9 +491,9 @@ void OtherSymbolTransducer::add_transition
bool OtherSymbolTransducer::has_symbol
(const HfstBasicTransducer &t,const std::string &sym)
-{
+{
const std::set<std::string> &s = t.get_alphabet();
- return s.find(sym) != s.end();
+ return s.find(sym) != s.end();
}
void OtherSymbolTransducer::set_final
@@ -520,49 +522,49 @@ OtherSymbolTransducer OtherSymbolTransducer::get_inverse_of_upper_projection
HfstBasicTransducer new_fst;
HfstState state=0;
- for (HfstBasicTransducer::const_iterator it = fst.begin();
- it != fst.end();
+ for (HfstBasicTransducer::const_iterator it = fst.begin();
+ it != fst.end();
++it)
{
new_fst.add_state(state);
if (fst.is_final_state(state))
{ new_fst.set_final_weight(state,fst.get_final_weight(state)); }
- for (HfstBasicTransducer::HfstTransitions::const_iterator jt
+ for (hfst::implementations::HfstBasicTransitions::const_iterator jt
= it->begin();
jt != it->end();
++jt)
{
HfstBasicTransition arc = *jt;
std::string input = arc.get_transition_data().get_input_symbol();
- std::string output = arc.get_transition_data().get_output_symbol();
+ std::string output = arc.get_transition_data().get_output_symbol();
HfstState target = arc.get_target_state();
if (input == HFST_UNKNOWN)
{
- add_transition(new_fst,state,target,HFST_UNKNOWN,HFST_UNKNOWN);
- for (HandySet<std::string>::const_iterator kt =
+ add_transition(new_fst,state,target,HFST_UNKNOWN,HFST_UNKNOWN);
+ for (HandySet<std::string>::const_iterator kt =
output_symbols.begin();
kt != output_symbols.end(); ++kt)
- {
+ {
if (has_symbol(fst,*kt))
{ add_transition(new_fst,state,target,HFST_UNKNOWN,*kt); }
- }
+ }
}
else
{
HfstBasicTransition arc = *jt;
add_transition
- (new_fst,state,arc.get_target_state(),input,output);
- for (HandySet<SymbolPair>::const_iterator kt =
+ (new_fst,state,arc.get_target_state(),input,output);
+ for (HandySet<SymbolPair>::const_iterator kt =
symbol_pairs.begin();
kt != symbol_pairs.end(); ++kt)
{
if (kt->first == input && has_symbol(fst,kt->second))
{ add_transition(new_fst,state,target,input,kt->second); }
- }
+ }
if (input == TWOLC_EPSILON)
- {
- add_transition(new_fst,state,target,HFST_EPSILON,HFST_EPSILON);
- add_transition(new_fst,state,target,TWOLC_EPSILON,HFST_UNKNOWN);
+ {
+ add_transition(new_fst,state,target,HFST_EPSILON,HFST_EPSILON);
+ add_transition(new_fst,state,target,TWOLC_EPSILON,HFST_UNKNOWN);
}
else if (input != TWOLC_DIAMOND)
{ add_transition(new_fst,state,target,input,HFST_UNKNOWN); }
@@ -620,12 +622,12 @@ OtherSymbolTransducer &OtherSymbolTransducer::term_complemented(void)
}
HfstTransducer OtherSymbolTransducer::get_transducer(void) const
-{
-#ifndef TEST_OTHER_SYMBOL_TRANSDUCER
+{
+#ifndef TEST_OTHER_SYMBOL_TRANSDUCER
if (is_broken)
{ throw UndefinedSymbolPairsFound(); }
#endif
- return transducer;
+ return transducer;
}
void OtherSymbolTransducer::get_initial_transition_pairs
@@ -636,13 +638,13 @@ void OtherSymbolTransducer::get_initial_transition_pairs
HfstBasicTransducer fst(this->transducer);
HfstBasicTransducer::const_iterator start_state_it = fst.begin();
- for (HfstBasicTransducer::HfstTransitions::const_iterator jt
+ for (hfst::implementations::HfstBasicTransitions::const_iterator jt
= start_state_it->begin();
jt != start_state_it->end();
++jt)
{
std::string input = jt->get_transition_data().get_input_symbol();
- std::string output = jt->get_transition_data().get_output_symbol();
+ std::string output = jt->get_transition_data().get_output_symbol();
pair_container.push_back(SymbolPair(input,output));
}
}
@@ -656,11 +658,11 @@ bool have_common_string(HfstState state1,HfstState state2,
if (fst1.is_final_state(state1) && fst2.is_final_state(state2))
{ return true; }
- const HfstBasicTransducer::HfstTransitions &fst1_transitions = fst1[state1];
- const HfstBasicTransducer::HfstTransitions &fst2_transitions = fst2[state2];
+ const hfst::implementations::HfstBasicTransitions &fst1_transitions = fst1[state1];
+ const hfst::implementations::HfstBasicTransitions &fst2_transitions = fst2[state2];
HandyMap<SymbolPair,HfstState> fst1_transition_map;
- for (HfstBasicTransducer::HfstTransitions::const_iterator it =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator it =
fst1_transitions.begin();
it != fst1_transitions.end();
++it)
@@ -668,7 +670,7 @@ bool have_common_string(HfstState state1,HfstState state2,
it->get_output_symbol())] =
it->get_target_state(); }
- for (HfstBasicTransducer::HfstTransitions::const_iterator it =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator it =
fst2_transitions.begin();
it != fst2_transitions.end();
++it)
@@ -716,10 +718,10 @@ bool OtherSymbolTransducer::is_subset(const OtherSymbolTransducer &another)
bool OtherSymbolTransducer::empty(const HfstBasicTransducer &fsm)
{
HfstState state=0;
- for (HfstBasicTransducer::const_iterator it = fsm.begin();
- it != fsm.end();
+ for (HfstBasicTransducer::const_iterator it = fsm.begin();
+ it != fsm.end();
++it)
- {
+ {
if (fsm.is_final_state(state))
{ return false; }
state++;
@@ -733,6 +735,8 @@ bool OtherSymbolTransducer::is_empty(void) const
std::ostream &operator<<(std::ostream &out,const OtherSymbolTransducer &o)
{ return out << "FST:" << std::endl << o.transducer; }
+#endif // #ifndef TEST_OTHER_SYMBOL_TRANSDUCER
+
#ifdef TEST_OTHER_SYMBOL_TRANSDUCER
#include <cassert>
#include <iostream>
@@ -741,24 +745,24 @@ int main(void)
{
bool have_openfst = false;
#if HAVE_OPENFST
- have_openfst = true;
+ have_openfst = true;
#endif // HAVE_OPENFST
bool have_sfst = false;
#if HAVE_SFST
- have_sfst = true;
+ have_sfst = true;
#endif // HAVE_SFST
bool have_foma = false;
#if HAVE_FOMA
- have_foma = true;
+ have_foma = true;
#endif // HAVE_FOMA
-ImplementationType transducer_type
-= (have_openfst ? hfst::TROPICAL_OPENFST_TYPE :
+ImplementationType transducer_type
+= (have_openfst ? hfst::TROPICAL_OPENFST_TYPE :
have_sfst ? hfst::SFST_TYPE :
- have_foma ? hfst::FOMA_TYPE :
+ have_foma ? hfst::FOMA_TYPE :
hfst::ERROR_TYPE);
OtherSymbolTransducer::set_transducer_type(transducer_type);
diff --git a/tools/src/hfst-twolc/src/rule_src/OtherSymbolTransducer.h b/libhfst/src/parsers/rule_src/OtherSymbolTransducer.h
similarity index 96%
rename from tools/src/hfst-twolc/src/rule_src/OtherSymbolTransducer.h
rename to libhfst/src/parsers/rule_src/OtherSymbolTransducer.h
index a519dbc..5b77826 100644
--- a/tools/src/hfst-twolc/src/rule_src/OtherSymbolTransducer.h
+++ b/libhfst/src/parsers/rule_src/OtherSymbolTransducer.h
@@ -24,7 +24,7 @@
# include <config.h>
#endif
-#include "../../../../../libhfst/src/HfstTransducer.h"
+#include "../../../../libhfst/src/HfstTransducer.h"
#include "HfstTwolcDefs.h"
#include "grammar_defs.h"
#include "../string_src/string_manipulation.h"
@@ -58,7 +58,7 @@ static const std::string TWOLC_DIAMOND = "__HFST_TWOLC_DIAMOND";
static const std::string TWOLC_EPSILON = "__HFST_TWOLC_0";
static const std::string TWOLC_FREELY_INSERT = "__HFST_TWOLC_FREELY_INSERT";
-//! @brief Pointer to a member function of @a HfstTransducer taking zero
+//! @brief Pointer to a member function of @a HfstTransducer taking zero
//! arguments.
typedef HfstTransducer &(HfstTransducer::*HfstTransducerZeroArgMember) (void);
@@ -110,7 +110,7 @@ typedef HfstTransducer &(HfstTransducer::*HfstTransducerSubstPairMember)
typedef HfstTransducer &(HfstTransducer::*HfstTransducerSubstPairFstMember)
(const SymbolPair &,HfstTransducer &,bool);
-//! @brief For calling a pointer to a member function passed as argument.
+//! @brief For calling a pointer to a member function passed as argument.
#define CALL_MEMBER_FN(object,ptr_to_member) ((object).*(ptr_to_member))
class Rule;
@@ -163,13 +163,13 @@ class OtherSymbolTransducer
//! @brief Apply @a p onto @a this taking @a another as argument and
//! minimize @a this.
//!
- //! Before applying @a p, harmonize the other-symbols in @a this and
+ //! Before applying @a p, harmonize the other-symbols in @a this and
//! @a another.
- OtherSymbolTransducer
+ OtherSymbolTransducer
&apply(const HfstTransducerOneArgMember,
const OtherSymbolTransducer &another);
- OtherSymbolTransducer
+ OtherSymbolTransducer
&apply(const HfstTransducerBoolArgMember,
const OtherSymbolTransducer &another);
@@ -181,40 +181,40 @@ class OtherSymbolTransducer
//! @brief Apply @a p onto @a this taking @a num as argument and
//! minimize @a this.
- OtherSymbolTransducer
+ OtherSymbolTransducer
&apply(const HfstTransducerOneNumArgMember,unsigned int number);
//! @brief Apply @a p onto @a this taking @a num1 and @a num2 as arguments
//! and minimize @a this.
- OtherSymbolTransducer
+ OtherSymbolTransducer
&apply(const HfstTransducerTwoNumArgMember,unsigned int num1,
unsigned int num2);
//! @brief Apply @a p onto @a this taking @a pair as argument and
//! minimize @a this.
- OtherSymbolTransducer
+ OtherSymbolTransducer
&apply
(const HfstTransducerOneSymbolPairArgMember,const SymbolPair &pair);
- OtherSymbolTransducer
+ OtherSymbolTransducer
&apply
(const HfstTransducerOneSymbolPairBoolArgMember,const SymbolPair &pair, bool b);
- //! @brief Apply @a p onto @a this taking @a str1, @a str2, @a b1 and @a b2
+ //! @brief Apply @a p onto @a this taking @a str1, @a str2, @a b1 and @a b2
//! as arguments and minimize @a this.
- OtherSymbolTransducer
- &apply(const HfstTransducerSubstMember p,const std::string &str1,
+ OtherSymbolTransducer
+ &apply(const HfstTransducerSubstMember p,const std::string &str1,
const std::string &str2, bool b1, bool b2);
//! @brief Apply @a p onto @a this taking @a p1 and @a p2 as arguments and
//! minimize @a this.
- OtherSymbolTransducer
+ OtherSymbolTransducer
&apply(const HfstTransducerSubstPairMember p,const SymbolPair &p1,
const SymbolPair &p2);
//! @brief Apply @a p onto @a this taking @a p1 and @a p2 as arguments and
//! minimize @a this.
- OtherSymbolTransducer
+ OtherSymbolTransducer
&apply(const HfstTransducerSubstPairFstMember p,const SymbolPair &p1,
const OtherSymbolTransducer &t, bool b);
@@ -236,7 +236,7 @@ class OtherSymbolTransducer
//! @brief Set the state @a state final.
static void set_final(HfstBasicTransducer ¢er_t,size_t state);
- //! @brief Return true, iff the intersection of @a this and @a another is
+ //! @brief Return true, iff the intersection of @a this and @a another is
//! empty. If it is non-empty, store the first common string found in v.
bool is_empty_intersection(const OtherSymbolTransducer &another,
StringVector &);
@@ -244,15 +244,15 @@ class OtherSymbolTransducer
//! @brief Return true, iff @a another is a subset of the @a this.
bool is_subset(const OtherSymbolTransducer &another);
- //! @brief Return a copy where all output symbols are replaced
+ //! @brief Return a copy where all output symbols are replaced
//! by other-symbols.
OtherSymbolTransducer get_inverse_of_upper_projection(void);
- //! @brief Perform the effect of <tt>?* X ?*</tt> where <tt>X</tt> is
+ //! @brief Perform the effect of <tt>?* X ?*</tt> where <tt>X</tt> is
//! @a this.
OtherSymbolTransducer &contained(void);
- //! @brief Perform the effect of <tt>?* X ?* - ?* X ?* X ?* </tt> where
+ //! @brief Perform the effect of <tt>?* X ?* - ?* X ?* X ?* </tt> where
//! <tt>X</tt> is @a this.
OtherSymbolTransducer &contained_once(void);
@@ -260,12 +260,12 @@ class OtherSymbolTransducer
//! @a this.
OtherSymbolTransducer &negated(void);
- //! @brief Perform the equivalent of <tt>? - X</tt> where <tt>X</tt> is
+ //! @brief Perform the equivalent of <tt>? - X</tt> where <tt>X</tt> is
//! @a this.
OtherSymbolTransducer &term_complemented(void);
//! @brief Return a rule context <tt>X D ?* D Y</tt> where <tt>X</tt> is the
- //! left context, <tt>Y</tt> is the right context, <tt>D</tt> is the
+ //! left context, <tt>Y</tt> is the right context, <tt>D</tt> is the
//! diamond symbol <tt>__HFST_TWOLC_DIAMOND</tt>.
static OtherSymbolTransducer get_context(OtherSymbolTransducer &left,
OtherSymbolTransducer &right);
@@ -297,7 +297,7 @@ class OtherSymbolTransducer
//! @brief Container for OtherSymbolTransducers.
typedef std::vector<OtherSymbolTransducer> OtherSymbolTransducerVector;
-//! @brief Container for named regular expressions represented as
+//! @brief Container for named regular expressions represented as
//! @a OtherSymbolTransducer objects.
typedef std::map<std::string,OtherSymbolTransducer> NameToRegexMap;
diff --git a/tools/src/hfst-twolc/src/rule_src/RightArrowRule.cc b/libhfst/src/parsers/rule_src/RightArrowRule.cc
similarity index 93%
rename from tools/src/hfst-twolc/src/rule_src/RightArrowRule.cc
rename to libhfst/src/parsers/rule_src/RightArrowRule.cc
index 3257425..332b02b 100644
--- a/tools/src/hfst-twolc/src/rule_src/RightArrowRule.cc
+++ b/libhfst/src/parsers/rule_src/RightArrowRule.cc
@@ -42,23 +42,23 @@ int main(void)
{
bool have_openfst = false;
#if HAVE_OPENFST
- have_openfst = true;
+ have_openfst = true;
#endif // HAVE_OPENFST
bool have_sfst = false;
#if HAVE_SFST
- have_sfst = true;
+ have_sfst = true;
#endif // HAVE_SFST
bool have_foma = false;
#if HAVE_FOMA
- have_foma = true;
+ have_foma = true;
#endif // HAVE_FOMA
-ImplementationType transducer_type
-= have_openfst ? hfst::TROPICAL_OPENFST_TYPE :
+ImplementationType transducer_type
+= have_openfst ? hfst::TROPICAL_OPENFST_TYPE :
have_sfst ? hfst::SFST_TYPE :
- have_foma ? hfst::FOMA_TYPE :
+ have_foma ? hfst::FOMA_TYPE :
hfst::ERROR_TYPE;
OtherSymbolTransducer::set_transducer_type(transducer_type);
@@ -80,7 +80,7 @@ ImplementationType transducer_type
apply(&HfstTransducer::concatenate,unknown);
OtherSymbolTransducer b_c_pair("b","c");
OtherSymbolTransducer context = unknown;
- context.
+ context.
apply(&HfstTransducer::concatenate,b_c_pair).
apply(&HfstTransducer::concatenate,diamond).
apply(&HfstTransducer::concatenate,unknown).
diff --git a/tools/src/hfst-twolc/src/rule_src/RightArrowRule.h b/libhfst/src/parsers/rule_src/RightArrowRule.h
similarity index 100%
rename from tools/src/hfst-twolc/src/rule_src/RightArrowRule.h
rename to libhfst/src/parsers/rule_src/RightArrowRule.h
diff --git a/tools/src/hfst-twolc/src/rule_src/RightArrowRuleContainer.cc b/libhfst/src/parsers/rule_src/RightArrowRuleContainer.cc
similarity index 86%
rename from tools/src/hfst-twolc/src/rule_src/RightArrowRuleContainer.cc
rename to libhfst/src/parsers/rule_src/RightArrowRuleContainer.cc
index d670ae5..aef9374 100644
--- a/tools/src/hfst-twolc/src/rule_src/RightArrowRuleContainer.cc
+++ b/libhfst/src/parsers/rule_src/RightArrowRuleContainer.cc
@@ -29,29 +29,29 @@ void RightArrowRuleContainer::add_rule_and_display_and_resolve_conflicts
(ConflictResolvingRightArrowRule * rule,std::ostream &out)
{
if (center_to_rule_map.has_key(rule->center_pair))
- {
+ {
if (report_right_arrow_conflicts)
- {
+ {
out << "There is a =>-rule conflict between "
<< Rule::get_print_name
- (center_to_rule_map[rule->center_pair]->name)
- << " and " << Rule::get_print_name(rule->name) << "."
+ (center_to_rule_map[rule->center_pair]->name)
+ << " and " << Rule::get_print_name(rule->name) << "."
<< std::endl
- << "Resolving the conflict by joining contexts."
+ << "Resolving the conflict by joining contexts."
<< std::endl << std::endl;
}
if (resolve_right_arrow_conflicts)
- {
- center_to_rule_map[rule->center_pair]->resolve_conflict(*rule);
+ {
+ center_to_rule_map[rule->center_pair]->resolve_conflict(*rule);
rule->is_empty = true;
}
else
{ rule_vector.push_back(rule); }
}
else
- {
- center_to_rule_map[rule->center_pair] = rule;
+ {
+ center_to_rule_map[rule->center_pair] = rule;
rule_vector.push_back(rule);
}
}
diff --git a/tools/src/hfst-twolc/src/rule_src/RightArrowRuleContainer.h b/libhfst/src/parsers/rule_src/RightArrowRuleContainer.h
similarity index 95%
rename from tools/src/hfst-twolc/src/rule_src/RightArrowRuleContainer.h
rename to libhfst/src/parsers/rule_src/RightArrowRuleContainer.h
index ed946e9..273e67a 100644
--- a/tools/src/hfst-twolc/src/rule_src/RightArrowRuleContainer.h
+++ b/libhfst/src/parsers/rule_src/RightArrowRuleContainer.h
@@ -33,7 +33,7 @@ class RightArrowRuleContainer : public RuleContainer
protected:
static bool report_right_arrow_conflicts;
static bool resolve_right_arrow_conflicts;
- typedef HandyMap<SymbolPair,ConflictResolvingRightArrowRule*>
+ typedef HandyMap<SymbolPair,ConflictResolvingRightArrowRule*>
CenterToRuleMap;
CenterToRuleMap center_to_rule_map;
public:
diff --git a/tools/src/hfst-twolc/src/rule_src/Rule.cc b/libhfst/src/parsers/rule_src/Rule.cc
similarity index 94%
rename from tools/src/hfst-twolc/src/rule_src/Rule.cc
rename to libhfst/src/parsers/rule_src/Rule.cc
index 26bde0a..8d84f06 100644
--- a/tools/src/hfst-twolc/src/rule_src/Rule.cc
+++ b/libhfst/src/parsers/rule_src/Rule.cc
@@ -23,7 +23,7 @@ Rule::Rule(const std::string &name,
name(unescape_name(name)),
center(center)
{
- OtherSymbolTransducerVector contexts_copy = contexts;
+ OtherSymbolTransducerVector contexts_copy = contexts;
for (OtherSymbolTransducerVector::iterator it = contexts_copy.begin();
it != contexts_copy.end();
++it)
@@ -42,16 +42,16 @@ Rule::Rule(const std::string &name,
it != v.end();
++it)
{
- if (! (*it)->empty())
- {
+ if (! (*it)->empty())
+ {
rule_transducer.apply
- (&HfstTransducer::intersect,(*it)->rule_transducer);
+ (&HfstTransducer::intersect,(*it)->rule_transducer);
is_empty = false;
}
}
}
-Rule::~Rule(void)
+Rule::~Rule(void)
{}
std::string Rule::get_print_name(const std::string &s)
@@ -172,21 +172,21 @@ OtherSymbolTransducer Rule::get_center
apply(&HfstTransducer::concatenate,diamond).
apply(&HfstTransducer::concatenate,restricted_center).
apply(&HfstTransducer::concatenate,diamond).
- apply(&HfstTransducer::concatenate,unknown);
+ apply(&HfstTransducer::concatenate,unknown);
return center;
}
void Rule::add_missing_symbols_freely(const SymbolRange &diacritics)
{
- std::set<std::string> symbol_set =
+ std::set<std::string> symbol_set =
HfstBasicTransducer(rule_transducer.get_transducer()).get_alphabet();
for (SymbolRange::const_iterator it = diacritics.begin();
it != diacritics.end();
++it)
- {
+ {
if (symbol_set.find(*it) == symbol_set.end())
- {
- rule_transducer.add_symbol_to_alphabet(*it);
+ {
+ rule_transducer.add_symbol_to_alphabet(*it);
rule_transducer.apply(&HfstTransducer::insert_freely,
SymbolPair(*it,*it), true); }
}
diff --git a/tools/src/hfst-twolc/src/rule_src/Rule.h b/libhfst/src/parsers/rule_src/Rule.h
similarity index 97%
rename from tools/src/hfst-twolc/src/rule_src/Rule.h
rename to libhfst/src/parsers/rule_src/Rule.h
index 4b1f5a8..d7fa700 100644
--- a/tools/src/hfst-twolc/src/rule_src/Rule.h
+++ b/libhfst/src/parsers/rule_src/Rule.h
@@ -50,7 +50,7 @@ class Rule
const OtherSymbolTransducer ¢er,
const OtherSymbolTransducerVector &contexts);
- //! @brief Construct a rule which is equivalent to the intersection of the
+ //! @brief Construct a rule which is equivalent to the intersection of the
//! rules in @a v.
Rule(const std::string &name,
const RuleVector &v);
@@ -82,13 +82,13 @@ class Rule
static OtherSymbolTransducer get_center
(const OtherSymbolTransducer &restricted_center);
- //! Replace "__HFST_TWOLC_SPACE" with " " and replace
+ //! Replace "__HFST_TWOLC_SPACE" with " " and replace
//! "__HFST_TWOLC_RULE_NAME=" with "".
static std::string get_print_name(const std::string &s);
//! Freely add symbols in @a diacritics, which do not occur in the alphabet
//! of this rule.
- void add_missing_symbols_freely(const SymbolRange &diacritics);
+ void add_missing_symbols_freely(const SymbolRange &diacritics);
};
diff --git a/tools/src/hfst-twolc/src/rule_src/RuleContainer.cc b/libhfst/src/parsers/rule_src/RuleContainer.cc
similarity index 95%
rename from tools/src/hfst-twolc/src/rule_src/RuleContainer.cc
rename to libhfst/src/parsers/rule_src/RuleContainer.cc
index 91d25fc..06f8b15 100644
--- a/tools/src/hfst-twolc/src/rule_src/RuleContainer.cc
+++ b/libhfst/src/parsers/rule_src/RuleContainer.cc
@@ -40,9 +40,9 @@ void RuleContainer::compile(std::ostream &msg_out,bool be_verbose)
++it)
{
if (be_verbose)
- { msg_out << "Compiling " << Rule::get_print_name((*it)->get_name())
- << std::endl; }
- (*it)->compile();
+ { msg_out << "Compiling " << Rule::get_print_name((*it)->get_name())
+ << std::endl; }
+ (*it)->compile();
}
}
@@ -52,11 +52,11 @@ void RuleContainer::store
for (RuleVector::iterator it = rule_vector.begin();
it != rule_vector.end();
++it)
- {
+ {
if (be_verbose)
- { msg_out << "Storing " << Rule::get_print_name((*it)->get_name())
+ { msg_out << "Storing " << Rule::get_print_name((*it)->get_name())
<< std::endl; }
- (*it)->store(out);
+ (*it)->store(out);
}
}
diff --git a/tools/src/hfst-twolc/src/rule_src/RuleContainer.h b/libhfst/src/parsers/rule_src/RuleContainer.h
similarity index 100%
rename from tools/src/hfst-twolc/src/rule_src/RuleContainer.h
rename to libhfst/src/parsers/rule_src/RuleContainer.h
diff --git a/tools/src/hfst-twolc/src/rule_src/TwolCGrammar.cc b/libhfst/src/parsers/rule_src/TwolCGrammar.cc
similarity index 95%
rename from tools/src/hfst-twolc/src/rule_src/TwolCGrammar.cc
rename to libhfst/src/parsers/rule_src/TwolCGrammar.cc
index 6d94a84..19ded10 100644
--- a/tools/src/hfst-twolc/src/rule_src/TwolCGrammar.cc
+++ b/libhfst/src/parsers/rule_src/TwolCGrammar.cc
@@ -36,8 +36,8 @@ TwolCGrammar::TwolCGrammar(bool be_quiet,
}
void TwolCGrammar::define_diacritics(const SymbolRange &diacritics)
-{
- this->diacritics = diacritics;
+{
+ this->diacritics = diacritics;
OtherSymbolTransducer::define_diacritics(diacritics);
}
@@ -57,21 +57,21 @@ void TwolCGrammar::add_rule(const std::string &name,
case op::LEFT:
rule = new ConflictResolvingLeftArrowRule(name,center,contexts);
left_arrow_rule_container.add_rule_and_display_and_resolve_conflicts
- (static_cast<ConflictResolvingLeftArrowRule*>(rule),std::cerr);
+ (static_cast<ConflictResolvingLeftArrowRule*>(rule),std::cerr);
break;
case op::LEFT_RIGHT:
rule = new ConflictResolvingRightArrowRule(name,center,contexts);
right_arrow_rule_container.add_rule_and_display_and_resolve_conflicts
- (static_cast<ConflictResolvingRightArrowRule*>(rule),std::cerr);
+ (static_cast<ConflictResolvingRightArrowRule*>(rule),std::cerr);
name_to_rule_subcases[get_original_name(name)].insert(rule);
rule = new ConflictResolvingLeftArrowRule(name,center,contexts);
left_arrow_rule_container.add_rule_and_display_and_resolve_conflicts
- (static_cast<ConflictResolvingLeftArrowRule*>(rule),std::cerr);
+ (static_cast<ConflictResolvingLeftArrowRule*>(rule),std::cerr);
break;
case op::NOT_LEFT:
rule = new LeftRestrictionArrowRule(name,center,contexts);
other_rule_container.add_rule
- (static_cast<LeftRestrictionArrowRule*>(rule));
+ (static_cast<LeftRestrictionArrowRule*>(rule));
break;
default:
assert(false);
@@ -134,7 +134,7 @@ void TwolCGrammar::add_rule(const std::string &name,
{
Rule * rule;
- std::string center_name =
+ std::string center_name =
name + " CENTER=" + it->first + ":" + it->second;
switch (oper)
@@ -147,22 +147,22 @@ void TwolCGrammar::add_rule(const std::string &name,
case op::LEFT:
rule = new ConflictResolvingLeftArrowRule(center_name,*it,contexts);
left_arrow_rule_container.add_rule_and_display_and_resolve_conflicts
- (static_cast<ConflictResolvingLeftArrowRule*>(rule),std::cerr);
+ (static_cast<ConflictResolvingLeftArrowRule*>(rule),std::cerr);
break;
case op::LEFT_RIGHT:
rule = new ConflictResolvingRightArrowRule(center_name,*it,contexts);
right_arrow_rule_container.add_rule_and_display_and_resolve_conflicts
- (static_cast<ConflictResolvingRightArrowRule*>(rule),std::cerr);
+ (static_cast<ConflictResolvingRightArrowRule*>(rule),std::cerr);
name_to_rule_subcases[get_original_name(center_name)].insert(rule);
rule = new ConflictResolvingLeftArrowRule(center_name,*it,contexts);
left_arrow_rule_container.add_rule_and_display_and_resolve_conflicts
- (static_cast<ConflictResolvingLeftArrowRule*>(rule),std::cerr);
+ (static_cast<ConflictResolvingLeftArrowRule*>(rule),std::cerr);
break;
case op::NOT_LEFT:
rule = new LeftRestrictionArrowRule(center_name,*it,contexts);
other_rule_container.add_rule
- (static_cast<LeftRestrictionArrowRule*>(rule));
+ (static_cast<LeftRestrictionArrowRule*>(rule));
break;
default:
assert(false);
@@ -203,23 +203,23 @@ int main(void)
bool have_openfst = false;
#if HAVE_OPENFST
- have_openfst = true;
+ have_openfst = true;
#endif // HAVE_OPENFST
bool have_sfst = false;
#if HAVE_SFST
- have_sfst = true;
+ have_sfst = true;
#endif // HAVE_SFST
bool have_foma = false;
#if HAVE_FOMA
- have_foma = true;
+ have_foma = true;
#endif // HAVE_FOMA
-ImplementationType transducer_type
-= have_openfst ? hfst::TROPICAL_OPENFST_TYPE :
+ImplementationType transducer_type
+= have_openfst ? hfst::TROPICAL_OPENFST_TYPE :
have_sfst ? hfst::SFST_TYPE :
- have_foma ? hfst::FOMA_TYPE :
+ have_foma ? hfst::FOMA_TYPE :
hfst::ERROR_TYPE;
OtherSymbolTransducer::set_transducer_type(transducer_type);
@@ -247,7 +247,7 @@ ImplementationType transducer_type
OtherSymbolTransducer b_c_pair("b","c");
OtherSymbolTransducer context = unknown;
- context.
+ context.
apply(&HfstTransducer::concatenate,b_c_pair).
apply(&HfstTransducer::concatenate,diamond).
apply(&HfstTransducer::concatenate,unknown).
@@ -262,7 +262,7 @@ ImplementationType transducer_type
OtherSymbolTransducer a_d_pair("a","d");
OtherSymbolTransducer context1 = unknown;
- context1.
+ context1.
//apply(&HfstTransducer::concatenate,a_d_pair).
apply(&HfstTransducer::concatenate,b_c_pair).
apply(&HfstTransducer::concatenate,diamond).
diff --git a/tools/src/hfst-twolc/src/rule_src/TwolCGrammar.h b/libhfst/src/parsers/rule_src/TwolCGrammar.h
similarity index 94%
rename from tools/src/hfst-twolc/src/rule_src/TwolCGrammar.h
rename to libhfst/src/parsers/rule_src/TwolCGrammar.h
index 895996c..b9f7504 100644
--- a/tools/src/hfst-twolc/src/rule_src/TwolCGrammar.h
+++ b/libhfst/src/parsers/rule_src/TwolCGrammar.h
@@ -33,15 +33,15 @@
namespace op
{
enum OPERATOR
- {
- RIGHT,
- LEFT,
- NOT_LEFT,
- LEFT_RIGHT,
- RE_RIGHT,
- RE_LEFT,
- RE_NOT_LEFT,
- RE_LEFT_RIGHT
+ {
+ RIGHT,
+ LEFT,
+ NOT_LEFT,
+ LEFT_RIGHT,
+ RE_RIGHT,
+ RE_LEFT,
+ RE_NOT_LEFT,
+ RE_LEFT_RIGHT
};
}
diff --git a/libhfst/src/parsers/sfst-compiler.yy b/libhfst/src/parsers/sfst-compiler.yy
new file mode 100644
index 0000000..1c6533d
--- /dev/null
+++ b/libhfst/src/parsers/sfst-compiler.yy
@@ -0,0 +1,225 @@
+%{
+/*******************************************************************/
+/* */
+/* FILE sfst-compiler.yy */
+/* MODULE sfst-compiler */
+/* PROGRAM HFST */
+/* */
+/*******************************************************************/
+
+#ifdef WINDOWS
+#include <io.h>
+#endif
+
+#include "SfstCompiler.h"
+#include "HfstTransducer.h"
+
+extern int sfstlineno;
+extern char *sfsttext;
+int sfstlex( void );
+
+using namespace hfst;
+using std::cerr;
+
+namespace hfst
+{
+ extern SfstCompiler * sfst_compiler;
+}
+
+bool DEBUG = false;
+
+void sfsterror(char *text)
+
+{
+ cerr << "\n" << sfst_compiler->filename << ":" << sfstlineno << ": " << text << " at: ";
+ cerr << sfsttext << "\naborted.\n";
+ throw HfstException();
+}
+
+void warn(char *text)
+
+{
+ cerr << "\n" << sfst_compiler->filename << ":" << sfstlineno << ": warning: " << text << "!\n";
+}
+
+void warn2(const char *text, char *text2)
+
+{
+ cerr << "\n" << sfst_compiler->filename << ":" << sfstlineno << ": warning: " << text << ": ";
+ cerr << text2 << "\n";
+}
+
+%}
+
+%name-prefix="sfst"
+
+%union {
+ int number;
+ hfst::Twol_Type type;
+ hfst::Repl_Type rtype;
+ char *name;
+ char *value;
+ unsigned char uchar;
+ unsigned int longchar;
+ hfst::Character character;
+ hfst::HfstTransducer *expression;
+ hfst::Range *range;
+ hfst::Ranges *ranges;
+ hfst::Contexts *contexts;
+}
+
+%token <number> NEWLINE ALPHA COMPOSE PRINT POS INSERT SUBSTITUTE SWITCH
+%token <type> ARROW
+%token <rtype> REPLACE
+%token <name> SYMBOL VAR SVAR RVAR RSVAR
+%token <value> STRING STRING2 UTF8CHAR
+%token <uchar> CHARACTER
+
+%type <uchar> SCHAR
+%type <longchar> LCHAR
+%type <character> CODE
+%type <expression> RE
+%type <range> RANGE VALUE VALUES
+%type <ranges> RANGES
+%type <contexts> CONTEXT CONTEXT2 CONTEXTS CONTEXTS2
+
+%left PRINT INSERT SUBSTITUTE
+%left ARROW REPLACE
+%left COMPOSE
+%left '|'
+%left '-'
+%left '&'
+%left SEQ
+%left '!' '^' '_'
+%left '*' '+'
+%%
+
+ALL: ASSIGNMENTS RE NEWLINES { sfst_compiler->set_result(sfst_compiler->result($2, sfst_compiler->switch_)); }
+ ;
+
+ASSIGNMENTS: ASSIGNMENTS ASSIGNMENT {}
+ | ASSIGNMENTS NEWLINE {}
+ | /* nothing */ {}
+ ;
+
+ASSIGNMENT: VAR '=' RE { if (DEBUG) { printf("defining transducer variable \"%s\"..\n", $1); }; if (sfst_compiler->def_var($1,$3)) warn2("assignment of empty transducer to",$1); if(DEBUG) printf("done\n"); }
+ | RVAR '=' RE { if (DEBUG) { printf("defining agreement transducer variable \"%s\"..\n", $1); }; if (sfst_compiler->def_rvar($1,$3)) warn2("assignment of empty transducer to",$1); }
+ | SVAR '=' VALUES { if (DEBUG) { printf("defining range variable \"%s\"..\n", $1); }; if (sfst_compiler->def_svar($1,$3)) warn2("assignment of empty symbol range to",$1); }
+ | RSVAR '=' VALUES { if (DEBUG) { printf("defining agreement range variable \"%s\"..\n", $1); }; if (sfst_compiler->def_svar($1,$3)) warn2("assignment of empty symbol range to",$1); }
+ | RE PRINT STRING { sfst_compiler->write_to_file($1, sfst_compiler->foldername.c_str(), $3); }
+ | ALPHA RE { if (DEBUG) { printf("defining alphabet..\n"); }; sfst_compiler->def_alphabet($2); delete $2; }
+ ;
+
+RE: RE ARROW CONTEXTS2 { $$ = sfst_compiler->restriction($1,$2,$3,0); }
+ | RE '^' ARROW CONTEXTS2 { $$ = sfst_compiler->restriction($1,$3,$4,1); }
+ | RE '_' ARROW CONTEXTS2 { $$ = sfst_compiler->restriction($1,$3,$4,-1); }
+ | RE REPLACE CONTEXT2 { $1 = sfst_compiler->explode($1); $1->minimize(); $$ = sfst_compiler->replace_in_context($1, $2, $3, false); }
+ | RE REPLACE '?' CONTEXT2 { $1 = sfst_compiler->explode($1); $1->minimize(); $$ = sfst_compiler->replace_in_context($1, $2, $4, true); }
+ | RE REPLACE '(' ')' { $1 = sfst_compiler->explode($1); $1->minimize(); $$ = sfst_compiler->replace($1, $2, false); }
+ | RE REPLACE '?' '(' ')' { $1 = sfst_compiler->explode($1); $1->minimize(); $$ = sfst_compiler->replace($1, $2, true); }
+ | RE RANGE ARROW RANGE RE { $$ = sfst_compiler->make_rule($1,$2,$3,$4,$5, sfst_compiler->compiler_type); }
+ | RE RANGE ARROW RANGE { $$ = sfst_compiler->make_rule($1,$2,$3,$4,NULL, sfst_compiler->compiler_type); }
+ | RANGE ARROW RANGE RE { $$ = sfst_compiler->make_rule(NULL,$1,$2,$3,$4, sfst_compiler->compiler_type); }
+ | RANGE ARROW RANGE { $$ = sfst_compiler->make_rule(NULL,$1,$2,$3,NULL, sfst_compiler->compiler_type); }
+ | RE COMPOSE RE { $1->compose(*$3); delete $3; $$ = $1; }
+ | '{' RANGES '}' ':' '{' RANGES '}' { $$ = sfst_compiler->make_mapping($2,$6,sfst_compiler->compiler_type); }
+ | RANGE ':' '{' RANGES '}' { $$ = sfst_compiler->make_mapping(sfst_compiler->add_range($1,NULL),$4,sfst_compiler->compiler_type); }
+ | '{' RANGES '}' ':' RANGE { $$ = sfst_compiler->make_mapping($2,sfst_compiler->add_range($5,NULL),sfst_compiler->compiler_type); }
+ | RE INSERT CODE ':' CODE { $$ = sfst_compiler->insert_freely($1,$3,$5); }
+ | RE INSERT CODE { $$ = sfst_compiler->insert_freely($1,$3,$3); }
+ | RE SUBSTITUTE CODE ':' CODE { $$ = sfst_compiler->substitute($1,$3,$5); }
+ | RE SUBSTITUTE CODE ':' CODE ':' CODE ':' CODE { $$ = sfst_compiler->substitute($1,$3,$5,$7,$9); }
+ | RE SUBSTITUTE CODE ':' CODE '(' RE ')' { $$ = sfst_compiler->substitute($1,$3,$5,$7); }
+ | RANGE ':' RANGE { $$ = sfst_compiler->new_transducer($1,$3,sfst_compiler->compiler_type); }
+ | RANGE { $$ = sfst_compiler->new_transducer($1,$1,sfst_compiler->compiler_type); }
+ | VAR { if (DEBUG) { printf("calling transducer variable \"%s\"\n", $1); }; $$ = sfst_compiler->var_value($1); }
+ | RVAR { if (DEBUG) { printf("calling agreement transducer variable \"%s\"\n", $1); }; $$ = sfst_compiler->rvar_value($1,sfst_compiler->compiler_type); }
+ | RE '*' { $1->repeat_star(); $$ = $1; }
+ | RE '+' { $1->repeat_plus(); $$ = $1; }
+ | RE '?' { $1->optionalize(); $$ = $1; }
+ | RE RE %prec SEQ { $1->concatenate(*$2); delete $2; $$ = $1; }
+ | '!' RE { $$ = sfst_compiler->negation($2); }
+ | SWITCH RE { $2->invert(); $$ = $2; }
+ | '^' RE { $2->output_project(); $$ = $2; }
+ | '_' RE { $2->input_project(); $$ = $2; }
+ | RE '&' RE { $1->intersect(*$3); delete $3; $$ = $1; }
+ | RE '-' RE { $1->subtract(*$3); delete $3; $$ = $1; }
+ | RE '|' RE { $1->disjunct(*$3); delete $3; $$ = $1; }
+ | '(' RE ')' { $$ = $2; }
+ | STRING { $$ = sfst_compiler->read_words(sfst_compiler->foldername.c_str(), $1, sfst_compiler->compiler_type); }
+ | STRING2 { try { $$ = sfst_compiler->read_transducer(sfst_compiler->foldername.c_str(), $1, sfst_compiler->compiler_type); } catch (HfstException e) { printf("\nAn error happened when reading file \"%s\"\n", $1); throw HfstException(); } }
+ ;
+
+RANGES: RANGE RANGES { $$ = sfst_compiler->add_range($1,$2); }
+ | { $$ = NULL; }
+ ;
+
+RANGE: '[' VALUES ']' { $$=$2; }
+ | '[' '^' VALUES ']' { $$=sfst_compiler->complement_range($3); }
+ | '[' RSVAR ']' { if (DEBUG) { printf("calling agreement range variable \"%s\"\n", $2); }; $$=sfst_compiler->rsvar_value($2); }
+ | '.' { $$=NULL; }
+ | CODE { $$=sfst_compiler->add_value($1,NULL); }
+ ;
+
+CONTEXTS2: CONTEXTS { $$ = $1; }
+ | '(' CONTEXTS ')' { $$ = $2; }
+ ;
+
+CONTEXTS: CONTEXT ',' CONTEXTS { $$ = sfst_compiler->add_context($1,$3); }
+ | CONTEXT { $$ = $1; }
+ ;
+
+CONTEXT2: CONTEXT { $$ = $1; }
+ | '(' CONTEXT ')' { $$ = $2; }
+ ;
+
+CONTEXT : RE POS RE { $$ = sfst_compiler->make_context($1, $3); }
+ | POS RE { $$ = sfst_compiler->make_context(NULL, $2); }
+ | RE POS { $$ = sfst_compiler->make_context($1, NULL); }
+ ;
+
+VALUES: VALUE VALUES { $$=sfst_compiler->append_values($1,$2); }
+ | VALUE { $$ = $1; }
+ ;
+
+VALUE: LCHAR '-' LCHAR { $$=sfst_compiler->add_values($1,$3,NULL); }
+ | SVAR { if (DEBUG) { printf("calling range variable \"%s\"", $1); }; $$=sfst_compiler->svar_value($1); }
+ | LCHAR { $$=sfst_compiler->add_value(sfst_compiler->character_code($1),NULL); }
+ | CODE { $$=sfst_compiler->add_value($1,NULL); }
+ | SCHAR { $$=sfst_compiler->add_value($1,NULL); }
+ ;
+
+LCHAR: CHARACTER { $$=$1; }
+ | UTF8CHAR { $$=sfst_compiler->utf8toint($1); free($1); }
+ | SCHAR { $$=$1; }
+ ;
+
+CODE: CHARACTER { $$=sfst_compiler->character_code($1); }
+ | UTF8CHAR { $$=sfst_compiler->symbol_code($1); }
+ | SYMBOL { $$=sfst_compiler->symbol_code($1); }
+ ;
+
+SCHAR: '.' { $$=(unsigned char)sfst_compiler->character_code('.'); }
+ | '!' { $$=(unsigned char)sfst_compiler->character_code('!'); }
+ | '?' { $$=(unsigned char)sfst_compiler->character_code('?'); }
+ | '{' { $$=(unsigned char)sfst_compiler->character_code('{'); }
+ | '}' { $$=(unsigned char)sfst_compiler->character_code('}'); }
+ | ')' { $$=(unsigned char)sfst_compiler->character_code(')'); }
+ | '(' { $$=(unsigned char)sfst_compiler->character_code('('); }
+ | '&' { $$=(unsigned char)sfst_compiler->character_code('&'); }
+ | '|' { $$=(unsigned char)sfst_compiler->character_code('|'); }
+ | '*' { $$=(unsigned char)sfst_compiler->character_code('*'); }
+ | '+' { $$=(unsigned char)sfst_compiler->character_code('+'); }
+ | ':' { $$=(unsigned char)sfst_compiler->character_code(':'); }
+ | ',' { $$=(unsigned char)sfst_compiler->character_code(','); }
+ | '=' { $$=(unsigned char)sfst_compiler->character_code('='); }
+ | '_' { $$=(unsigned char)sfst_compiler->character_code('_'); }
+ | '^' { $$=(unsigned char)sfst_compiler->character_code('^'); }
+ | '-' { $$=(unsigned char)sfst_compiler->character_code('-'); }
+ ;
+
+NEWLINES: NEWLINE NEWLINES {}
+ | /* nothing */ {}
+ ;
+
+%%
diff --git a/libhfst/src/parsers/sfst-scanner.ll b/libhfst/src/parsers/sfst-scanner.ll
new file mode 100644
index 0000000..2badc4b
--- /dev/null
+++ b/libhfst/src/parsers/sfst-scanner.ll
@@ -0,0 +1,202 @@
+%option 8Bit batch yylineno nounput noyywrap prefix="sfst"
+/*header-file="sfst-scanner.h"*/
+
+/* the "incl" state is used to pick up the name of an include file */
+%x incl
+
+%{
+/*******************************************************************/
+/* */
+/* FILE scanner.ll */
+/* MODULE scanner */
+/* PROGRAM HFST */
+/* */
+/*******************************************************************/
+
+#include <string.h>
+
+#include "SfstCompiler.h"
+
+#ifdef YACC_USE_PARSER_H_EXTENSION
+ #include "sfst-compiler.h"
+#else
+ #include "sfst-compiler.hh"
+#endif
+
+#include "SfstBasic.h"
+#include "SfstUtf8.h"
+#include "HfstTransducer.h"
+
+extern void sfsterror(char*);
+
+#undef YY_FATAL_ERROR
+#define YY_FATAL_ERROR(msg) sfsterror(msg);
+
+using namespace hfst;
+using namespace sfst_basic;
+
+namespace hfst
+{
+ extern SfstCompiler * sfst_compiler;
+}
+
+#define MAX_INCLUDE_DEPTH 10
+
+int Include_Stack_Ptr = 0;
+YY_BUFFER_STATE Include_Stack[MAX_INCLUDE_DEPTH];
+char *Name_Stack[MAX_INCLUDE_DEPTH];
+int Lineno_Stack[MAX_INCLUDE_DEPTH];
+
+bool UTF8=true;
+
+static char *unquote(char *string, bool del_quote=true) {
+ char *s=string, *result=string;
+ if (del_quote)
+ string++;
+
+ while (*string) {
+ if (*string == '\\')
+ string++;
+ *(s++) = *(string++);
+ }
+
+ if (del_quote)
+ s--;
+ *s = '\0';
+
+ return fst_strdup(result);
+}
+
+static void print_lineno() {
+ if (!sfst_compiler->Verbose)
+ return;
+ fputc('\r',stderr);
+ for( int i=0; i<Include_Stack_Ptr; i++ )
+ fputs(" ", stderr);
+ fprintf(stderr,"%s: %d", sfst_compiler->filename.c_str(), sfstlineno);
+}
+
+extern void sfsterror(char *text);
+
+%}
+
+CC [\x80-\xbf]
+C1 [A-Za-z0-9._/\-]
+C2 [A-Za-z0-9._/\-&()+,=?\^|~]
+C3 [A-Za-z0-9._/\-&()+,=?\^|~#<>]
+C4 [A-Za-z0-9._/\-&()+,=?\^|~$<>]
+C5 [\!-;\?-\[\]-\~=]
+FN [A-Za-z0-9._/\-*+]
+
+%%
+
+^[ \t]*\#use[ \t]*hopcroft[ \t]*\n { hfst::set_minimization_algorithm(hfst::HOPCROFT);};
+^[ \t]*\#use[ \t]*default[ \t]*\n { hfst::set_minimization_algorithm(hfst::BRZOZOWSKI);};
+
+
+#include BEGIN(incl);
+<incl>[ \t]* /* eat the whitespace */
+<incl>{FN}+ { sfst_compiler->error2("Missing quotes",sfsttext); }
+<incl>\"{FN}+\" { /* got the include file name */
+ FILE *file;
+ char *name=fst_strdup(sfsttext+1);
+ name[strlen(name)-1] = 0;
+ if ( Include_Stack_Ptr >= MAX_INCLUDE_DEPTH ) {
+ fprintf( stderr, "Includes nested too deeply" );
+ throw HfstException();
+ }
+ if (sfst_compiler->Verbose) fputc('\n', stderr);
+ file = fopen( name, "rt" );
+ if (!file)
+ sfst_compiler->error2("Can't open include file", name);
+ else {
+ Name_Stack[Include_Stack_Ptr] = strdup(sfst_compiler->filename.c_str());
+ sfst_compiler->set_filename(std::string(name));
+ Lineno_Stack[Include_Stack_Ptr] = sfstlineno;
+ sfstlineno = 1;
+ Include_Stack[Include_Stack_Ptr++]=YY_CURRENT_BUFFER;
+ sfst_switch_to_buffer(sfst_create_buffer(sfstin, YY_BUF_SIZE));
+ sfstin = file;
+ print_lineno();
+ BEGIN(INITIAL);
+ }
+ }
+<<EOF>> {
+ if (sfst_compiler->Verbose)
+ fputc('\n', stderr);
+ if ( --Include_Stack_Ptr < 0 )
+ yyterminate();
+ else {
+ //free(FileName);
+ sfst_compiler->set_filename(std::string(Name_Stack[Include_Stack_Ptr]));
+ sfstlineno = Lineno_Stack[Include_Stack_Ptr];
+ sfst_delete_buffer( YY_CURRENT_BUFFER );
+ sfst_switch_to_buffer(Include_Stack[Include_Stack_Ptr]);
+ }
+ }
+
+
+^[ \t]*\%.*\r?\n { print_lineno(); /* ignore comments */ }
+
+\%.*\\[ \t]*\r?\n { print_lineno(); /* ignore comments */ }
+
+\%.* { /* ignore comments */ }
+
+
+^[ \t]*ALPHABET[ \t]*= { return ALPHA; }
+
+\|\| { return COMPOSE; }
+"<=>" { sfstlval.type = twol_both; return ARROW; }
+"=>" { sfstlval.type = twol_right;return ARROW; }
+"<=" { sfstlval.type = twol_left; return ARROW; }
+"^->" { sfstlval.rtype = repl_up; return REPLACE; }
+"_->" { sfstlval.rtype = repl_down; return REPLACE; }
+"/->" { sfstlval.rtype = repl_right;return REPLACE; }
+"\\->" { sfstlval.rtype = repl_left; return REPLACE; }
+">>" { return PRINT; }
+"<<" { return INSERT; }
+"<<<" { return SUBSTITUTE; }
+"__" { return POS; }
+"^_" { return SWITCH; }
+
+[.,{}\[\]()&!?|*+:=_\^\-] { return sfsttext[0]; }
+
+\$=({C3}|(\\.))+\$ { sfstlval.name = fst_strdup(sfsttext); return RVAR; }
+
+\$({C3}|(\\.))+\$ { sfstlval.name = fst_strdup(sfsttext); return VAR; }
+
+#=({C4}|(\\.))+# { sfstlval.name = fst_strdup(sfsttext); return RSVAR; }
+
+#({C4}|(\\.))+# { sfstlval.name = fst_strdup(sfsttext); return SVAR; }
+
+\<({C5}|\\.)*\> { sfstlval.name = unquote(sfsttext,false); return SYMBOL; }
+
+\"<{FN}+>\" {
+ sfstlval.value = fst_strdup(sfsttext+2);
+ sfstlval.value[strlen(sfstlval.value)-2] = 0;
+ return STRING2;
+ }
+
+\"{FN}+\" {
+ sfstlval.value = fst_strdup(sfsttext+1);
+ sfstlval.value[strlen(sfstlval.value)-1] = 0;
+ return STRING;
+ }
+
+[ \t] { /* ignored */ }
+\\[ \t]*([ \t]\%.*)?\r?\n { print_lineno(); /* ignored */ }
+\r?\n { print_lineno(); return NEWLINE; }
+
+\\[0-9]+ { long l=atol(sfsttext+1);
+ if (l <= 1114112) { sfstlval.value=fst_strdup(sfst_utf8::int2utf8((unsigned)l)); return UTF8CHAR; }
+ sfsterror(strdup("invalid expression"));
+ }
+
+
+\\. { sfstlval.value=fst_strdup(sfsttext+1); return UTF8CHAR; }
+[\x00-\x7f] { sfstlval.value=fst_strdup(sfsttext); return UTF8CHAR; }
+[\xc0-\xdf]{CC} { sfstlval.value=fst_strdup(sfsttext); return UTF8CHAR; }
+[\xe0-\xef]{CC}{2} { sfstlval.value=fst_strdup(sfsttext); return UTF8CHAR; }
+[\xf0-\xff]{CC}{3} { sfstlval.value=fst_strdup(sfsttext); return UTF8CHAR; }
+
+%%
diff --git a/tools/src/hfst-twolc/src/string_src/string_manipulation.cc b/libhfst/src/parsers/string_src/string_manipulation.cc
similarity index 92%
rename from tools/src/hfst-twolc/src/string_src/string_manipulation.cc
rename to libhfst/src/parsers/string_src/string_manipulation.cc
index b629818..a01cb3b 100644
--- a/tools/src/hfst-twolc/src/string_src/string_manipulation.cc
+++ b/libhfst/src/parsers/string_src/string_manipulation.cc
@@ -34,19 +34,19 @@ std::string remove_sign(std::string str,const std::string &sign)
{ return replace_substr(str,sign,""); }
std::string unescape(std::string str)
-{
+{
if (str.find('\n') != std::string::npos)
{ throw FaultyStringInput("unescape",str ); }
- // Change "%%" to "\n", remove all remaining %'s
+ // Change "%%" to "\n", remove all remaining %'s
// and change all '\n's to '%'.
return
replace_substr(remove_sign(replace_substr(str,"%%","\n"),'%'),"\n","%");
}
-int strcmp_unescaped(const std::string &str1,
+int strcmp_unescaped(const std::string &str1,
const std::string &str2)
{
- // Remove all escapes from str1 and str2 and
+ // Remove all escapes from str1 and str2 and
// compare them.
std::string str1_copy = unescape(str1);
std::string str2_copy = unescape(str2);
@@ -54,16 +54,16 @@ int strcmp_unescaped(const std::string &str1,
}
std::string remove_white_space(std::string str)
-{
+{
if (str.find('\n') != std::string::npos)
{ throw FaultyStringInput("remove_white_space",str); }
- str =
+ str =
replace_substr(remove_sign(replace_substr(str,"% ","\n"),' '),
"\n","__HFST_TWOLC_SPACE");
- str =
+ str =
replace_substr(remove_sign(replace_substr(str,"%\t","\n"),'\t'),
"\n","__HFST_TWOLC_TAB");
- str =
+ str =
replace_substr(remove_sign(replace_substr(str,"%\r","\n"),'\r'),
"\n","__HFST_TWOLC_CR");
str =
@@ -75,16 +75,16 @@ std::string remove_white_space(std::string str)
std::string unescape_and_remove_white_space(std::string str)
{ return unescape(remove_white_space(str)); }
-std::string unquote(const std::string &str)
-{
+std::string unquote(const std::string &str)
+{
if (str.size() < 2 || str[0] != '"' || str[str.size()-1] != '"')
{ throw FaultyStringInput("unquote",str); }
- // Return the substring of str spanning from the
- // second to the next to final character.
- return str.substr(1,str.size()-2);
+ // Return the substring of str spanning from the
+ // second to the next to final character.
+ return str.substr(1,str.size()-2);
}
-int str2int(const std::string &str)
+int str2int(const std::string &str)
{
std::istringstream in(str);
int number;
@@ -129,7 +129,7 @@ StringVector &StringVector::add_values(const StringVector &another)
std::string unescape_name(const std::string &name)
{
- std::string new_name =
+ std::string new_name =
replace_substr(replace_substr(name,"__HFST_TWOLC_RULE_NAME=",""),
"__HFST_TWOLC_SPACE"," ");
return new_name;
@@ -172,22 +172,22 @@ int main(void)
s3 = unquote("\"ab\"");
assert(s3 == "ab");
try
- {
- s3 = unquote("");
+ {
+ s3 = unquote("");
assert(false);
}
catch (const FaultyStringInput &fsi)
{ /* nothing */ }
try
- {
- s3 = unquote("\"");
+ {
+ s3 = unquote("\"");
assert(false);
}
catch (const FaultyStringInput &fsi)
{ /* nothing */ }
try
- {
- s3 = unquote("a");
+ {
+ s3 = unquote("a");
assert(false);
}
catch (const FaultyStringInput &fsi)
@@ -240,7 +240,7 @@ int main(void)
assert(str2int("-1 3") == -1);
assert(str2int("20") == 20);
try
- {
+ {
str2int("");
assert(false);
}
@@ -248,7 +248,7 @@ int main(void)
{ /* nothing */ }
try
- {
+ {
str2int("a");
assert(false);
}
@@ -266,7 +266,7 @@ int main(void)
//assert(remove_white_space("foo% bar%\tbaz%\r") == "foo bar\tbaz\r");
//assert(remove_white_space("foo% \tbar") == "foo bar");
try
- {
+ {
(void)remove_white_space("foo\nbar");
assert(false);
}
@@ -276,7 +276,7 @@ int main(void)
// unescape_and_remove_white_space(...)
//assert(unescape_and_remove_white_space(" foo%%% ") == "foo% ");
try
- {
+ {
(void)unescape_and_remove_white_space("foo\nbar");
assert(false);
}
diff --git a/tools/src/hfst-twolc/src/string_src/string_manipulation.h b/libhfst/src/parsers/string_src/string_manipulation.h
similarity index 94%
rename from tools/src/hfst-twolc/src/string_src/string_manipulation.h
rename to libhfst/src/parsers/string_src/string_manipulation.h
index 2dad808..e3402e3 100644
--- a/tools/src/hfst-twolc/src/string_src/string_manipulation.h
+++ b/libhfst/src/parsers/string_src/string_manipulation.h
@@ -29,14 +29,14 @@
#include <cstdlib>
#include <cstring>
#include <cassert>
-#include <iostream>
+#include <iostream>
#include <vector>
-//! @brief Thrown when a string manipulation function receives incorrect
-//! string input.
+//! @brief Thrown when a string manipulation function receives incorrect
+//! string input.
struct FaultyStringInput
{
- //! @var Name of the function which threw this instance.
+ //! @var Name of the function which threw this instance.
std::string function;
//! @var The incorrect input @function received.
@@ -55,7 +55,7 @@ std::string new_string(size_t lgth);
//! @brief Safe copy of a @a char *. Free using @a free.
char * string_copy(const char * str);
-//! @brief Recursively substitute each occurrence of @a substr with
+//! @brief Recursively substitute each occurrence of @a substr with
//! @a replacement.
//! @param str perform substitution on this string.
//! @param substr the substring to be substituted.
@@ -74,22 +74,22 @@ std::string remove_sign(std::string str,char sign);
//! @brief Return @a str with all occurrences of @a sign removed.
std::string remove_sign(std::string str,const std::string &sign);
-//! @brief For a string enclosed in quotation marks, return the unquoted
-//! string.
+//! @brief For a string enclosed in quotation marks, return the unquoted
+//! string.
//! @pre @a str is of the form <tt>\".*\"</tt>
//! @note Only the first layer of quotation is removed.
std::string unquote(const std::string &str);
-//! @brief Perform equivalent of perl <tt>s/%(%?)/$1/g</tt>.
+//! @brief Perform equivalent of perl <tt>s/%(%?)/$1/g</tt>.
//! @pre @a str doesn't contain '\\n'
std::string unescape(const std::string str);
-//! @brief Compare @a str1 and @a str2 after unescape() has been applied on
+//! @brief Compare @a str1 and @a str2 after unescape() has been applied on
//! them.
//! @pre neighted @a str1 nor @a str2 contains '\\n'.
-//! @return Return true if @a str1, thus compared, is alphabetically less than
+//! @return Return true if @a str1, thus compared, is alphabetically less than
//! @a str2.
-int strcmp_unescaped(const std::string &str1,
+int strcmp_unescaped(const std::string &str1,
const std::string &str2);
//! @brief Transform a string of form <tt>(-)[0-9]+</tt> into an integer.
@@ -106,7 +106,7 @@ std::string remove_white_space(std::string str);
std::string unescape_and_remove_white_space(std::string str);
//! @brief Copy the first complete c-string found in @a source to @a target.
-//! @pre The space reserved for @a source may not be smaller than the space
+//! @pre The space reserved for @a source may not be smaller than the space
//! reserved for @a target.
void copy_c_string(char * target,const char * source);
@@ -138,7 +138,7 @@ class StringVector : public std::vector<std::string>
public:
//! @brief Initialize empty.
StringVector(void);
- //! @brief Split @a s to tokens at spaces and store the tokens in @a this.
+ //! @brief Split @a s to tokens at spaces and store the tokens in @a this.
StringVector(const std::string &s);
//! @brief Add the values in @a another at the end of @a this.
diff --git a/tools/src/hfst-twolc/src/variable_src/ConstContainerIterator.h b/libhfst/src/parsers/variable_src/ConstContainerIterator.h
similarity index 90%
rename from tools/src/hfst-twolc/src/variable_src/ConstContainerIterator.h
rename to libhfst/src/parsers/variable_src/ConstContainerIterator.h
index 116770e..d7341b3 100644
--- a/tools/src/hfst-twolc/src/variable_src/ConstContainerIterator.h
+++ b/libhfst/src/parsers/variable_src/ConstContainerIterator.h
@@ -42,7 +42,7 @@ template <class T> class ConstContainerIterator
static ConstContainerIterator begin(const TVector &v)
{
ConstContainerIterator i;
- for (typename TVector::const_iterator it = v.begin();
+ for (typename TVector::const_iterator it = v.begin();
it != v.end(); ++it)
{
i.iterator_vector.push_back(it->begin());
@@ -54,7 +54,7 @@ template <class T> class ConstContainerIterator
static ConstContainerIterator end(const TVector &v)
{
ConstContainerIterator i;
- for (typename TVector::const_iterator it = v.begin();
+ for (typename TVector::const_iterator it = v.begin();
it != v.end(); ++it)
{
i.iterator_vector.push_back(it->end());
@@ -64,7 +64,7 @@ template <class T> class ConstContainerIterator
return i;
};
- public:
+ public:
//! @brief Empty instance.
ConstContainerIterator(void)
{}
@@ -79,14 +79,14 @@ template <class T> class ConstContainerIterator
//! @brief Prevent memory-leaks with polymorphism.
virtual ~ConstContainerIterator(void) {};
- //! @brief Assign @a another to this.
+ //! @brief Assign @a another to this.
ConstContainerIterator &operator=(const ConstContainerIterator &another)
- {
+ {
if (this != &another)
{
iterator_vector = another.iterator_vector;
- begin_iterator_vector = another.begin_iterator_vector;
- end_iterator_vector = another.end_iterator_vector;
+ begin_iterator_vector = another.begin_iterator_vector;
+ end_iterator_vector = another.end_iterator_vector;
}
return *this;
};
@@ -108,9 +108,9 @@ template <class T> class ConstContainerIterator
if (iterator_vector.at(i) + 1 == end_iterator_vector.at(i))
{ iterator_vector.at(i) = begin_iterator_vector.at(i); }
else
- {
+ {
++iterator_vector.at(i);
- found_a_non_final_iterator = true;
+ found_a_non_final_iterator = true;
break;
}
}
@@ -121,7 +121,7 @@ template <class T> class ConstContainerIterator
//! Return an iterator which points @a i steps further than @a this.
ConstContainerIterator operator+(size_t i) const
- {
+ {
ConstContainerIterator it(*this);
for (size_t n = 0; n < i; ++n) { ++it; }
return it;
@@ -130,15 +130,15 @@ template <class T> class ConstContainerIterator
//! @brief Set values for appropriate variables in @a vvm.
void set_values(VariableValueMap * vvm) const
{
- for (typename TIteratorVector::const_iterator it =
+ for (typename TIteratorVector::const_iterator it =
iterator_vector.begin();
- it != iterator_vector.end();
+ it != iterator_vector.end();
++it)
{ it->set_values(vvm); }
}
//! @brief Dereference. Needed only for complienace with c++ definition of
- //! forward iterators. Genuine functionality is implemented with
+ //! forward iterators. Genuine functionality is implemented with
//! @a set_values().
TVector operator*(void)
{ return TVector(); }
diff --git a/tools/src/hfst-twolc/src/variable_src/MatchedConstContainerIterator.h b/libhfst/src/parsers/variable_src/MatchedConstContainerIterator.h
similarity index 74%
rename from tools/src/hfst-twolc/src/variable_src/MatchedConstContainerIterator.h
rename to libhfst/src/parsers/variable_src/MatchedConstContainerIterator.h
index 236c260..f14cfd7 100644
--- a/tools/src/hfst-twolc/src/variable_src/MatchedConstContainerIterator.h
+++ b/libhfst/src/parsers/variable_src/MatchedConstContainerIterator.h
@@ -2,7 +2,7 @@
//!
//! @author Miikka Silfverberg
//!
-//! @brief Specialization of @a ConstContainerIterator for "matched" variable
+//! @brief Specialization of @a ConstContainerIterator for "matched" variable
//! blocks.
// This program is free software: you can redistribute it and/or modify
@@ -27,28 +27,28 @@
#include "ConstContainerIterator.h"
//! @brief Iterator for a block of variables with matcher <tt>MATCHED</tt>.
-template <class T> class MatchedConstContainerIterator :
+template <class T> class MatchedConstContainerIterator :
public ConstContainerIterator<T>
{
public:
- //! @brief Construct from super class instance.
+ //! @brief Construct from super class instance.
MatchedConstContainerIterator(const ConstContainerIterator<T> &another)
- {
- ConstContainerIterator<T>::operator=(another);
+ {
+ ConstContainerIterator<T>::operator=(another);
- int set_sizes =
- ConstContainerIterator<T>::begin_iterator_vector.size() == 0 ?
- 0
- :
- ConstContainerIterator<T>::end_iterator_vector[0] -
+ int set_sizes =
+ ConstContainerIterator<T>::begin_iterator_vector.size() == 0 ?
+ 0
+ :
+ ConstContainerIterator<T>::end_iterator_vector[0] -
ConstContainerIterator<T>::begin_iterator_vector[0];
- for (size_t i = 0;
- i < ConstContainerIterator<T>::begin_iterator_vector.size();
+ for (size_t i = 0;
+ i < ConstContainerIterator<T>::begin_iterator_vector.size();
++i)
{
- if (ConstContainerIterator<T>::end_iterator_vector[i] -
- ConstContainerIterator<T>::begin_iterator_vector[i]
+ if (ConstContainerIterator<T>::end_iterator_vector[i] -
+ ConstContainerIterator<T>::begin_iterator_vector[i]
!= set_sizes)
{ throw UnequalSetSize(); }
}
@@ -57,14 +57,14 @@ public ConstContainerIterator<T>
//! @brief Increment.
int operator++(void)
{
- for (size_t i = 0;
- i < ConstContainerIterator<T>::iterator_vector.size();
+ for (size_t i = 0;
+ i < ConstContainerIterator<T>::iterator_vector.size();
++i)
{ ++ConstContainerIterator<T>::iterator_vector.at(i); }
return 1;
- };
+ };
MatchedConstContainerIterator operator+(size_t i) const
- {
+ {
MatchedConstContainerIterator it(*this);
for (size_t n = 0; n < i; ++n) { ++it; }
return it;
diff --git a/tools/src/hfst-twolc/src/variable_src/MixedConstContainerIterator.h b/libhfst/src/parsers/variable_src/MixedConstContainerIterator.h
similarity index 82%
rename from tools/src/hfst-twolc/src/variable_src/MixedConstContainerIterator.h
rename to libhfst/src/parsers/variable_src/MixedConstContainerIterator.h
index 4926eef..4d299e4 100644
--- a/tools/src/hfst-twolc/src/variable_src/MixedConstContainerIterator.h
+++ b/libhfst/src/parsers/variable_src/MixedConstContainerIterator.h
@@ -2,7 +2,7 @@
//!
//! @author Miikka Silfverberg
//!
-//! @brief Specialization of @a ConstContainerIterator for "mixed" variable
+//! @brief Specialization of @a ConstContainerIterator for "mixed" variable
//! blocks.
// This program is free software: you can redistribute it and/or modify
@@ -28,16 +28,16 @@
#include "ConstContainerIterator.h"
//! @brief Iterator for a block of variables with matcher <tt>MIXED</tt>
-template <class T> class MixedConstContainerIterator :
+template <class T> class MixedConstContainerIterator :
public ConstContainerIterator<T>
{
protected:
bool didnt_end(void)
{
- for (size_t i = 0; i < ConstContainerIterator<T>::iterator_vector.size();
+ for (size_t i = 0; i < ConstContainerIterator<T>::iterator_vector.size();
++i)
- {
- if(ConstContainerIterator<T>::iterator_vector.at(i) !=
+ {
+ if(ConstContainerIterator<T>::iterator_vector.at(i) !=
ConstContainerIterator<T>::end_iterator_vector.at(i))
{ return true; }
}
@@ -46,11 +46,11 @@ public ConstContainerIterator<T>
bool equal_indices(void)
{
IndexSet index_set;
- for (size_t i = 0; i < ConstContainerIterator<T>::iterator_vector.size();
+ for (size_t i = 0; i < ConstContainerIterator<T>::iterator_vector.size();
++i)
- {
- size_t index =
- ConstContainerIterator<T>::iterator_vector.at(i) -
+ {
+ size_t index =
+ ConstContainerIterator<T>::iterator_vector.at(i) -
ConstContainerIterator<T>::begin_iterator_vector.at(i);
if (index_set.has_element(index))
{ return true; }
@@ -59,22 +59,22 @@ public ConstContainerIterator<T>
return false;
}
public:
- //! @brief Construct from super class instance.
+ //! @brief Construct from super class instance.
MixedConstContainerIterator(const ConstContainerIterator<T> &another)
- {
- ConstContainerIterator<T>::operator=(another);
+ {
+ ConstContainerIterator<T>::operator=(another);
while (didnt_end() && equal_indices())
{ operator++(); }
}
//! @brief Increment.
int operator++(void)
{
- do { ConstContainerIterator<T>::operator++(); }
+ do { ConstContainerIterator<T>::operator++(); }
while (didnt_end() && equal_indices());
return 1;
- }
+ }
MixedConstContainerIterator operator+(size_t i) const
- {
+ {
MixedConstContainerIterator it(*this);
for (size_t n = 0; n < i; ++n) { ++it; }
return it;
diff --git a/tools/src/hfst-twolc/src/variable_src/RuleSymbolVector.cc b/libhfst/src/parsers/variable_src/RuleSymbolVector.cc
similarity index 93%
rename from tools/src/hfst-twolc/src/variable_src/RuleSymbolVector.cc
rename to libhfst/src/parsers/variable_src/RuleSymbolVector.cc
index a9ef6b2..4f24f24 100644
--- a/tools/src/hfst-twolc/src/variable_src/RuleSymbolVector.cc
+++ b/libhfst/src/parsers/variable_src/RuleSymbolVector.cc
@@ -23,7 +23,7 @@ RuleSymbolVector::RuleSymbolVector(const VariableValueMap &vvm):
std::string RuleSymbolVector::replace_variables(void)
{
std::string result;
- for (std::vector<std::string>::const_iterator it =
+ for (std::vector<std::string>::const_iterator it =
std::vector<std::string>::begin();
it != std::vector<std::string>::end();
++it)
@@ -44,7 +44,7 @@ std::string RuleSymbolVector::replace_variables(void)
(symbol.size()-1,
"__HFST_TWOLC_SPACE"+ it->first + "=" + it->second); }
}
- result +=
+ result +=
(vvm.has_key(symbol) ? vvm.get_value(symbol) : symbol) + " ";
}
return result;
@@ -53,7 +53,7 @@ std::string RuleSymbolVector::replace_variables(void)
std::string RuleSymbolVector::replace_variables(const RuleCenter ¢er)
{
std::string result;
- for (std::vector<std::string>::const_iterator it =
+ for (std::vector<std::string>::const_iterator it =
std::vector<std::string>::begin();
it != std::vector<std::string>::end();
++it)
@@ -77,15 +77,15 @@ std::string RuleSymbolVector::replace_variables(const RuleCenter ¢er)
else if (symbol == "__HFST_TWOLC_RULE_CENTER")
{ symbol = center.first + " __HFST_TWOLC_: " + center.second; }
- result +=
+ result +=
(vvm.has_key(symbol) ? vvm.get_value(symbol) : symbol) + " ";
}
return result;
}
RuleSymbolVector &RuleSymbolVector::push_back(const std::string &s)
-{
- std::vector<std::string>::push_back(s);
+{
+ std::vector<std::string>::push_back(s);
return *this;
}
diff --git a/tools/src/hfst-twolc/src/variable_src/RuleSymbolVector.h b/libhfst/src/parsers/variable_src/RuleSymbolVector.h
similarity index 93%
rename from tools/src/hfst-twolc/src/variable_src/RuleSymbolVector.h
rename to libhfst/src/parsers/variable_src/RuleSymbolVector.h
index fa41a7e..840ff9f 100644
--- a/tools/src/hfst-twolc/src/variable_src/RuleSymbolVector.h
+++ b/libhfst/src/parsers/variable_src/RuleSymbolVector.h
@@ -35,15 +35,15 @@ class RuleSymbolVector : public std::vector<std::string>
protected:
const VariableValueMap &vvm;
public:
- //! @brief Create empty instance whose variable values are read from
+ //! @brief Create empty instance whose variable values are read from
//! @a vvm.
RuleSymbolVector(const VariableValueMap &vvm);
- //! @brief Replace variables with their values given by
+ //! @brief Replace variables with their values given by
//! @a vvm and return a string representation of the result.
std::string replace_variables(void);
- //! @brief Replace variables with their values given by
+ //! @brief Replace variables with their values given by
//! @a vvm and return a string representation of the result.
std::string replace_variables(const RuleCenter ¢er);
diff --git a/tools/src/hfst-twolc/src/variable_src/RuleVariables.cc b/libhfst/src/parsers/variable_src/RuleVariables.cc
similarity index 99%
rename from tools/src/hfst-twolc/src/variable_src/RuleVariables.cc
rename to libhfst/src/parsers/variable_src/RuleVariables.cc
index b0131f8..de1b115 100644
--- a/tools/src/hfst-twolc/src/variable_src/RuleVariables.cc
+++ b/libhfst/src/parsers/variable_src/RuleVariables.cc
@@ -72,7 +72,7 @@ void RuleVariables::clear(void)
bool RuleVariables::empty(void) const
{
- return
+ return
freely_blocks.begin() == freely_blocks.end() &&
matched_blocks.begin() == matched_blocks.end() &&
mixed_blocks.begin() == mixed_blocks.end();
diff --git a/tools/src/hfst-twolc/src/variable_src/RuleVariables.h b/libhfst/src/parsers/variable_src/RuleVariables.h
similarity index 100%
rename from tools/src/hfst-twolc/src/variable_src/RuleVariables.h
rename to libhfst/src/parsers/variable_src/RuleVariables.h
diff --git a/tools/src/hfst-twolc/src/variable_src/RuleVariablesConstIterator.cc b/libhfst/src/parsers/variable_src/RuleVariablesConstIterator.cc
similarity index 97%
rename from tools/src/hfst-twolc/src/variable_src/RuleVariablesConstIterator.cc
rename to libhfst/src/parsers/variable_src/RuleVariablesConstIterator.cc
index 9dace53..4d04641 100644
--- a/tools/src/hfst-twolc/src/variable_src/RuleVariablesConstIterator.cc
+++ b/libhfst/src/parsers/variable_src/RuleVariablesConstIterator.cc
@@ -67,7 +67,7 @@ RuleVariablesConstIterator &RuleVariablesConstIterator::operator=
bool RuleVariablesConstIterator::operator==
(const RuleVariablesConstIterator &another)
-{ return f_it == another.f_it && ma_it == another.ma_it &&
+{ return f_it == another.f_it && ma_it == another.ma_it &&
mi_it == another.mi_it; }
bool RuleVariablesConstIterator::operator!=
@@ -81,11 +81,11 @@ void RuleVariablesConstIterator::operator++(void)
if (ma_it + 1 == ma_end)
{
if (mi_it + 1 == mi_end)
- {
+ {
f_it = f_end;
ma_it = ma_end;
mi_it = mi_end;
- return;
+ return;
}
else
{ ++mi_it; }
diff --git a/tools/src/hfst-twolc/src/variable_src/RuleVariablesConstIterator.h b/libhfst/src/parsers/variable_src/RuleVariablesConstIterator.h
similarity index 96%
rename from tools/src/hfst-twolc/src/variable_src/RuleVariablesConstIterator.h
rename to libhfst/src/parsers/variable_src/RuleVariablesConstIterator.h
index b2f01d5..3196e94 100644
--- a/tools/src/hfst-twolc/src/variable_src/RuleVariablesConstIterator.h
+++ b/libhfst/src/parsers/variable_src/RuleVariablesConstIterator.h
@@ -2,7 +2,7 @@
//!
//! @author Miikka Silfverberg
//!
-//! @brief Const iterator for @a RuleVariables objects.
+//! @brief Const iterator for @a RuleVariables objects.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -58,7 +58,7 @@ class RuleVariablesConstIterator
//! Increment.
void operator++(void);
- //! @brief Return an iterator which is equivalent to @a this incremented
+ //! @brief Return an iterator which is equivalent to @a this incremented
//! @a i times.
RuleVariablesConstIterator operator+(size_t i);
diff --git a/tools/src/hfst-twolc/src/variable_src/VariableBlock.h b/libhfst/src/parsers/variable_src/VariableBlock.h
similarity index 85%
rename from tools/src/hfst-twolc/src/variable_src/VariableBlock.h
rename to libhfst/src/parsers/variable_src/VariableBlock.h
index 888fbaf..3a4fbe8 100644
--- a/tools/src/hfst-twolc/src/variable_src/VariableBlock.h
+++ b/libhfst/src/parsers/variable_src/VariableBlock.h
@@ -30,8 +30,8 @@
#include "MixedConstContainerIterator.h"
//! @brief Prototype for variable blocks.
-template<class IT> class VariableBlock :
-public VariableContainer<VariableValues,IT>
+template<class IT> class VariableBlock :
+public VariableContainer<VariableValues,IT>
{
public:
VariableBlock(void):
@@ -45,25 +45,25 @@ public VariableContainer<VariableValues,IT>
{
if (it->empty())
{ throw EmptyContainer(); }
- VariableContainer<VariableValues,IT>::add_object(*it);
+ VariableContainer<VariableValues,IT>::add_object(*it);
}
}
};
-//! @brief Container for the variables in a variable block with matcher
+//! @brief Container for the variables in a variable block with matcher
//! <tt>FREELY</tt>
-typedef VariableBlock<ConstContainerIterator<VariableValues> >
+typedef VariableBlock<ConstContainerIterator<VariableValues> >
FreelyVariableBlock;
-//! @brief Container for the variables in a variable block with matcher
+//! @brief Container for the variables in a variable block with matcher
//! <tt>MATCHED</tt>
-typedef
-VariableBlock<MatchedConstContainerIterator<VariableValues> >
+typedef
+VariableBlock<MatchedConstContainerIterator<VariableValues> >
MatchedVariableBlock;
-//! @brief Container for the variables in a variable block with matcher
+//! @brief Container for the variables in a variable block with matcher
//! <tt>MIXED</tt>
-typedef VariableBlock<MixedConstContainerIterator<VariableValues> >
+typedef VariableBlock<MixedConstContainerIterator<VariableValues> >
MixedVariableBlock;
#endif // VARIABLE_BLOCK_H_
diff --git a/tools/src/hfst-twolc/src/variable_src/VariableBlockContainer.h b/libhfst/src/parsers/variable_src/VariableBlockContainer.h
similarity index 88%
rename from tools/src/hfst-twolc/src/variable_src/VariableBlockContainer.h
rename to libhfst/src/parsers/variable_src/VariableBlockContainer.h
index f0e9050..8ab0a72 100644
--- a/tools/src/hfst-twolc/src/variable_src/VariableBlockContainer.h
+++ b/libhfst/src/parsers/variable_src/VariableBlockContainer.h
@@ -26,30 +26,30 @@
#include "VariableBlock.h"
//! Iterator for @a FreelyVariableBlock objects.
-typedef ConstContainerIterator<FreelyVariableBlock>
+typedef ConstContainerIterator<FreelyVariableBlock>
FreelyVariableBlockConstIterator;
//! Iterator for @a MatchedVariableBlock objects.
-typedef ConstContainerIterator<MatchedVariableBlock>
+typedef ConstContainerIterator<MatchedVariableBlock>
MatchedVariableBlockConstIterator;
//! Iterator for @a MixedVariableBlock objects.
-typedef ConstContainerIterator<MixedVariableBlock>
+typedef ConstContainerIterator<MixedVariableBlock>
MixedVariableBlockConstIterator;
//! Container for @a FreelyVariableBlock objects.
-typedef
-VariableContainer<FreelyVariableBlock,FreelyVariableBlockConstIterator>
+typedef
+VariableContainer<FreelyVariableBlock,FreelyVariableBlockConstIterator>
FreelyVariableBlockContainer;
//! Container for @a MatchedVariableBlock objects.
-typedef
-VariableContainer<MatchedVariableBlock,MatchedVariableBlockConstIterator>
+typedef
+VariableContainer<MatchedVariableBlock,MatchedVariableBlockConstIterator>
MatchedVariableBlockContainer;
//! Container for @a MixedVariableBlock objects.
-typedef
-VariableContainer<MixedVariableBlock,MixedVariableBlockConstIterator>
+typedef
+VariableContainer<MixedVariableBlock,MixedVariableBlockConstIterator>
MixedVariableBlockContainer;
#endif // VARIABLE_BLOCK_CONTAINER_H_
diff --git a/tools/src/hfst-twolc/src/variable_src/VariableContainer.h b/libhfst/src/parsers/variable_src/VariableContainer.h
similarity index 92%
rename from tools/src/hfst-twolc/src/variable_src/VariableContainer.h
rename to libhfst/src/parsers/variable_src/VariableContainer.h
index 198c909..57b4904 100644
--- a/tools/src/hfst-twolc/src/variable_src/VariableContainer.h
+++ b/libhfst/src/parsers/variable_src/VariableContainer.h
@@ -17,7 +17,7 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
#ifndef VARIABLE_CONTAINER_H_
-#define VARIABLE_CONTAINER_H
+#define VARIABLE_CONTAINER_H_
#ifdef HAVE_CONFIG_H
# include <config.h>
@@ -27,7 +27,7 @@
#include "VariableContainerBase.h"
//! @brief Generic container-type for subsets of rule variables.
-template<class T, class IT> class VariableContainer :
+template<class T, class IT> class VariableContainer :
public VariableContainerBase<T>
{
public:
@@ -43,4 +43,4 @@ public VariableContainerBase<T>
{ return const_iterator::end(VariableContainerBase<T>::T_vector); }
};
-#endif // VARIABLE_CONTAINER_H
+#endif // VARIABLE_CONTAINER_H_
diff --git a/tools/src/hfst-twolc/src/variable_src/VariableContainerBase.h b/libhfst/src/parsers/variable_src/VariableContainerBase.h
similarity index 97%
rename from tools/src/hfst-twolc/src/variable_src/VariableContainerBase.h
rename to libhfst/src/parsers/variable_src/VariableContainerBase.h
index 06cc797..0a469a9 100644
--- a/tools/src/hfst-twolc/src/variable_src/VariableContainerBase.h
+++ b/libhfst/src/parsers/variable_src/VariableContainerBase.h
@@ -17,7 +17,7 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
#ifndef VARIABLE_CONTAINER_BASE_H_
-#define VARIABLE_CONTAINER_BASE_H
+#define VARIABLE_CONTAINER_BASE_H_
#ifdef HAVE_CONFIG_H
# include <config.h>
diff --git a/tools/src/hfst-twolc/src/variable_src/VariableDefs.h b/libhfst/src/parsers/variable_src/VariableDefs.h
similarity index 100%
rename from tools/src/hfst-twolc/src/variable_src/VariableDefs.h
rename to libhfst/src/parsers/variable_src/VariableDefs.h
diff --git a/tools/src/hfst-twolc/src/variable_src/VariableValueIterator.h b/libhfst/src/parsers/variable_src/VariableValueIterator.h
similarity index 97%
rename from tools/src/hfst-twolc/src/variable_src/VariableValueIterator.h
rename to libhfst/src/parsers/variable_src/VariableValueIterator.h
index 5675786..3c2cbe8 100644
--- a/tools/src/hfst-twolc/src/variable_src/VariableValueIterator.h
+++ b/libhfst/src/parsers/variable_src/VariableValueIterator.h
@@ -95,8 +95,8 @@ template <class IT> class VariableValueIterator
friend class VariableValues;
};
-//! @brief Const version of @a VariableValueIterator.
-typedef VariableValueIterator<std::vector<std::string>::const_iterator>
+//! @brief Const version of @a VariableValueIterator.
+typedef VariableValueIterator<std::vector<std::string>::const_iterator>
ConstVariableValueIterator;
#endif // VARIABLE_VALUE_ITERATOR_H_
diff --git a/tools/src/hfst-twolc/src/variable_src/VariableValues.cc b/libhfst/src/parsers/variable_src/VariableValues.cc
similarity index 100%
rename from tools/src/hfst-twolc/src/variable_src/VariableValues.cc
rename to libhfst/src/parsers/variable_src/VariableValues.cc
diff --git a/tools/src/hfst-twolc/src/variable_src/VariableValues.h b/libhfst/src/parsers/variable_src/VariableValues.h
similarity index 100%
rename from tools/src/hfst-twolc/src/variable_src/VariableValues.h
rename to libhfst/src/parsers/variable_src/VariableValues.h
diff --git a/libhfst/src/parsers/xfst-lexer.ll b/libhfst/src/parsers/xfst-lexer.ll
index 5338fa1..28e92de 100644
--- a/libhfst/src/parsers/xfst-lexer.ll
+++ b/libhfst/src/parsers/xfst-lexer.ll
@@ -1,13 +1,13 @@
%option 8Bit batch noyywrap prefix="hxfst"
%{
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
//! @file xfst-lexer.ll
@@ -30,14 +30,18 @@ namespace hfst {
#include "xfst-parser.hh"
#endif
-
#include "xfst-utils.h"
#include "XfstCompiler.h"
#include <assert.h>
+#include "HfstDataTypes.h"
+
extern void hxfsterror(const char *text);
+#undef YY_FATAL_ERROR
+#define YY_FATAL_ERROR(msg) hxfsterror(msg)
+
int source_stack_size = 0;
%}
@@ -102,12 +106,12 @@ LWSP [\t ]*
}
^{LWSP}("apply up"|"up"){LWSP}(""|"\r")$ {
- if (hfst::xfst::xfst_->getReadInteractiveTextFromStdin())
+ if (hfst::xfst::xfst_->getReadInteractiveTextFromStdin())
{
// let XfstCompiler take care of the input to apply up command
return APPLY_UP;
}
- else
+ else
{
// read input to apply up command
BEGIN(APPLY_STATE);
@@ -120,14 +124,14 @@ LWSP [\t ]*
}
^{LWSP}("apply down"|"down"){LWSP}(""|"\r")$ {
- if (hfst::xfst::xfst_->getReadInteractiveTextFromStdin())
+ if (hfst::xfst::xfst_->getReadInteractiveTextFromStdin())
{
// let XfstCompiler take care of the input to apply down command
return APPLY_DOWN;
}
- else
+ else
{
- // read input to apply down command
+ // read input to apply down command
BEGIN(APPLY_STATE);
return APPLY_DOWN;
}
@@ -465,8 +469,8 @@ LWSP [\t ]*
}
^{LWSP}("quit"|"exit"|"bye"|"stop"|"hyvästi"|"au revoir"|"näkemiin"|"viszlát"|"auf wiedersehen"|"has") {
- hxfstlval.name = strdup("");
- return QUIT;
+ hxfstlval.name = strdup("");
+ return QUIT;
}
^{LWSP}("lexc"|"read lexc") {
@@ -652,22 +656,22 @@ LWSP [\t ]*
// search for a special end string
std::string str(hxfsttext);
- std::size_t end_found = str.find("<ctrl-d>");
+ std::size_t end_found = str.find("<ctrl-d>");
- // CASE 1: no end string found: the rest is input to apply
+ // CASE 1: no end string found: the rest is input to apply
if (end_found == std::string::npos) {
- hxfstlval.text = hxfsttext;
+ hxfstlval.text = strdup(hxfsttext);
return APPLY_INPUT;
}
// CASE 2: there are other commands after the input to apply
unsigned int total_length = (unsigned int)strlen(hxfsttext);
- unsigned int endpos = (unsigned int)end_found;
+ unsigned int endpos = (unsigned int)end_found;
// copy the input to apply and set is as return value
char * buf = (char*) malloc(endpos + 1);
for (unsigned int i=0; i < endpos; i++)
- {
+ {
buf[i] = hxfsttext[i];
}
buf[endpos] = '\0';
@@ -675,18 +679,18 @@ LWSP [\t ]*
free(buf);
// put back the rest of the input text, excluding the "<ctrl-d>"
- if (total_length > 0)
+ if (total_length > 0)
{
// unput modifies hxfsttext, so it must be copied before unputting
char * text_read = strdup(hxfsttext);
// unputting must be done in reverse order
- for(unsigned int i=total_length-1;
+ for(unsigned int i=total_length-1;
i >= (endpos + (unsigned int)strlen("<ctrl-d>"));
i--)
{
unput(*(text_read+i));
}
- free(text_read);
+ free(text_read);
}
return APPLY_INPUT;
@@ -702,14 +706,14 @@ LWSP [\t ]*
unsigned int total_length = (unsigned int)strlen(hxfsttext);
// compile regex to find out where it ends
- // as a positive side effect, the regex is also conveniently compiled
+ // as a positive side effect, the regex is also conveniently compiled
// into a transducer which is stored in XfstCompiler::latest_regex_compiled
(void) hfst::xfst::xfst_->compile_regex(hxfsttext, chars_read);
// copy the input to regex and set is as return value
char * buf = (char*) malloc(chars_read+1);
for (unsigned int i=0; i < chars_read; i++)
- {
+ {
buf[i] = hxfsttext[i];
}
buf[chars_read] = '\0';
@@ -717,7 +721,7 @@ LWSP [\t ]*
free(buf);
// put back the rest of the input text
- if (total_length > 0)
+ if (total_length > 0)
{
// unput modifies hxfsttext, so it must be copied before unputting
char * text_read = strdup(hxfsttext);
@@ -726,7 +730,7 @@ LWSP [\t ]*
{
unput(*(text_read+i));
}
- free(text_read);
+ free(text_read);
}
return REGEX;
@@ -736,20 +740,20 @@ LWSP [\t ]*
// ^ include directive
FILE * tmp = NULL;
- if ((tmp = fopen(hfst::xfst::strstrip(hxfsttext), "r" )) != NULL)
+ if ((tmp = hfst::hfst_fopen(hfst::xfst::strstrip(hxfsttext), "r" )) != NULL)
{
//printf("Opening file '%s'.\n", hfst::xfst::strstrip(hxfsttext));
// push the included text onto the lexer stack
hxfstpush_buffer_state(hxfst_create_buffer(tmp, 32000));
++source_stack_size;
- }
- else
+ }
+ else
{
char buffer [1024];
sprintf(buffer, "Error opening file '%s'\n",hfst::xfst::strstrip(hxfsttext));
hxfsterror(buffer);
- }
- BEGIN(INITIAL);
+ }
+ BEGIN(INITIAL);
}
">"{WSP}*{NAMETOKEN} {
@@ -827,7 +831,7 @@ LWSP [\t ]*
if (source_stack_size < 0) {
yyterminate();
}
- // EOF encountered because reaching end of included input
+ // EOF encountered because reaching end of included input
else {
hxfstpop_buffer_state();
}
diff --git a/libhfst/src/parsers/xfst-parser.yy b/libhfst/src/parsers/xfst-parser.yy
index 8183cc2..ab9301c 100644
--- a/libhfst/src/parsers/xfst-parser.yy
+++ b/libhfst/src/parsers/xfst-parser.yy
@@ -1,11 +1,11 @@
%{
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
//! @file xfst-parser.yy
@@ -47,7 +47,7 @@ int hxfstlex(void);
// just cos I use the llloc struct
%locations
-%union
+%union
{
char* name;
char* text;
@@ -95,11 +95,11 @@ int hxfstlex(void);
%type <text> COMMAND_SEQUENCE NAMETOKEN_LIST QUOTED_NAMETOKEN_LIST LABEL_LIST LABEL
%%
-XFST_SCRIPT: COMMAND_LIST
+XFST_SCRIPT: COMMAND_LIST
| COMMAND_LIST CTRLD
;
-COMMAND_LIST: COMMAND_LIST COMMAND
+COMMAND_LIST: COMMAND_LIST COMMAND
| COMMAND
;
@@ -121,7 +121,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
hfst::xfst::xfst_->apply_up(stdin); CHECK;
}
| APPLY_UP APPLY_INPUT END_COMMAND {
- hfst::xfst::xfst_->apply_up($2); CHECK;
+ hfst::xfst::xfst_->apply_up($2); CHECK; free($2);
}
| APPLY_UP NAMETOKEN END_COMMAND {
hfst::xfst::xfst_->apply_up($2);
@@ -140,7 +140,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
hfst::xfst::xfst_->apply_down(stdin); CHECK;
}
| APPLY_DOWN APPLY_INPUT END_COMMAND {
- hfst::xfst::xfst_->apply_down($2); CHECK;
+ hfst::xfst::xfst_->apply_down($2); CHECK; free($2);
}
| APPLY_DOWN NAMETOKEN END_COMMAND {
hfst::xfst::xfst_->apply_down($2);
@@ -241,11 +241,11 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
free($2); CHECK;
}
// help
- | APROPOS END_COMMAND {
+ | APROPOS END_COMMAND {
hfst::xfst::xfst_->apropos($1);
free($1); CHECK;
}
- | DESCRIBE END_COMMAND {
+ | DESCRIBE END_COMMAND {
hfst::xfst::xfst_->describe($1); CHECK;
}
// stack
@@ -296,11 +296,11 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
hfst::xfst::xfst_->eliminate_flags(); CHECK;
}
// system
- | ECHO {
+ | ECHO {
hfst::xfst::xfst_->echo($1);
free($1); CHECK;
}
- | QUIT {
+ | QUIT {
hfst::xfst::xfst_->quit($1);
free($1);
return EXIT_SUCCESS;
@@ -311,6 +311,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
}
| SOURCE NAMETOKEN END_COMMAND {
hxfsterror("source not implemented yywrap\n");
+ free($2);
return EXIT_FAILURE;
}
| SYSTEM {
@@ -453,6 +454,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
if (strcmp($2, "upper") && strcmp($2, "lower"))
{
hxfsterror("should be upper or lower");
+ free($2);
return EXIT_FAILURE;
}
hfst::xfst::xfst_->print_arc_count($2, &hfst::xfst::xfst_->get_output_stream());
@@ -545,26 +547,26 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
//std::ofstream oss($2);
std::ofstream oss($2);
hfst::xfst::xfst_->print_longest_string(&oss);
- //hfst::xfst::xfst_fclose(f, $2);
+ //hfst::xfst::xfst_fclose(f, $2);
oss.close();
CHECK;
}
| PRINT_LONGEST_STRING END_COMMAND {
- //hfst::xfst::xfst_->print_longest_string(&hfst::xfst::xfst_->get_output_stream());
- hfst::xfst::xfst_->print_longest_string(&hfst::xfst::xfst_->get_output_stream());
+ //hfst::xfst::xfst_->print_longest_string(&hfst::xfst::xfst_->get_output_stream());
+ hfst::xfst::xfst_->print_longest_string(&hfst::xfst::xfst_->get_output_stream());
CHECK;
}
| PRINT_LONGEST_STRING_SIZE REDIRECT_OUT END_COMMAND {
//std::ofstream oss($2);
std::ofstream oss($2);
hfst::xfst::xfst_->print_longest_string_size(&oss);
- //hfst::xfst::xfst_fclose(f, $2);
+ //hfst::xfst::xfst_fclose(f, $2);
oss.close();
CHECK;
}
| PRINT_LONGEST_STRING_SIZE END_COMMAND {
- //hfst::xfst::xfst_->print_longest_string_size(&hfst::xfst::xfst_->get_output_stream());
- hfst::xfst::xfst_->print_longest_string_size(&hfst::xfst::xfst_->get_output_stream());
+ //hfst::xfst::xfst_->print_longest_string_size(&hfst::xfst::xfst_->get_output_stream());
+ hfst::xfst::xfst_->print_longest_string_size(&hfst::xfst::xfst_->get_output_stream());
CHECK;
}
| PRINT_NAME REDIRECT_OUT END_COMMAND {
@@ -584,7 +586,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
CHECK;
}
| PRINT_SHORTEST_STRING END_COMMAND {
- //hfst::xfst::xfst_->print_shortest_string(&hfst::xfst::xfst_->get_output_stream());
+ //hfst::xfst::xfst_->print_shortest_string(&hfst::xfst::xfst_->get_output_stream());
hfst::xfst::xfst_->print_shortest_string(&hfst::xfst::xfst_->get_output_stream());
CHECK;
}
@@ -593,24 +595,24 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
std::ofstream oss($2);
hfst::xfst::xfst_->print_shortest_string_size(&oss);
//hfst::xfst::xfst_fclose(f, $2);
- oss.close();
+ oss.close();
CHECK;
}
| PRINT_SHORTEST_STRING_SIZE END_COMMAND {
- //hfst::xfst::xfst_->print_shortest_string_size(&hfst::xfst::xfst_->get_output_stream());
- hfst::xfst::xfst_->print_shortest_string_size(&hfst::xfst::xfst_->get_output_stream());
+ //hfst::xfst::xfst_->print_shortest_string_size(&hfst::xfst::xfst_->get_output_stream());
+ hfst::xfst::xfst_->print_shortest_string_size(&hfst::xfst::xfst_->get_output_stream());
CHECK;
}
| PRINT_LOWER_WORDS NAMETOKEN NAMETOKEN END_COMMAND {
hfst::xfst::xfst_->print_lower_words($2, hfst::xfst::nametoken_to_number($3), &hfst::xfst::xfst_->get_output_stream());
- free($2); CHECK;
+ free($2); free($3); CHECK;
}
| PRINT_LOWER_WORDS NAMETOKEN NAMETOKEN REDIRECT_OUT END_COMMAND {
//std::ofstream oss($4, "w");
std::ofstream oss($4);
hfst::xfst::xfst_->print_lower_words($2, hfst::xfst::nametoken_to_number($3), &oss);
- free($2);
- //hfst::xfst::xfst_fclose(f, $4);
+ free($2); free($3);
+ //hfst::xfst::xfst_fclose(f, $4);
oss.close();
CHECK;
}
@@ -620,6 +622,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
hfst::xfst::xfst_->print_lower_words(NULL, i, &hfst::xfst::xfst_->get_output_stream());
else
hfst::xfst::xfst_->print_lower_words($2, 0, &hfst::xfst::xfst_->get_output_stream());
+ free($2);
CHECK;
}
| PRINT_LOWER_WORDS END_COMMAND {
@@ -633,29 +636,30 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
hfst::xfst::xfst_->print_lower_words(NULL, i, &oss);
else
hfst::xfst::xfst_->print_lower_words($2, 0, &oss);
- //hfst::xfst::xfst_fclose(f, $3);
+ //hfst::xfst::xfst_fclose(f, $3);
oss.close();
+ free($2);
CHECK;
}
| PRINT_LOWER_WORDS REDIRECT_OUT END_COMMAND {
//std::ofstream oss($2);
std::ofstream oss($2);
hfst::xfst::xfst_->print_lower_words(NULL, 0, &oss);
- //hfst::xfst::xfst_fclose(f, $2);
+ //hfst::xfst::xfst_fclose(f, $2);
oss.close();
CHECK;
}
| PRINT_RANDOM_LOWER NAMETOKEN NAMETOKEN END_COMMAND {
hfst::xfst::xfst_->print_random_lower($2, hfst::xfst::nametoken_to_number($3), &hfst::xfst::xfst_->get_output_stream());
- free($2); CHECK;
+ free($2); free($3); CHECK;
}
| PRINT_RANDOM_LOWER NAMETOKEN NAMETOKEN REDIRECT_OUT END_COMMAND {
//std::ofstream oss($4, "w");
std::ofstream oss($4);
hfst::xfst::xfst_->print_random_lower($2, hfst::xfst::nametoken_to_number($3), &oss);
- free($2);
+ free($2); free($3);
oss.close();
- //hfst::xfst::xfst_fclose(f, $4);
+ //hfst::xfst::xfst_fclose(f, $4);
CHECK;
}
| PRINT_RANDOM_LOWER NAMETOKEN END_COMMAND {
@@ -664,7 +668,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
hfst::xfst::xfst_->print_random_lower(NULL, i, &hfst::xfst::xfst_->get_output_stream());
else
hfst::xfst::xfst_->print_random_lower($2, 15, &hfst::xfst::xfst_->get_output_stream());
- CHECK;
+ free($2);CHECK;
}
| PRINT_RANDOM_LOWER END_COMMAND {
hfst::xfst::xfst_->print_random_lower(NULL, 15, &hfst::xfst::xfst_->get_output_stream()); CHECK;
@@ -677,15 +681,15 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
hfst::xfst::xfst_->print_random_lower(NULL, i, &oss);
else
hfst::xfst::xfst_->print_random_lower($2, 15, &oss);
- //hfst::xfst::xfst_fclose(f, $3);
+ //hfst::xfst::xfst_fclose(f, $3);
oss.close();
- CHECK;
+ free($2); CHECK;
}
| PRINT_RANDOM_LOWER REDIRECT_OUT END_COMMAND {
//std::ofstream oss($2);
std::ofstream oss($2);
hfst::xfst::xfst_->print_random_lower(NULL, 15, &oss);
- //hfst::xfst::xfst_fclose(f, $2);
+ //hfst::xfst::xfst_fclose(f, $2);
oss.close();
CHECK;
}
@@ -697,8 +701,8 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
//std::ofstream oss($4, "w");
std::ofstream oss($4);
hfst::xfst::xfst_->print_upper_words($2, hfst::xfst::nametoken_to_number($3), &oss);
- free($2);
- //hfst::xfst::xfst_fclose(f, $4);
+ free($2); free($3);
+ //hfst::xfst::xfst_fclose(f, $4);
oss.close();
CHECK;
}
@@ -708,7 +712,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
hfst::xfst::xfst_->print_upper_words(NULL, i, &hfst::xfst::xfst_->get_output_stream());
else
hfst::xfst::xfst_->print_upper_words($2, 0, &hfst::xfst::xfst_->get_output_stream());
- CHECK;
+ free($2); CHECK;
}
| PRINT_UPPER_WORDS END_COMMAND {
hfst::xfst::xfst_->print_upper_words(NULL, 0, &hfst::xfst::xfst_->get_output_stream()); CHECK;
@@ -721,28 +725,28 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
hfst::xfst::xfst_->print_upper_words(NULL, i, &oss);
else
hfst::xfst::xfst_->print_upper_words($2, 0, &oss);
- //hfst::xfst::xfst_fclose(f, $3);
+ //hfst::xfst::xfst_fclose(f, $3);
oss.close();
- CHECK;
+ free($2); CHECK;
}
| PRINT_UPPER_WORDS REDIRECT_OUT END_COMMAND {
//std::ofstream oss($2);
- std::ofstream oss($2);
+ std::ofstream oss($2);
hfst::xfst::xfst_->print_upper_words(NULL, 0, &oss);
- //hfst::xfst::xfst_fclose(f, $2);
+ //hfst::xfst::xfst_fclose(f, $2);
oss.close();
CHECK;
}
| PRINT_RANDOM_UPPER NAMETOKEN NAMETOKEN END_COMMAND {
hfst::xfst::xfst_->print_random_upper($2, hfst::xfst::nametoken_to_number($3), &hfst::xfst::xfst_->get_output_stream());
- free($2); CHECK;
+ free($2); free($3); CHECK;
}
| PRINT_RANDOM_UPPER NAMETOKEN NAMETOKEN REDIRECT_OUT END_COMMAND {
//std::ofstream oss($4, "w");
std::ofstream oss($4);
hfst::xfst::xfst_->print_random_upper($2, hfst::xfst::nametoken_to_number($3), &oss);
- free($2);
- //hfst::xfst::xfst_fclose(f, $4);
+ free($2); free($3);
+ //hfst::xfst::xfst_fclose(f, $4);
oss.close();
CHECK;
}
@@ -752,7 +756,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
hfst::xfst::xfst_->print_random_upper(NULL, i, &hfst::xfst::xfst_->get_output_stream());
else
hfst::xfst::xfst_->print_random_upper($2, 15, &hfst::xfst::xfst_->get_output_stream());
- CHECK;
+ free($2); CHECK;
}
| PRINT_RANDOM_UPPER END_COMMAND {
hfst::xfst::xfst_->print_random_upper(NULL, 15, &hfst::xfst::xfst_->get_output_stream()); CHECK;
@@ -765,9 +769,9 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
hfst::xfst::xfst_->print_random_upper(NULL, i, &oss);
else
hfst::xfst::xfst_->print_random_upper($2, 15, &oss);
- //hfst::xfst::xfst_fclose(f, $3);
+ //hfst::xfst::xfst_fclose(f, $3);
oss.close();
- CHECK;
+ free($2); CHECK;
}
| PRINT_RANDOM_UPPER REDIRECT_OUT END_COMMAND {
//std::ofstream oss($2);
@@ -779,14 +783,14 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
}
| PRINT_WORDS NAMETOKEN NAMETOKEN END_COMMAND {
hfst::xfst::xfst_->print_words($2, hfst::xfst::nametoken_to_number($3), &hfst::xfst::xfst_->get_output_stream());
- free($2); CHECK;
+ free($2); free($3); CHECK;
}
| PRINT_WORDS NAMETOKEN NAMETOKEN REDIRECT_OUT END_COMMAND {
//std::ofstream oss($4, "w");
std::ofstream oss($4);
hfst::xfst::xfst_->print_words($2, hfst::xfst::nametoken_to_number($3), &oss);
- free($2);
- //hfst::xfst::xfst_fclose(f, $4);
+ free($2); free($3);
+ //hfst::xfst::xfst_fclose(f, $4);
oss.close();
CHECK;
}
@@ -796,7 +800,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
hfst::xfst::xfst_->print_words(NULL, i, &hfst::xfst::xfst_->get_output_stream());
else
hfst::xfst::xfst_->print_words($2, 0, &hfst::xfst::xfst_->get_output_stream());
- CHECK;
+ free($2); CHECK;
}
| PRINT_WORDS END_COMMAND {
hfst::xfst::xfst_->print_words(NULL, 0, &hfst::xfst::xfst_->get_output_stream()); CHECK;
@@ -809,28 +813,28 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
hfst::xfst::xfst_->print_words(NULL, i, &oss);
else
hfst::xfst::xfst_->print_words($2, 0, &oss);
- //hfst::xfst::xfst_fclose(f, $3);
+ //hfst::xfst::xfst_fclose(f, $3);
oss.close();
- CHECK;
+ free($2); CHECK;
}
| PRINT_WORDS REDIRECT_OUT END_COMMAND {
//std::ofstream oss($2);
std::ofstream oss($2);
hfst::xfst::xfst_->print_words(NULL, 0, &oss);
oss.close();
- //hfst::xfst::xfst_fclose(f, $2);
+ //hfst::xfst::xfst_fclose(f, $2);
CHECK;
}
| PRINT_RANDOM_WORDS NAMETOKEN NAMETOKEN END_COMMAND {
hfst::xfst::xfst_->print_random_words($2, hfst::xfst::nametoken_to_number($3), &hfst::xfst::xfst_->get_output_stream());
- free($2); CHECK;
+ free($2); free($3); CHECK;
}
| PRINT_RANDOM_WORDS NAMETOKEN NAMETOKEN REDIRECT_OUT END_COMMAND {
//std::ofstream oss($4, "w");
std::ofstream oss($4);
hfst::xfst::xfst_->print_random_words($2, hfst::xfst::nametoken_to_number($3), &oss);
- free($2);
- //hfst::xfst::xfst_fclose(f, $4);
+ free($2); free($3);
+ //hfst::xfst::xfst_fclose(f, $4);
oss.close();
CHECK;
}
@@ -840,7 +844,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
hfst::xfst::xfst_->print_random_words(NULL, i, &hfst::xfst::xfst_->get_output_stream());
else
hfst::xfst::xfst_->print_random_words($2, 15, &hfst::xfst::xfst_->get_output_stream());
- CHECK;
+ free($2); CHECK;
}
| PRINT_RANDOM_WORDS END_COMMAND {
hfst::xfst::xfst_->print_random_words(NULL, 15, &hfst::xfst::xfst_->get_output_stream()); CHECK;
@@ -853,15 +857,15 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
hfst::xfst::xfst_->print_random_words(NULL, i, &oss);
else
hfst::xfst::xfst_->print_random_words($2, 15, &oss);
- //hfst::xfst::xfst_fclose(f, $3);
+ //hfst::xfst::xfst_fclose(f, $3);
oss.close();
- CHECK;
+ free($2); CHECK;
}
| PRINT_RANDOM_WORDS REDIRECT_OUT END_COMMAND {
//std::ofstream oss($2);
std::ofstream oss($2);
hfst::xfst::xfst_->print_random_words(NULL, 15, &oss);
- //hfst::xfst::xfst_fclose(f, $2);
+ //hfst::xfst::xfst_fclose(f, $2);
oss.close();
CHECK;
}
@@ -912,6 +916,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
| PRINT_SIGMA_WORD_COUNT NAMETOKEN END_COMMAND {
if (strcmp($2, "upper") && strcmp($2, "lower"))
{
+ free($2);
hxfsterror("must be upper or lower\n");
return EXIT_FAILURE;
}
@@ -980,7 +985,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
hfst::xfst::xfst_->write_definition($2, 0);
free($2); CHECK;
}
- | SAVE_DEFINITIONS REDIRECT_OUT END_COMMAND {
+ | SAVE_DEFINITIONS REDIRECT_OUT END_COMMAND {
hfst::xfst::xfst_->write_definitions($2); CHECK;
}
| SAVE_DEFINITIONS END_COMMAND {
@@ -998,7 +1003,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
| SAVE_PROLOG NAMETOKEN END_COMMAND {
std::ofstream oss($2);
hfst::xfst::xfst_->write_prolog(&oss);
- oss.close(); CHECK;
+ oss.close(); free($2); CHECK;
}
| SAVE_PROLOG END_COMMAND {
hfst::xfst::xfst_->write_prolog(&hfst::xfst::xfst_->get_output_stream()); CHECK;
@@ -1031,7 +1036,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
| READ_PROLOG NAMETOKEN END_COMMAND {
FILE * f = hfst::xfst::xfst_->xfst_fopen($2, "r"); CHECK;
hfst::xfst::xfst_->read_prolog(f);
- hfst::xfst::xfst_->xfst_fclose(f, $2); CHECK;
+ hfst::xfst::xfst_->xfst_fclose(f, $2); free($2); CHECK;
}
| READ_PROLOG END_COMMAND {
hfst::xfst::xfst_->read_prolog(stdin); CHECK;
@@ -1082,7 +1087,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
free($2); CHECK;
}
| READ_LEXC NAMETOKEN_LIST CTRLD {
- hfst::xfst::xfst_->read_lexc_from_file(""); CHECK;
+ hfst::xfst::xfst_->read_lexc_from_file(""); free($2); CHECK;
}
| READ_ATT NAMETOKEN END_COMMAND {
hfst::xfst::xfst_->read_att_from_file($2);
@@ -1108,7 +1113,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
std::ofstream oss($2);
hfst::xfst::xfst_->write_att(&oss);
oss.close();
- free($2); CHECK;
+ free($2); free($3); free($4); CHECK;
}
// net ops
| CLEANUP END_COMMAND {
@@ -1202,11 +1207,13 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
hfst::xfst::xfst_->prompt(); CHECK;
}
| NAMETOKEN END_COMMAND {
- if ( hfst::xfst::xfst_->unknown_command($1) != 0)
- {
+ if ( hfst::xfst::xfst_->unknown_command($1) != 0)
+ {
hxfsterror("Command not recognized.\n");
+ free($1);
YYABORT;
}
+ free($1);
}
;
@@ -1232,6 +1239,7 @@ COMMAND_SEQUENCE: COMMAND_SEQUENCE NAMETOKEN {
s++;
}
*r = '\0';
+ free($2);
}
| COMMAND_SEQUENCE SEMICOLON {
$$ = $1;
@@ -1261,6 +1269,7 @@ NAMETOKEN_LIST: NAMETOKEN_LIST NAMETOKEN {
s++;
}
*r = '\0';
+ free($1); free($2);
}
| NAMETOKEN {
$$ = $1;
@@ -1291,6 +1300,7 @@ QUOTED_NAMETOKEN_LIST: QUOTED_NAMETOKEN_LIST NAMETOKEN {
*r = '"';
r++;
*r = '\0';
+ free($1); free($2);
}
| NAMETOKEN {
$$ = static_cast<char*>(malloc(sizeof(char)*strlen($1)+3));
@@ -1313,6 +1323,7 @@ QUOTED_NAMETOKEN_LIST: QUOTED_NAMETOKEN_LIST NAMETOKEN {
LABEL: NAMETOKEN COLON NAMETOKEN {
$$ = strdup((std::string($1) + std::string(":") + std::string($3)).c_str());
+ free($1); free($3);
}
| NAMETOKEN {
$$ = $1;
@@ -1339,6 +1350,7 @@ LABEL_LIST: LABEL_LIST LABEL {
s++;
}
*r = '\0';
+ free($1); free($2);
}
| LABEL {
$$ = $1;
@@ -1353,7 +1365,7 @@ int hxfstparse(void);
// gah, bison/flex error mechanism here
void
hxfsterror(const char* text)
-{
+{
hfst::xfst::xfst_->error() << text << std::endl;
hfst::xfst::xfst_->flush(&hfst::xfst::xfst_->error());
//fprintf(stderr, "%s\n", text);
diff --git a/libhfst/src/parsers/xfst-utils.cc b/libhfst/src/parsers/xfst-utils.cc
index 4db72b4..c196159 100644
--- a/libhfst/src/parsers/xfst-utils.cc
+++ b/libhfst/src/parsers/xfst-utils.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
//! @file xfst-utils.cc
@@ -36,6 +36,11 @@
#include "xfst-utils.h"
+// for hfst::size_t_to_int
+#ifndef HAVE_GETLINE
+ #include "HfstDataTypes.h"
+#endif // HAVE_GETLINE
+
using std::string;
// flex stuffa
@@ -50,7 +55,7 @@ namespace hfst { namespace xfst {
getline(char** s, size_t* n, FILE* f)
{
*s = static_cast<char*>(calloc(sizeof(char),*n));
- char* r = fgets(*s, *n, f);
+ char* r = fgets(*s, hfst::size_t_to_int(*n), f);
if (r == 0)
{
//fprintf(stderr, "unable to read in substitute getline\n");
@@ -60,27 +65,6 @@ namespace hfst { namespace xfst {
}
#endif
- //void
- //xfst_fclose(FILE * file, const char * name)
- //{
- //return fclose(file))
- //{
- // //fprintf(stderr, "Unable to close %s: %s\n", name, strerror(errno));
- // throw strerr;
- //}
- //}
-
- //FILE*
- //xfst_fopen(const char* path, const char* mode)
- //{
- //FILE* f = fopen(path, mode);
- //if (f == NULL)
- // {
- // fprintf(stderr, "Unable to open %s: %s\n", path, strerror(errno));
- // }
- //return f;
- //}
-
char*
strdup_token_part()
{
@@ -248,7 +232,7 @@ strdup_nonconst_part(const char* token, const char* prefix,
}
-} }
+} }
// vim: set ft=cpp.doxygen:
diff --git a/libhfst/src/parsers/xfst-utils.h b/libhfst/src/parsers/xfst-utils.h
index 7fd27f1..c49429d 100644
--- a/libhfst/src/parsers/xfst-utils.h
+++ b/libhfst/src/parsers/xfst-utils.h
@@ -1,16 +1,16 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
//! @file lexc-utils.h
//!
//! @brief Various string handling methods for HFST lexc.
-//!
+//!
//! @author Tommi A. Pirinen
#ifndef GUARD_lexc_utils_h
@@ -33,10 +33,6 @@ namespace hfst { namespace xfst {
ssize_t getline(char** line, size_t* n, FILE* f);
#endif
-//! @brief Open file and print XFSTish stuff
-//FILE* xfst_fopen(const char* path, const char* mode);
-//! @brief Close \a file that has \a name.
-// void xfst_fclose(FILE * file, const char * name);
//! @brief Strips initial and final white space and strdups
char* strstrip(const char* s);
@@ -44,7 +40,7 @@ char* strstrip(const char* s);
//! @brief extracts the variable substring part from token.
//! Omits constant string prefix, suffix and optionally strips spaces.
-char* strdup_nonconst_part(const char* token,
+char* strdup_nonconst_part(const char* token,
const char* prefix,
const char* suffix,
bool strip);
diff --git a/libhfst/src/parsers/xfst_help_message.cc b/libhfst/src/parsers/xfst_help_message.cc
index d0ba649..5411250 100644
--- a/libhfst/src/parsers/xfst_help_message.cc
+++ b/libhfst/src/parsers/xfst_help_message.cc
@@ -1,18 +1,18 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include <string>
#include <vector>
-namespace hfst
+namespace hfst
{
-namespace xfst
+namespace xfst
{
typedef std::vector<std::string> StringVector;
@@ -65,7 +65,7 @@ bool word_found_in_text(const std::string & str_, const std::string & text_)
// Find str_ in text_:
std::size_t pos = text.find(str);
// (1) If not found, return false
- if (pos == std::string::npos)
+ if (pos == std::string::npos)
{
return false;
}
@@ -113,9 +113,9 @@ StringVector namelist_to_name_vector(const std::string & namelist)
}
// Append help message for command known by names in \a namelist, taking arguments
-// listed in \a arguments, and described by \a description to \a message.
+// listed in \a arguments, and described by \a description to \a message.
// \a all_names defines whether all names in \a namelist are included or just the first one.
-void append_help_message(const std::string & namelist, const std::string & arguments,
+void append_help_message(const std::string & namelist, const std::string & arguments,
const std::string & description, std::string & message, bool all_names = true)
{
size_t NAME_AND_ARGUMENTS_FIELD_WIDTH = 30;
@@ -126,7 +126,7 @@ void append_help_message(const std::string & namelist, const std::string & argum
message.append(" ");
message.append(arguments);
- size_t name_and_arguments_length
+ size_t name_and_arguments_length
= names.front().length() + 1 + arguments.length();
if (name_and_arguments_length > NAME_AND_ARGUMENTS_FIELD_WIDTH)
@@ -176,7 +176,7 @@ bool text_matches_some_name(const std::string & text, const std::string & nameli
bool get_help_message(const std::string & text, std::string & message, int help_mode, bool skip_ambiguous_cases=false);
-bool handle_ambiguous_case(const std::string & name, const std::string & namelist,
+bool handle_ambiguous_case(const std::string & name, const std::string & namelist,
const std::string & text, std::string & message, int help_mode)
{
if (help_mode == HELP_MODE_ALL_COMMANDS ||
@@ -196,16 +196,16 @@ bool handle_ambiguous_case(const std::string & name, const std::string & namelis
if (it != names.begin())
message.append("##\n");
(void)get_help_message(*it, message, help_mode, true);
- }
+ }
return true;
}
// If \a text matches (depending on \a help_mode) a command known by names
// listed in \a names, taking arguments \a arguments, described by \a description,
// add help message for the command to \a message. \a all_names defines whether
-// all names in \a names are included or just the first one.
-// Return whether the search should continue (depends on \a help_mode).
-bool handle_case(const std::string & names, const std::string & arguments,
+// all names in \a names are included or just the first one.
+// Return whether the search should continue (depends on \a help_mode).
+bool handle_case(const std::string & names, const std::string & arguments,
const std::string & description, const std::string & text,
std::string & message, int help_mode, bool all_names=true)
{
@@ -223,7 +223,7 @@ bool handle_case(const std::string & names, const std::string & arguments,
}
return false; // continue search
}
- else // HELP_MODE_ONE_COMMAND
+ else // HELP_MODE_ONE_COMMAND
{
if (text_matches_some_name(text, names))
{
@@ -236,7 +236,7 @@ bool handle_case(const std::string & names, const std::string & arguments,
// Generate help message(s) for command(s) named \a text and append the help message(s)
// to \a message. \a help_mode defines whether we are generating help messages for \a text,
-// all commands (in that case, \a message is ignored) or for commands that contain or
+// all commands (in that case, \a message is ignored) or for commands that contain or
// whose help messages contain the word \a text. \a skip_ambiguous_cases defines whether
// ambiguous cases where \a text matches more than one command are ignored.
// @return Whether the help message could be generated.
@@ -252,15 +252,15 @@ bool handle_case(const std::string & names, const std::string & arguments,
// To see whether any help message was appended.
std::string message_at_start(message);
- COMMAND("ambiguous upper, ambiguous", "",
+ COMMAND("ambiguous upper, ambiguous", "",
"returns the input words which have multiple paths in a transducer");
AMBIGUOUS_COMMAND("apply", "apply down, apply up");
- CONT_COMMAND("apply down, down", "<string>",
+ CONT_COMMAND("apply down, down", "<string>",
"apply <string> down to the top network on stack");
- COMMAND("apply down, down", "",
+ COMMAND("apply down, down", "",
"enter apply down mode (Ctrl-D exits)");
CONT_COMMAND("apply up, up", "<string>",
@@ -274,7 +274,7 @@ bool handle_case(const std::string & names, const std::string & arguments,
//COMMAND("add properties, add", "", "<not implemented>");
//COMMAND("alias", "", "<not implemented>");
//COMMAND("cleanup net, cleanup", "", "<not implemented>");
- COMMAND("clear stack, clear", "",
+ COMMAND("clear stack, clear", "",
"clears the stack");
//COMMAND("collect epsilon-loops, epsilon-loops", "", "<not implemented>");
//"compile-replace lower, com-rep lower"
@@ -286,7 +286,7 @@ bool handle_case(const std::string & names, const std::string & arguments,
COMMAND("crossproduct net, crossproduct", "", "cross-product of top two FSMs on stack");
CONT_COMMAND("define", "<name> <r.e.>", "define a network");
COMMAND("define", "<fname>(<v1,..,vn>) <r.e.>", "define function");
- COMMAND("determinize net, determinize, determinise net, determinise", "", "determinizes top FSM on stack");
+ COMMAND("determinize net, determinize, determinise net, determinise", "", "determinizes top FSM on stack");
COMMAND("echo", "<string>", "echo a string");
//COMMAND("edit properties, edit", "", "<not implemented>");
//"epsilon-remove net, epsilon-remove"
@@ -430,7 +430,7 @@ bool handle_case(const std::string & names, const std::string & arguments,
COMMAND("variable compose-tristate", "", "use the tristate composition algorithm");
COMMAND("variable show-flags", "", "show flag diacritics in `apply'");
COMMAND("variable obey-flags", "", "obey flag diacritics in `apply'");
- COMMAND("variable minimal", "", "minimize resulting FSMs");
+ COMMAND("variable minimal", "", "ON = minimize resulting FSMs, OFF = remove epsilons and determinize");
COMMAND("variable print-pairs", "", "always print both sides when applying");
COMMAND("variable print-space", "", "print spaces between symbols");
COMMAND("variable print-sigma", "", "print the alphabet when printing network");
diff --git a/libhfst/src/parsers/xfst_help_message.h b/libhfst/src/parsers/xfst_help_message.h
index cd61ec7..c5ef95f 100644
--- a/libhfst/src/parsers/xfst_help_message.h
+++ b/libhfst/src/parsers/xfst_help_message.h
@@ -1,15 +1,15 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
-namespace hfst
+namespace hfst
{
-namespace xfst
+namespace xfst
{
typedef std::vector<std::string> StringVector;
@@ -36,30 +36,30 @@ typedef std::vector<std::string> StringVector;
StringVector namelist_to_name_vector(const std::string & namelist);
// Append help message for command known by names in \a namelist, taking arguments
-// listed in \a arguments, and described by \a description to \a message.
+// listed in \a arguments, and described by \a description to \a message.
// \a all_names defines whether all names in \a namelist are included or just the first one.
-void append_help_message(const std::string & namelist, const std::string & arguments,
+void append_help_message(const std::string & namelist, const std::string & arguments,
const std::string & description, std::string & message, bool all_names = true);
bool text_matches_some_name(const std::string & text, const std::string & namelist);
bool get_help_message(const std::string & text, std::string & message, int help_mode, bool skip_ambiguous_cases=false);
-bool handle_ambiguous_case(const std::string & name, const std::string & namelist,
+bool handle_ambiguous_case(const std::string & name, const std::string & namelist,
const std::string & text, std::string & message, int help_mode);
// If \a text matches (depending on \a help_mode) a command known by names
// listed in \a names, taking arguments \a arguments, described by \a description,
// add help message for the command to \a message. \a all_names defines whether
-// all names in \a names are included or just the first one.
-// Return whether the search should continue (depends on \a help_mode).
-bool handle_case(const std::string & names, const std::string & arguments,
+// all names in \a names are included or just the first one.
+// Return whether the search should continue (depends on \a help_mode).
+bool handle_case(const std::string & names, const std::string & arguments,
const std::string & description, const std::string & text,
std::string & message, int help_mode, bool all_names=true);
// Generate help message(s) for command(s) named \a text and append the help message(s)
// to \a message. \a help_mode defines whether we are generating help messages for \a text,
-// all commands (in that case, \a message is ignored) or for commands that contain or
+// all commands (in that case, \a message is ignored) or for commands that contain or
// whose help messages contain the word \a text. \a skip_ambiguous_cases defines whether
// ambiguous cases where \a text matches more than one command are ignored.
// @return Whether the help message could be generated.
diff --git a/libhfst/src/parsers/xre_lex.ll b/libhfst/src/parsers/xre_lex.ll
index 811a646..e5be7f1 100644
--- a/libhfst/src/parsers/xre_lex.ll
+++ b/libhfst/src/parsers/xre_lex.ll
@@ -2,13 +2,13 @@
%{
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
#include <string.h>
@@ -27,9 +27,14 @@
#undef YY_INPUT
#define YY_INPUT(buf, retval, maxlen) (retval = hfst::xre::getinput(buf, maxlen))
+extern int xreerror(const char*);
+
+#undef YY_FATAL_ERROR
+#define YY_FATAL_ERROR(msg) xreerror(msg);
+
// These variablese are used when scanning a regex for a given SYMBOL
// when performing variable substitution in function definition.
-namespace hfst {
+namespace hfst {
namespace xre {
extern unsigned int cr; // number of characters read
extern std::set<unsigned int> positions; // positions of a given SYMBOL
@@ -63,9 +68,9 @@ EC "%"{U8C}
/* any ASCII */
A7 [\x00-\x7e]
/* special meaning in xre testing " */
-A7RESTRICTED [- |<>%!,^:";@0~\\&?$+*/_(){}\]\[-]
+A7RESTRICTED [- |<>%!,^:";@0~\\&?$+*/_(){}\]\[-]
/* non-restricted ASCII testing " */
-A7UNRESTRICTED [\x21-\x7e]{-}[- |<>%!,^:";@0~\\&?$+*/_(){}\]\[-]
+A7UNRESTRICTED [\x21-\x7e]{-}[- |<>%!,.^:";@0~\\&?$+*/_(){}\]\[-]
WEIGHT (-|\+)?[0-9]+(\.[0-9]+)?
@@ -148,7 +153,7 @@ BRACED [{]([^}]|[\300-\337].|[\340-\357]..|[\360-\367]...)+[}]
"\\\\\\" { CR; return LEFT_QUOTIENT; }
-"^"{WSP}*({UINTEGER}|"0")","({UINTEGER}|"0") {
+"^"{WSP}*({UINTEGER}|"0")","({UINTEGER}|"0") {
CR;
yylval->values = hfst::xre::get_n_to_k(yytext);
return CATENATE_N_TO_K;
@@ -160,19 +165,19 @@ BRACED [{]([^}]|[\300-\337].|[\340-\357]..|[\360-\367]...)+[}]
return CATENATE_N_TO_K;
}
-"^>"{WSP}*({UINTEGER}|"0") {
+"^>"{WSP}*({UINTEGER}|"0") {
CR;
yylval->value = strtol(yytext + 2, 0, 10);
- return CATENATE_N_PLUS;
+ return CATENATE_N_PLUS;
}
-"^<"{WSP}*({UINTEGER}|"0") {
+"^<"{WSP}*({UINTEGER}|"0") {
CR;
yylval->value = strtol(yytext + 2, 0, 10);
return CATENATE_N_MINUS;
}
-"^"{WSP}*({UINTEGER}|"0") {
+"^"{WSP}*({UINTEGER}|"0") {
CR;
yylval->value = strtol(yytext + 1, 0, 10);
return CATENATE_N;
@@ -183,7 +188,7 @@ BRACED [{]([^}]|[\300-\337].|[\340-\357]..|[\360-\367]...)+[}]
".u" { CR; return XRE_UPPER; }
".l" { CR; return XRE_LOWER; }
-"@bin\""[^""]+"\""|"@\""[^""]+"\"" {
+"@bin\""[^""]+"\""|"@\""[^""]+"\"" {
CR;
yylval->label = hfst::xre::get_quoted(yytext);
return READ_BIN;
@@ -226,15 +231,19 @@ BRACED [{]([^}]|[\300-\337].|[\340-\357]..|[\360-\367]...)+[}]
":" { CR; return PAIR_SEPARATOR; }
"::"{WEIGHT} {
- CR;
+ CR;
yylval->weight = hfst::xre::get_weight(yytext + 2);
return WEIGHT;
}
"\""[^""]+"\"" {
- yylval->label = hfst::xre::parse_quoted(yytext);
+ unsigned int length = 0;
+ yylval->label = hfst::xre::parse_quoted(yytext, length);
CR;
- return QUOTED_LITERAL;
+ if (length < 2)
+ return QUOTED_LITERAL;
+ else
+ return QUOTED_MULTICHAR_LITERAL;
}
@@ -254,10 +263,10 @@ BRACED [{]([^}]|[\300-\337].|[\340-\357]..|[\360-\367]...)+[}]
}
yylval->label = hfst::xre::strip_percents(yytext);
CR;
- return SYMBOL;
-}
+ return MULTICHAR_SYMBOL;
+}
-{NAME_CH}({NAME_CH}|"0")* {
+{NAME_CH} {
if (hfst::xre::position_symbol != NULL) {
if (strcmp(hfst::xre::position_symbol, yytext) == 0) {
hfst::xre::positions.insert(hfst::xre::cr);
@@ -266,7 +275,30 @@ BRACED [{]([^}]|[\300-\337].|[\340-\357]..|[\360-\367]...)+[}]
yylval->label = hfst::xre::strip_percents(yytext);
CR;
return SYMBOL;
-}
+}
+
+{NAME_CH}({NAME_CH}|"0")+ {
+ if (hfst::xre::position_symbol != NULL) {
+ if (strcmp(hfst::xre::position_symbol, yytext) == 0) {
+ hfst::xre::positions.insert(hfst::xre::cr);
+ }
+ }
+ yylval->label = hfst::xre::strip_percents(yytext);
+ CR;
+ return MULTICHAR_SYMBOL;
+}
+
+".#." {
+ if (hfst::xre::position_symbol != NULL) {
+ if (strcmp(hfst::xre::position_symbol, yytext) == 0) {
+ hfst::xre::positions.insert(hfst::xre::cr);
+ }
+ }
+ yylval->label = hfst::xre::strip_percents(yytext);
+ CR;
+ return MULTICHAR_SYMBOL;
+}
+
{NAME_CH}({NAME_CH}|"0")*"(" {
CR;
@@ -280,19 +312,19 @@ BRACED [{]([^}]|[\300-\337].|[\340-\357]..|[\360-\367]...)+[}]
return FUNCTION_NAME;
}
-";" {
- CR;
+";" {
+ CR;
return END_OF_EXPRESSION;
}
-{LWSP}+ { CR; /*fprintf(stderr, "ignoring whitespace '%s'..\n", yytext); */ /* ignorable whitespace */ }
+{LWSP}+ { hfst::xre::count_lines(yytext); /*fprintf(stderr, "ignoring whitespace '%s'..\n", yytext); */ /* ignorable whitespace */ }
("!"|"#")[^\n]*$ { CR; /* fprintf(stderr, "ignoring comment '%s'..\n", yytext); */ /* ignore comments */ }
("!"|"#")[^\n]* { CR; /* fprintf(stderr, "ignoring comment '%s'..\n", yytext); */ /* ignore comments */ }
-. {
- CR;
+. {
+ CR;
return LEXER_ERROR;
}
diff --git a/libhfst/src/parsers/xre_parse.yy b/libhfst/src/parsers/xre_parse.yy
index ada97af..8a32471 100644
--- a/libhfst/src/parsers/xre_parse.yy
+++ b/libhfst/src/parsers/xre_parse.yy
@@ -1,14 +1,14 @@
%{
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
-#define YYDEBUG 1
+#define YYDEBUG 1
#include <stdio.h>
#include <assert.h>
@@ -25,7 +25,7 @@ using namespace hfst::implementations;
#include "xre_utils.h"
-namespace hfst {
+namespace hfst {
namespace xre {
// number of characters read, used for scanning function definition xre for argument symbols
extern unsigned int cr;
@@ -45,8 +45,8 @@ namespace hfst {
}
bool is_weighted()
- {
- return (hfst::xre::format == hfst::TROPICAL_OPENFST_TYPE ||
+ {
+ return (hfst::xre::format == hfst::TROPICAL_OPENFST_TYPE ||
hfst::xre::format == hfst::LOG_OPENFST_TYPE);
}
}
@@ -94,14 +94,14 @@ int xrelex ( YYSTYPE * , yyscan_t );
hfst::HfstTransducerVector* transducerVector;
std::pair<hfst::xeroxRules::ReplaceArrow, std::vector<hfst::xeroxRules::Rule> >* replaceRuleVectorWithArrow;
- std::pair< hfst::xeroxRules::ReplaceArrow, hfst::xeroxRules::Rule>* replaceRuleWithArrow;
+ std::pair< hfst::xeroxRules::ReplaceArrow, hfst::xeroxRules::Rule>* replaceRuleWithArrow;
std::pair< hfst::xeroxRules::ReplaceArrow, hfst::HfstTransducerPairVector>* mappingVectorWithArrow;
std::pair< hfst::xeroxRules::ReplaceArrow, hfst::HfstTransducerPair>* mappingWithArrow;
std::pair<hfst::xeroxRules::ReplaceType, hfst::HfstTransducerPairVector>* contextWithMark;
hfst::xeroxRules::ReplaceType replType;
- hfst::xeroxRules::ReplaceArrow replaceArrow;
+ hfst::xeroxRules::ReplaceArrow replaceArrow;
}
@@ -121,14 +121,14 @@ int xrelex ( YYSTYPE * , yyscan_t );
%type <transducerPairVector> CONTEXTS_VECTOR RESTR_CONTEXTS_VECTOR
%type <transducerPair> CONTEXT RESTR_CONTEXT
%type <replType> CONTEXT_MARK
-%type <label> HALFARC SUB2
+%type <label> HALFARC SUB2
%type <label> SYMBOL_OR_QUOTED
%type <transducerVector> REGEXP_LIST // function call
%type <label> FUNCTION // function call
%nonassoc <weight> WEIGHT
-%nonassoc <label> SYMBOL CURLY_BRACKETS
+%nonassoc <label> SYMBOL MULTICHAR_SYMBOL CURLY_BRACKETS
%left CROSS_PRODUCT COMPOSITION LENIENT_COMPOSITION INTERSECTION MERGE_RIGHT_ARROW MERGE_LEFT_ARROW
%left CENTER_MARKER MARKUP_MARKER
@@ -140,7 +140,7 @@ int xrelex ( YYSTYPE * , yyscan_t );
RTL_LONGEST_MATCH RTL_SHORTEST_MATCH
LTR_LONGEST_MATCH LTR_SHORTEST_MATCH
-%right REPLACE_CONTEXT_UU REPLACE_CONTEXT_LU
+%right REPLACE_CONTEXT_UU REPLACE_CONTEXT_LU
REPLACE_CONTEXT_UL REPLACE_CONTEXT_LL
%left UNION MINUS UPPER_MINUS LOWER_MINUS UPPER_PRIORITY_UNION
@@ -150,7 +150,7 @@ int xrelex ( YYSTYPE * , yyscan_t );
%left COMMACOMMA
-%left COMMA
+%left COMMA
%left BEFORE AFTER
@@ -167,47 +167,37 @@ int xrelex ( YYSTYPE * , yyscan_t );
%token EPSILON_TOKEN ANY_TOKEN BOUNDARY_MARKER
%token LEXER_ERROR
%token END_OF_EXPRESSION
-%token PAIR_SEPARATOR
-%nonassoc <label> QUOTED_LITERAL
+%token PAIR_SEPARATOR
+%nonassoc <label> QUOTED_LITERAL QUOTED_MULTICHAR_LITERAL
%%
-XRE: REGEXP1 { }
- |
- {
+XRE: REGEXP1 { $$ = $1; }
+ |
+ {
// only comments
hfst::xre::contains_only_comments = true;
return 0;
}
;
REGEXP1: REGEXP2 END_OF_EXPRESSION {
- // std::cerr << "regexp1:regexp2 end of expr \n"<< std::endl;
- // Symbols of form <foo> are not harmonized in xfst, that is why
- // they are escaped as @_<foo>_@ and need to be unescaped finally.
- // hfst::xre::last_compiled = & hfst::xre::unescape_enclosing_angle_brackets($1)->minimize();
- hfst::xre::last_compiled = & $1->minimize();
+ hfst::xre::last_compiled = $1;
$$ = hfst::xre::last_compiled;
if (hfst::xre::allow_extra_text_at_end) {
return 0;
}
}
| REGEXP2 {
-
- //std::cerr << "regexp1:regexp2\n"<< *$1 << std::endl;
- // Symbols of form <foo> are not harmonized in xfst, that is why
- // they are escaped as @_<foo>_@ and need to be unescaped finally.
- // hfst::xre::last_compiled = & hfst::xre::unescape_enclosing_angle_brackets($1)->minimize();
- hfst::xre::last_compiled = & $1->minimize();
+ hfst::xre::last_compiled = $1;
$$ = hfst::xre::last_compiled;
}
;
REGEXP2: REPLACE
- {
- $$ = & $1->minimize();
- // std::cerr << "regexp2:replace \n"<< std::endl;
+ {
+ $$ = & $1->optimize();
}
- | REGEXP2 COMPOSITION REPLACE
+ | REGEXP2 COMPOSITION REPLACE
{
if ($1->has_flag_diacritics() && $3->has_flag_diacritics())
{
@@ -220,10 +210,11 @@ REGEXP2: REPLACE
}
try {
- $$ = & $1->compose(*$3, harmonize_).minimize();
+ $$ = & $1->compose(*$3, harmonize_).optimize();
}
catch (const FlagDiacriticsAreNotIdentitiesException & e)
{
+ (void)e;
xreerror("Error: flag diacritics must be identities in composition if flag-is-epsilon is ON.\n"
"I.e. only FLAG:FLAG is allowed, not FLAG1:FLAG2, FLAG:bar or foo:FLAG\n"
"Apply twosided flag-diacritics (tfd) before composition.\n");
@@ -232,19 +223,20 @@ REGEXP2: REPLACE
delete $3;
}
| REGEXP2 CROSS_PRODUCT REPLACE {
- $$ = & $1->cross_product(*$3);
+ $$ = & $1->cross_product(*$3).optimize();
delete $3;
}
| REGEXP2 LENIENT_COMPOSITION REPLACE {
- $$ = & $1->lenient_composition(*$3).minimize();
+ $$ = & $1->lenient_composition(*$3).optimize();
delete $3;
}
| REGEXP2 MERGE_RIGHT_ARROW REPLACE {
try {
- $$ = hfst::xre::merge_first_to_second($1, $3);
+ $$ = & hfst::xre::merge_first_to_second($1, $3)->optimize();
}
catch (const TransducersAreNotAutomataException & e)
{
+ (void)e;
xreerror("Error: transducers must be automata in merge operation.");
delete $1;
delete $3;
@@ -253,13 +245,14 @@ REGEXP2: REPLACE
delete $1;
}
| REGEXP2 MERGE_LEFT_ARROW REPLACE {
- $$ = hfst::xre::merge_first_to_second($3, $1);
+ $$ = & hfst::xre::merge_first_to_second($3, $1)->optimize();
delete $3;
}
// substitute
| SUB1 HALFARC PAIR_SEPARATOR HALFARC COMMA HALFARC PAIR_SEPARATOR HALFARC RIGHT_BRACKET {
$1->substitute(StringPair($2,$4), StringPair($6,$8));
- $$ = $1;
+ $$ = & $1->optimize();
+ free($2); free($4); free($6); free($8);
}
| SUB1 SUB2 SUB3 {
@@ -267,11 +260,11 @@ REGEXP2: REPLACE
if (hfst::xre::is_definition($2))
{
hfst::xre::warn("warning: using definition as an ordinary label, cannot substitute\n");
- $$ = $1;
+ $$ = & $1->optimize();
}
else if (alpha.find($2) == alpha.end())
{
- $$ = $1;
+ $$ = & $1->optimize();
}
else
{
@@ -285,7 +278,7 @@ REGEXP2: REPLACE
if (empty.compare(*$3))
{
empty_replace_transducer=true;
- }
+ }
if (empty_replace_transducer)
{
@@ -293,13 +286,13 @@ REGEXP2: REPLACE
// as they will be removed anyway
hfst::xre::set_substitution_function_symbol($2);
tmpTr->substitute(&hfst::xre::substitution_function);
- }
+ }
// `[ a:b, b, x y ]
// substitute b with x | y
tmpTr->substitute(tmp, *$3, false); // no harmonization
- if (!empty_replace_transducer)
+ if (!empty_replace_transducer)
{
// a:b .o. b -> x | y
// [[a:b].i .o. b -> x | y].i - this is for cases when b is on left side
@@ -322,9 +315,9 @@ REGEXP2: REPLACE
replaceTr.insert_freely(StringPair(*it, *it), false);
}
}
- replaceTr.minimize();
+ replaceTr.optimize();
- tmpTr->compose(replaceTr).minimize();
+ tmpTr->compose(replaceTr).optimize();
tmpTr->invert().compose(replaceTr).invert();
}
@@ -332,9 +325,9 @@ REGEXP2: REPLACE
{
tmpTr->remove_from_alphabet($2);
}
- tmpTr->minimize();
+ tmpTr->optimize();
$$ = tmpTr;
- delete($1, $2, $3);
+ delete $1; delete $2; delete $3;
}
}
;
@@ -342,7 +335,7 @@ REGEXP2: REPLACE
SUB1: SUBSTITUTE_LEFT LEFT_BRACKET REPLACE COMMA { $$ = $3; } ; // first argument
SUB2: HALFARC COMMA { $$ = $1; } ; // symbol that needs to be replaced
SUB3: SYMBOL_LIST RIGHT_BRACKET { $$ = $1; } // symbol list
- |
+ |
RIGHT_BRACKET { $$ = new HfstTransducer(hfst::xre::format); } // an empty symbol list
;
@@ -350,7 +343,7 @@ SUB3: SYMBOL_LIST RIGHT_BRACKET { $$ = $1; } // symbol list
// Replace operators
///////////////////////////
-REPLACE : REGEXP3 { }
+REPLACE : REGEXP3 { $$ = $1; }
| PARALLEL_RULES
{
switch ( $1->first )
@@ -392,7 +385,7 @@ REPLACE : REGEXP3 { }
PARALLEL_RULES: PARALLEL_RULES COMMACOMMA RULE
{
- //std::cerr << "parallel_rules: parallel_rules ,, rule"<< std::endl;
+ //std::cerr << "parallel_rules: parallel_rules ,, rule"<< std::endl;
if ($3->first != $1->first)
{
xreerror("Replace type mismatch in parallel rules");
@@ -401,22 +394,23 @@ PARALLEL_RULES: PARALLEL_RULES COMMACOMMA RULE
Rule tmpRule($3->second);
$1->second.push_back(tmpRule);
$$ = new std::pair< ReplaceArrow, std::vector<Rule> > ($3->first, $1->second);
- delete $3;
+ delete $1; delete $3;
}
| RULE
{
- //std::cerr << "parallel_rules:rule"<< std::endl;
+ //std::cerr << "parallel_rules:rule"<< std::endl;
std::vector<Rule> * ruleVector = new std::vector<Rule>();
ruleVector->push_back($1->second);
$$ = new std::pair< ReplaceArrow, std::vector<Rule> > ($1->first, *ruleVector);
+ delete ruleVector;
delete $1;
}
;
RULE: MAPPINGPAIR_VECTOR
{
- // std::cerr << "rule: mapping_vector"<< std::endl;
+ // std::cerr << "rule: mapping_vector"<< std::endl;
// HfstTransducer allMappingsDisjuncted = disjunctVectorMembers($1->second);
Rule rule( $1->second );;
@@ -425,40 +419,40 @@ RULE: MAPPINGPAIR_VECTOR
}
| MAPPINGPAIR_VECTOR CONTEXTS_WITH_MARK
{
- // std::cerr << "rule: mapping_vector contextsWM"<< std::endl;
+ // std::cerr << "rule: mapping_vector contextsWM"<< std::endl;
// HfstTransducer allMappingsDisjuncted = disjunctVectorMembers($1->second);
Rule rule( $1->second, $2->second, $2->first );
$$ = new std::pair< ReplaceArrow, Rule> ($1->first, rule);
- delete $1, $2;
+ delete $1; delete $2;
}
;
// Mappings: ( ie. a -> b , c -> d , ... , g -> d)
MAPPINGPAIR_VECTOR: MAPPINGPAIR_VECTOR COMMA MAPPINGPAIR
{
- // std::cerr << "mapping_vector : mapping_vector comma mapping"<< std::endl;
+ // std::cerr << "mapping_vector : mapping_vector comma mapping"<< std::endl;
// check if new Arrow is the same as the first one
if ($1->first != $3->first)
{
hfst::xre::warn("Replace arrows should be the same. Calculated as if all replacements had the first arrow.");
- //exit(1);
}
$1->second.push_back($3->second);
$$ = new std::pair< ReplaceArrow, HfstTransducerPairVector> ($1->first, $1->second);
- delete $3;
+ delete $1; delete $3;
}
| MAPPINGPAIR
{
- // std::cerr << "mapping_vector : mapping"<< std::endl;
+ // std::cerr << "mapping_vector : mapping"<< std::endl;
HfstTransducerPairVector * mappingPairVector = new HfstTransducerPairVector();
mappingPairVector->push_back( $1->second );
$$ = new std::pair< ReplaceArrow, HfstTransducerPairVector> ($1->first, * mappingPairVector);
- delete $1;
+ delete mappingPairVector;
+ delete $1;
}
;
@@ -466,12 +460,12 @@ MAPPINGPAIR_VECTOR: MAPPINGPAIR_VECTOR COMMA MAPPINGPAIR
MAPPINGPAIR: REPLACE REPLACE_ARROW REPLACE
{
- hfst::xre::warn_about_special_symbols_in_replace($1);
- hfst::xre::warn_about_special_symbols_in_replace($3);
+ hfst::xre::warn_about_special_symbols_in_replace($1);
+ hfst::xre::warn_about_special_symbols_in_replace($3);
HfstTransducerPair mappingPair(*$1, *$3);
$$ = new std::pair< ReplaceArrow, HfstTransducerPair> ($2, mappingPair);
- delete $1, $3;
+ delete $1; delete $3;
}
| REPLACE REPLACE_ARROW REPLACE MARKUP_MARKER REPLACE
{
@@ -480,7 +474,7 @@ MAPPINGPAIR: REPLACE REPLACE_ARROW REPLACE
HfstTransducerPair mappingPair = create_mapping_for_mark_up_replace( tmpMappingPair, marks );
$$ = new std::pair< ReplaceArrow, HfstTransducerPair> ($2, mappingPair);
- delete $1, $3, $5;
+ delete $1; delete $3; delete $5;
}
| REPLACE REPLACE_ARROW REPLACE MARKUP_MARKER
{
@@ -490,7 +484,7 @@ MAPPINGPAIR: REPLACE REPLACE_ARROW REPLACE
HfstTransducerPair mappingPair = create_mapping_for_mark_up_replace( tmpMappingPair, marks );
$$ = new std::pair< ReplaceArrow, HfstTransducerPair> ($2, mappingPair);
- delete $1, $3;
+ delete $1; delete $3;
}
| REPLACE REPLACE_ARROW MARKUP_MARKER REPLACE
{
@@ -500,7 +494,7 @@ MAPPINGPAIR: REPLACE REPLACE_ARROW REPLACE
HfstTransducerPair mappingPair = create_mapping_for_mark_up_replace( tmpMappingPair, marks );
$$ = new std::pair< ReplaceArrow, HfstTransducerPair> ($2, mappingPair);
- delete $1, $4;
+ delete $1; delete $4;
}
| LEFT_BRACKET_DOTTED RIGHT_BRACKET_DOTTED REPLACE_ARROW REPLACE
{
@@ -516,7 +510,7 @@ MAPPINGPAIR: REPLACE REPLACE_ARROW REPLACE
{
HfstTransducerPair mappingPair(*$2, *$5);
$$ = new std::pair< ReplaceArrow, HfstTransducerPair> ($4, mappingPair);
- delete $2, $5;
+ delete $2; delete $5;
}
| REPLACE REPLACE_ARROW LEFT_BRACKET_DOTTED RIGHT_BRACKET_DOTTED
@@ -531,34 +525,35 @@ MAPPINGPAIR: REPLACE REPLACE_ARROW REPLACE
{
HfstTransducerPair mappingPair(*$1, *$4);
$$ = new std::pair< ReplaceArrow, HfstTransducerPair> ($2, mappingPair);
- delete $1, $4;
+ delete $1; delete $4;
}
- ;
+ ;
// Contexts: ( ie. || k _ f , ... , f _ s )
CONTEXTS_WITH_MARK: CONTEXT_MARK CONTEXTS_VECTOR
- {
+ {
$$ = new std::pair< ReplaceType, HfstTransducerPairVector> ($1, *$2);
- }
+ delete $2;
+ }
;
CONTEXTS_VECTOR: CONTEXT
{
HfstTransducerPairVector * ContextVector = new HfstTransducerPairVector();
ContextVector->push_back(*$1);
$$ = ContextVector;
- delete $1;
+ delete $1;
}
| CONTEXTS_VECTOR COMMA CONTEXT
{
$1->push_back(*$3);
$$ = $1;
- delete $3;
+ delete $3;
}
;
-CONTEXT: REPLACE CENTER_MARKER REPLACE
+CONTEXT: REPLACE CENTER_MARKER REPLACE
{
if (hfst::xre::has_non_identity_pairs($1)) // if non-identity symbols present..
{
@@ -572,24 +567,24 @@ CONTEXT: REPLACE CENTER_MARKER REPLACE
}
HfstTransducer t1(*$1);
- HfstTransducer t2(*$3);
+ HfstTransducer t2(*$3);
if (hfst::xre::is_weighted())
{
hfst::xre::has_weight_been_zeroed=false;
t1.transform_weights(&hfst::xre::zero_weights);
}
- t1.minimize().prune_alphabet(false);
+ t1.optimize().prune_alphabet(false);
if (hfst::xre::is_weighted())
{
t2.transform_weights(&hfst::xre::zero_weights);
hfst::xre::has_weight_been_zeroed=false;
}
- t2.minimize().prune_alphabet(false);
+ t2.optimize().prune_alphabet(false);
$$ = new HfstTransducerPair(t1, t2);
- delete $1, $3;
+ delete $1; delete $3;
}
| REPLACE CENTER_MARKER
{
@@ -607,11 +602,11 @@ CONTEXT: REPLACE CENTER_MARKER REPLACE
t1.transform_weights(&hfst::xre::zero_weights);
hfst::xre::has_weight_been_zeroed=false;
}
- t1.minimize().prune_alphabet(false);
+ t1.optimize().prune_alphabet(false);
- HfstTransducer epsilon(hfst::internal_epsilon, hfst::xre::format);
+ HfstTransducer epsilon(hfst::internal_epsilon, hfst::xre::format);
$$ = new HfstTransducerPair(t1, epsilon);
- delete $1;
+ delete $1;
}
| CENTER_MARKER REPLACE
{
@@ -630,11 +625,11 @@ CONTEXT: REPLACE CENTER_MARKER REPLACE
t1.transform_weights(&hfst::xre::zero_weights);
hfst::xre::has_weight_been_zeroed=false;
}
- t1.minimize().prune_alphabet(false);
+ t1.optimize().prune_alphabet(false);
HfstTransducer epsilon(hfst::internal_epsilon, hfst::xre::format);
$$ = new HfstTransducerPair(epsilon, t1);
- delete $2;
+ delete $2;
}
| CENTER_MARKER
{
@@ -646,7 +641,7 @@ CONTEXT: REPLACE CENTER_MARKER REPLACE
{
$$ = REPL_UP;
}
- | REPLACE_CONTEXT_LU
+ | REPLACE_CONTEXT_LU
{
$$ = REPL_RIGHT;
}
@@ -695,7 +690,7 @@ REPLACE_ARROW: REPLACE_RIGHT
;
////////////////
-REGEXP3: REGEXP4 { }
+REGEXP3: REGEXP4 { $$ = $1; }
| REGEXP3 SHUFFLE REGEXP4 {
xreerror("No shuffle");
//$$ = $1;
@@ -704,16 +699,16 @@ REGEXP3: REGEXP4 { }
}
| REGEXP3 BEFORE REGEXP4 {
$$ = new HfstTransducer( before (*$1, *$3) );
- delete $1, $3;
+ delete $1; delete $3;
}
| REGEXP3 AFTER REGEXP4 {
$$ = new HfstTransducer( after (*$1, *$3) );
- delete $1, $3;
+ delete $1; delete $3;
}
;
-REGEXP4: REGEXP5 { }
+REGEXP4: REGEXP5 { $$ = $1; }
// restriction rule
| REGEXP4 RIGHT_ARROW RESTR_CONTEXTS_VECTOR {
$$ = new HfstTransducer( restriction(*$1, *$3) ) ;
@@ -743,22 +738,22 @@ RESTR_CONTEXTS_VECTOR: RESTR_CONTEXT
HfstTransducerPairVector * ContextVector = new HfstTransducerPairVector();
ContextVector->push_back(*$1);
$$ = ContextVector;
- delete $1;
+ delete $1;
}
| RESTR_CONTEXTS_VECTOR COMMA RESTR_CONTEXT
{
$1->push_back(*$3);
$$ = $1;
- delete $3;
+ delete $3;
}
;
-RESTR_CONTEXT: REGEXP4 CENTER_MARKER REGEXP4
+RESTR_CONTEXT: REGEXP4 CENTER_MARKER REGEXP4
{
$$ = new HfstTransducerPair(*$1, *$3);
- delete $1, $3;
+ delete $1; delete $3;
}
| REGEXP4 CENTER_MARKER
{
@@ -768,13 +763,13 @@ RESTR_CONTEXT: REGEXP4 CENTER_MARKER REGEXP4
// std::cerr << "Epsilon: \n" << epsilon << std::endl;
$$ = new HfstTransducerPair(*$1, epsilon);
- delete $1;
+ delete $1;
}
| CENTER_MARKER REGEXP4
{
HfstTransducer epsilon(hfst::internal_epsilon, hfst::xre::format);
$$ = new HfstTransducerPair(epsilon, *$2);
- delete $2;
+ delete $2;
}
| CENTER_MARKER
{
@@ -784,14 +779,14 @@ RESTR_CONTEXT: REGEXP4 CENTER_MARKER REGEXP4
;
-REGEXP5: REGEXP6 { }
+REGEXP5: REGEXP6 { $$ = $1; }
| REGEXP5 UNION REGEXP6 {
$$ = & $1->disjunct(*$3, harmonize_);
delete $3;
}
| REGEXP5 INTERSECTION REGEXP6 {
// std::cerr << "Intersection: \n" << std::endl;
- $$ = & $1->intersect(*$3, harmonize_).minimize().prune_alphabet(false);
+ $$ = & $1->intersect(*$3, harmonize_).optimize().prune_alphabet(false);
delete $3;
}
| REGEXP5 MINUS REGEXP6 {
@@ -820,19 +815,21 @@ REGEXP5: REGEXP6 { }
right->invert();
left->invert();
$$ = & (left->priority_union(*right).invert());
- delete $1, $3;
+ delete $1; delete $3;
}
;
-REGEXP6: REGEXP7 { }
- | REGEXP6 REGEXP7 {
+REGEXP6: REGEXP7 { $$ = $1; }
+ | REGEXP6 REGEXP7 {
$$ = & $1->concatenate(*$2, harmonize_);
delete $2;
}
;
-REGEXP7: REGEXP8 { }
+REGEXP7: REGEXP8 { $$ = $1; }
| REGEXP7 IGNORING REGEXP8 {
+ // this is how ignoring is done in foma and xfst
+ $1->harmonize(*$3, true /*force harmonization also for foma type*/);
$$ = & $1->insert_freely(*$3, false); // no harmonization
delete $3;
}
@@ -850,11 +847,11 @@ REGEXP7: REGEXP8 { }
}
;
-REGEXP8: REGEXP9 { }
+REGEXP8: REGEXP9 { $$ = $1; }
| COMPLEMENT REGEXP8 {
// TODO: forbid pair complement (ie ~a:b)
HfstTransducer complement = HfstTransducer::identity_pair( hfst::xre::format );
- complement.repeat_star().minimize();
+ complement.repeat_star().optimize();
complement.subtract(*$2).prune_alphabet(false);
$$ = new HfstTransducer(complement);
delete $2;
@@ -879,7 +876,7 @@ REGEXP8: REGEXP9 { }
xreerror("Containment with weight only works with automata");
YYABORT;
}
- $$ = hfst::xre::contains_with_weight($3, $2);
+ $$ = hfst::xre::contains_with_weight($3, hfst::double_to_float($2));
delete $3;
}
| CONTAINMENT_ONCE REGEXP8 {
@@ -894,7 +891,7 @@ REGEXP8: REGEXP9 { }
}
;
-REGEXP9: REGEXP10 { }
+REGEXP9: REGEXP10 { $$ = $1; }
| REGEXP9 STAR {
$$ = & $1->repeat_star();
}
@@ -917,7 +914,7 @@ REGEXP9: REGEXP10 { }
$$ = & $1->repeat_n($2);
}
| REGEXP9 CATENATE_N_PLUS {
- //std::cerr << "value is ::::: \n"<< $2 << std::endl;
+ //std::cerr << "value is ::::: \n"<< $2 << std::endl;
$$ = & $1->repeat_n_plus($2+1);
}
| REGEXP9 CATENATE_N_MINUS {
@@ -929,7 +926,7 @@ REGEXP9: REGEXP10 { }
}
;
-REGEXP10: REGEXP11 { }
+REGEXP10: REGEXP11 { $$ = $1; }
| TERM_COMPLEMENT REGEXP10 {
HfstTransducer* any = new HfstTransducer(hfst::internal_identity,
hfst::xre::format);
@@ -945,9 +942,9 @@ REGEXP10: REGEXP11 { }
*/
;
-REGEXP11: REGEXP12 { }
+REGEXP11: REGEXP12 { $$ = $1; }
| LEFT_BRACKET REGEXP2 RIGHT_BRACKET {
- $$ = & $2->minimize();
+ $$ = & $2->optimize();
}
// [foo]:[bar]
| LEFT_BRACKET REGEXP2 RIGHT_BRACKET PAIR_SEPARATOR LEFT_BRACKET REGEXP2 RIGHT_BRACKET {
@@ -983,14 +980,14 @@ REGEXP11: REGEXP12 { }
delete $4;
}
| LEFT_BRACKET REGEXP2 RIGHT_BRACKET WEIGHT {
- $$ = & $2->set_final_weights($4, true).minimize();
+ $$ = & $2->set_final_weights(hfst::double_to_float($4), true).optimize();
}
| LEFT_PARENTHESIS REGEXP2 RIGHT_PARENTHESIS {
$$ = & $2->optionalize();
}
;
-// building 3rd argument in the substitute list
+// building 3rd argument in the substitute list
SYMBOL_LIST: HALFARC {
if (strcmp($1, hfst::internal_unknown.c_str()) == 0)
{
@@ -1001,7 +998,7 @@ SYMBOL_LIST: HALFARC {
$$ = new HfstTransducer($1, $1, hfst::xre::format);
}
free($1);
- }
+ }
| SYMBOL_LIST HALFARC {
HfstTransducer * tmp ;
if (strcmp($2, hfst::internal_unknown.c_str()) == 0)
@@ -1014,14 +1011,15 @@ SYMBOL_LIST: HALFARC {
}
$1->disjunct(*tmp, false); // do not harmonize
- $$ = & $1->minimize();
- delete $2, tmp;
+ $$ = & $1->optimize();
+ free($2);
+ delete tmp;
}
;
-REGEXP12: LABEL { }
- | LABEL WEIGHT {
- $$ = & $1->set_final_weights($2, true);
+REGEXP12: LABEL { $$ = $1; }
+ | LABEL WEIGHT {
+ $$ = & $1->set_final_weights(hfst::double_to_float($2), true);
}
| READ_BIN {
try {
@@ -1035,12 +1033,14 @@ REGEXP12: LABEL { }
char msg [256];
sprintf(msg, "Error reading transducer file '%s'.", $1);
xreerror(msg);
+ free($1);
YYABORT;
}
}
| READ_TEXT {
FILE * f = NULL;
- f = fopen($1, "r");
+ f = hfst::hfst_fopen($1, "r");
+ free($1);
if (f == NULL) {
xreerror("File cannot be opened.\n");
YYABORT;
@@ -1057,14 +1057,15 @@ REGEXP12: LABEL { }
tmp.disjunct(spv, 0);
}
fclose(f);
- HfstTransducer * retval = new HfstTransducer(tmp, hfst::xre::format);
- retval->minimize();
+ HfstTransducer * retval = new HfstTransducer(tmp, hfst::xre::format);
+ retval->optimize();
$$ = retval;
}
}
| READ_SPACED {
FILE * f = NULL;
- f = fopen($1, "r");
+ f = hfst::hfst_fopen($1, "r");
+ free($1);
if (f == NULL) {
xreerror("File cannot be opened.\n");
YYABORT;
@@ -1081,14 +1082,15 @@ REGEXP12: LABEL { }
tmp.disjunct(spv, 0);
}
fclose(f);
- HfstTransducer * retval = new HfstTransducer(tmp, hfst::xre::format);
- retval->minimize();
+ HfstTransducer * retval = new HfstTransducer(tmp, hfst::xre::format);
+ retval->optimize();
$$ = retval;
}
}
| READ_PROLOG {
FILE * f = NULL;
- f = fopen($1, "r");
+ f = hfst::hfst_fopen($1, "r");
+ free($1);
if (f == NULL) {
xreerror("File cannot be opened.\n");
YYABORT;
@@ -1099,7 +1101,7 @@ REGEXP12: LABEL { }
HfstBasicTransducer tmp = HfstBasicTransducer::read_in_prolog_format(f, linecount);
fclose(f);
HfstTransducer * retval = new HfstTransducer(tmp, hfst::xre::format);
- retval->minimize();
+ retval->optimize();
$$ = retval;
}
catch (const HfstException & e) {
@@ -1112,16 +1114,18 @@ REGEXP12: LABEL { }
}
| READ_RE {
FILE * f = NULL;
- f = fopen($1, "r");
+ f = hfst::hfst_fopen($1, "r");
if (f == NULL) {
xreerror("File cannot be opened.\n");
fclose(f);
+ free($1);
YYABORT;
}
else {
fclose(f);
// read the regex in a string
std::ifstream ifs($1);
+ free($1);
std::stringstream buffer;
buffer << ifs.rdbuf();
char * regex_string = strdup(buffer.str().c_str());
@@ -1201,6 +1205,7 @@ LABEL: HALFARC {
// function call
| FUNCTION REGEXP_LIST RIGHT_PARENTHESIS {
if (! hfst::xre::is_valid_function_call($1, $2)) {
+ delete $1; delete $2;
return EXIT_FAILURE;
}
else {
@@ -1213,17 +1218,20 @@ LABEL: HALFARC {
if (! hfst::xre::define_function_args($1, $2))
{
xreerror("Could not define function args.\n"); // TODO: more informative message
+ delete $1; delete $2;
YYABORT;
}
- // if we are scanning a function definition for argument symbols,
- // do not include the characters read when evaluating functions inside it
+ delete $2;
+ // if we are scanning a function definition for argument symbols,
+ // do not include the characters read when evaluating functions inside it
unsigned int chars_read = hfst::xre::cr;
int parse_retval = xreparse(scanner);
hfst::xre::cr = chars_read;
hfst::xre::undefine_function_args($1);
+ delete $1;
xre_delete_buffer(bs,scanner);
xrelex_destroy(scanner);
@@ -1246,6 +1254,14 @@ LABEL: HALFARC {
;
SYMBOL_OR_QUOTED: SYMBOL
+ | MULTICHAR_SYMBOL {
+ hfst::xre::check_multichar_symbol($1);
+ $$ = $1;
+ }
+ | QUOTED_MULTICHAR_LITERAL {
+ hfst::xre::check_multichar_symbol($1);
+ $$ = $1;
+ }
| QUOTED_LITERAL
;
@@ -1253,10 +1269,10 @@ HALFARC: SYMBOL_OR_QUOTED
{
// Symbols of form <foo> are not harmonized in xfst, that is why
// they need to be escaped as @_<foo>_ at .
- // $$ = hfst::xre::escape_enclosing_angle_brackets($1);
+ // $$ = hfst::xre::escape_enclosing_angle_brackets($1);
hfst::xre::warn_about_hfst_special_symbol($1);
hfst::xre::warn_about_xfst_special_symbol($1);
- $$ = $1;
+ $$ = $1;
}
| EPSILON_TOKEN {
$$ = strdup(hfst::internal_epsilon.c_str());
@@ -1269,13 +1285,13 @@ HALFARC: SYMBOL_OR_QUOTED
}
;
-REGEXP_LIST: REGEXP_LIST COMMA REGEXP2 {
+REGEXP_LIST: REGEXP_LIST COMMA REGEXP2 {
$$->push_back(*($3));
- delete $3;
- }
- | REGEXP2 {
+ delete $3;
+ }
+ | REGEXP2 {
$$ = new hfst::HfstTransducerVector();
- $$->push_back(*($1));
+ $$->push_back(*($1));
delete $1;
}
;
diff --git a/libhfst/src/parsers/xre_utils.cc b/libhfst/src/parsers/xre_utils.cc
index 24d65d2..1f16c63 100644
--- a/libhfst/src/parsers/xre_utils.cc
+++ b/libhfst/src/parsers/xre_utils.cc
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
/**
@@ -36,12 +36,14 @@ extern void xre_delete_buffer (YY_BUFFER_STATE, yyscan_t);
extern int xrelex_destroy (yyscan_t);
extern char * xreget_text(yyscan_t);
-namespace hfst {
+namespace hfst {
namespace xre {
extern unsigned int cr; // number of characters read, defined in XreCompiler.cc
+ extern unsigned int lr; // number of lines read, defined in XreCompiler.cc
bool allow_extra_text_at_end = false;
extern std::ostream * error_;
extern bool verbose_;
+ extern std::set<std::string> * defined_multichar_symbols_;
}
}
@@ -53,29 +55,31 @@ std::ostream * xreerrstr()
void xreflush(std::ostream * os)
{
hfst::xre::XreCompiler::flush(os);
-}
+}
int xreerror(yyscan_t scanner, const char* msg)
-{
- char buffer [1024];
-
- int n = sprintf(buffer, "*** xre parsing failed: %s\n", msg);
- if (strlen(hfst::xre::data) < 60)
- {
- n = sprintf(buffer+n, "*** parsing %s [near %s]\n", hfst::xre::data,
- xreget_text(scanner));
- }
- else
- {
- n = sprintf(buffer+n, "*** parsing %60s [near %s]...\n",
- hfst::xre::data, xreget_text(scanner));
- }
-
- buffer[1023] = '\0';
+{
if (hfst::xre::verbose_)
{
+ const char * scanner_msg = xreget_text(scanner);
+
+ char * buffer = (char*) malloc(strlen(msg) + strlen(hfst::xre::data) + strlen(scanner_msg) + 100);
+
+ int n = sprintf(buffer, "*** xre parsing failed: %s\n", msg);
+ if (strlen(hfst::xre::data) < 60)
+ {
+ n = sprintf(buffer+n, "*** parsing %s [near %s] on line %u\n%c", hfst::xre::data,
+ xreget_text(scanner), hfst::xre::lr, '\0');
+ }
+ else
+ {
+ n = sprintf(buffer+n, "*** parsing %60s [near %s] on line %u...\n%c",
+ hfst::xre::data, xreget_text(scanner), hfst::xre::lr, '\0');
+ }
+
std::ostream * err = xreerrstr();
*(err) << std::string(buffer);
+ free(buffer);
xreflush(err);
}
return 0;
@@ -85,7 +89,7 @@ int
xreerror(const char *msg)
{
char buffer [1024];
- int n = sprintf(buffer, "*** xre parsing failed: %s\n", msg);
+ (void) sprintf(buffer, "*** xre parsing failed: %s\n", msg);
buffer[1023] = '\0';
std::ostream * err = xreerrstr();
*err << std::string(buffer);
@@ -93,9 +97,9 @@ xreerror(const char *msg)
return 0;
}
-namespace hfst
-{
-namespace xre
+namespace hfst
+{
+namespace xre
{
char* data;
@@ -126,7 +130,7 @@ bool substitution_function(const hfst::StringPair &p, hfst::StringPairSet &sps)
if (p.first == substitution_function_symbol ||
p.second == substitution_function_symbol)
{
- sps.insert(hfst::StringPair(substitution_function_symbol,
+ sps.insert(hfst::StringPair(substitution_function_symbol,
substitution_function_symbol));
return true;
}
@@ -161,8 +165,8 @@ int
getinput(char *buf, int maxlen)
{
int retval = 0;
- if ( maxlen > len ) {
- maxlen = len;
+ if ( maxlen > (int)len ) {
+ maxlen = hfst::size_t_to_int(len);
}
memcpy(buf, data, maxlen);
data += maxlen;
@@ -182,6 +186,34 @@ strip_newline(char *s)
return s;
}
+void
+count_lines(const char * s)
+{
+ const char * c = s;
+ while(*c != '\0')
+ {
+ if (*c == '\n')
+ {
+ hfst::xre::lr += 1;
+ }
+ else if (*c == '\r')
+ {
+ c++;
+ if (*c == '\n')
+ {
+ hfst::xre::cr += 1;
+ }
+ else
+ {
+ c--;
+ }
+ hfst::xre::lr += 1;
+ }
+ hfst::xre::cr += 1;
+ c++;
+ }
+}
+
char*
strip_curly(const char *s)
{
@@ -285,17 +317,22 @@ get_quoted(const char *s)
}
char*
-parse_quoted(const char *s)
+parse_quoted(const char *s, unsigned int & length)
{
std::ostream * err = xreerrstr();
char* quoted = get_quoted(s);
+
char* rv = static_cast<char*>(malloc(sizeof(char)*strlen(quoted) + 1)); // added + 1
char* p = quoted;
char* r = rv;
while (*p != '\0')
{
- if (*p != '\\')
+ if (*p == '\n' || *p == '\r')
+ {
+ throw "Unescaped newline characters found inside quoted string.";
+ }
+ else if (*p != '\\')
{
*r = *p;
r++;
@@ -399,6 +436,10 @@ parse_quoted(const char *s)
}
*r = '\0';
free(quoted);
+
+ length =
+ hfst::HfstTokenizer::check_utf8_correctness_and_calculate_length(std::string(rv));
+
return rv;
}
@@ -448,7 +489,8 @@ unescape_enclosing_angle_brackets(HfstTransducer *t)
if (substitutions.size() == 0)
return t;
- t->substitute(substitutions).minimize();
+ t->substitute(substitutions);
+ t->optimize();
return t;
}
@@ -457,7 +499,7 @@ get_weight(const char *s)
{
double rv = -3.1415;
const char* weightstart = s;
- while ((*weightstart != '\0') &&
+ while ((*weightstart != '\0') &&
((*weightstart == ' ') || (*weightstart == '\t') ||
(*weightstart == ';')))
{
@@ -542,6 +584,7 @@ compile_first(const string& xre, map<string,HfstTransducer*>& defs,
bool tmp = hfst::xre::allow_extra_text_at_end;
hfst::xre::allow_extra_text_at_end = true;
hfst::xre::cr = 0;
+ hfst::xre::lr = 1;
int parse_retval = xreparse(scanner);
chars_read = hfst::xre::cr;
@@ -574,7 +617,7 @@ bool is_valid_function_call
std::map<std::string, unsigned int >::const_iterator name2args
= function_arguments.find(name);
- if (name2xre == function_definitions.end() ||
+ if (name2xre == function_definitions.end() ||
name2args == function_arguments.end())
{
std::ostream * err = xreerrstr();
@@ -589,10 +632,10 @@ bool is_valid_function_call
if ( number_of_args != args->size())
{
std::ostream * err = xreerrstr();
- *err << "Wrong number of arguments: function '" << name << "' expects "
+ *err << "Wrong number of arguments: function '" << name << "' expects "
<< (int)number_of_args << ", " << (int)args->size() << " given" << std::endl;
xreflush(err);
- //fprintf(stderr, "Wrong number of arguments: function '%s' expects %i, %i given\n",
+ //fprintf(stderr, "Wrong number of arguments: function '%s' expects %i, %i given\n",
// name, (int)number_of_args, (int)args->size());
return false;
}
@@ -602,7 +645,7 @@ bool is_valid_function_call
const char * get_function_xre(const char * name)
{
- std::map<std::string,std::string>::const_iterator it
+ std::map<std::string,std::string>::const_iterator it
= function_definitions.find(name);
if (it == function_definitions.end())
{
@@ -644,6 +687,7 @@ void undefine_function_args(const char * name)
ostringstream os;
os << arg_number;
std::string function_arg = "@" + std::string(name) + os.str() + "@";
+ delete definitions[function_arg];
definitions.erase(function_arg);
//fprintf(stderr, "undefined function arg: '%s', %i:\n", name, arg_number); // DEBUG
}
@@ -663,7 +707,7 @@ expand_definition(const char* symbol)
if (expand_definitions)
{
for (std::map<std::string,hfst::HfstTransducer*>::const_iterator it
- = definitions.begin(); it != definitions.end(); it++)
+ = definitions.begin(); it != definitions.end(); it++)
{
if (strcmp(it->first.c_str(), symbol) == 0)
{
@@ -681,7 +725,7 @@ expand_definition(HfstTransducer* tr, const char* symbol)
if (expand_definitions)
{
for (std::map<std::string,hfst::HfstTransducer*>::const_iterator it
- = definitions.begin(); it != definitions.end(); it++)
+ = definitions.begin(); it != definitions.end(); it++)
{
if (strcmp(it->first.c_str(), symbol) == 0)
{
@@ -698,7 +742,7 @@ expand_definition(HfstTransducer* tr, const char* symbol)
return tr;
}
-static const char * get_print_format(const char * symbol)
+ /*static const char * get_print_format(const char * symbol)
{
if (strcmp(hfst::internal_identity.c_str(), symbol) == 0)
return "?";
@@ -707,7 +751,7 @@ static const char * get_print_format(const char * symbol)
if (strcmp(hfst::internal_epsilon.c_str(), symbol) == 0)
return "0";
return symbol;
-}
+ }*/
HfstTransducer*
xfst_curly_label_to_transducer(const char* input, const char* output)
@@ -733,7 +777,7 @@ xfst_curly_label_to_transducer(const char* input, const char* output)
{
HfstTransducer tmp(hfst::internal_epsilon, *it, hfst::xre::format);
retval->concatenate(tmp, false);
- }
+ }
}
else if (strcmp(output, hfst::internal_unknown.c_str()) == 0)
{
@@ -754,7 +798,7 @@ xfst_curly_label_to_transducer(const char* input, const char* output)
{
HfstTransducer tmp(*it, hfst::internal_epsilon, hfst::xre::format);
retval->concatenate(tmp, false);
- }
+ }
}
else
{
@@ -765,7 +809,7 @@ xfst_curly_label_to_transducer(const char* input, const char* output)
retval = new HfstTransducer(istr, ostr, tok, hfst::xre::format);
}
- retval->minimize();
+ retval->minimize(); // it should be safe to minimize
return retval;
}
@@ -808,19 +852,19 @@ xfst_label_to_transducer(const char* input, const char* output)
{
retval = new HfstTransducer(hfst::internal_unknown, hfst::internal_unknown, hfst::xre::format);
HfstTransducer id(hfst::internal_identity, hfst::internal_identity, hfst::xre::format);
- retval->disjunct(id).minimize();
+ retval->disjunct(id).minimize(); // it should be safe to minimize
}
else if (input_is_unknown)
{
retval = new HfstTransducer(hfst::internal_unknown, output, hfst::xre::format);
HfstTransducer output_tr(output, output, hfst::xre::format);
- retval->disjunct(output_tr).minimize();
+ retval->disjunct(output_tr).minimize(); // it should be safe to minimize
}
else if (output_is_unknown)
{
retval = new HfstTransducer(input, hfst::internal_unknown, hfst::xre::format);
HfstTransducer input_tr(input, input, hfst::xre::format);
- retval->disjunct(input_tr).minimize();
+ retval->disjunct(input_tr).minimize(); // it should be safe to minimize
}
else
{
@@ -834,12 +878,12 @@ xfst_label_to_transducer(const char* input, const char* output)
// marker = [0:M ?]*
HfstTransducer marker("@_EPSILON_SYMBOL_@", "M", t->get_type());
HfstTransducer id("@_IDENTITY_SYMBOL_@", t->get_type());
- marker.concatenate(id).repeat_star().minimize();
+ marker.concatenate(id).repeat_star().minimize(); // it should be safe to minimize
// the rule
HfstTransducer right_context(*t);
- right_context.insert_freely(StringPair("M", "@_EPSILON_SYMBOL_@")).minimize();
- right_context.insert_freely(StringPair("M", "M")).minimize();
+ right_context.insert_freely(StringPair("M", "@_EPSILON_SYMBOL_@")).optimize();
+ right_context.insert_freely(StringPair("M", "M")).optimize();
HfstTransducer left_context("@_EPSILON_SYMBOL_@", t->get_type());
HfstTransducerPair context(left_context, right_context);
@@ -868,7 +912,8 @@ xfst_label_to_transducer(const char* input, const char* output)
return new HfstTransducer(rule);
- marker.compose(rule).minimize();
+ marker.compose(rule);
+ marker.optimize();
return new HfstTransducer(marker);
}
@@ -876,9 +921,10 @@ xfst_label_to_transducer(const char* input, const char* output)
HfstTransducer * contains(const HfstTransducer * t)
{
HfstTransducer any(hfst::internal_identity, hfst::xre::format);
- any.repeat_star().minimize();
+ any.repeat_star().minimize(); // it should be safe to minimize
HfstTransducer * retval = new HfstTransducer(any);
- retval->concatenate(*t).concatenate(any).minimize();
+ retval->concatenate(*t).concatenate(any);
+ retval->optimize();
return retval;
}
@@ -910,14 +956,16 @@ xfst_label_to_transducer(const char* input, const char* output)
// noT = ?* - $[t]
// (strings that do not contain t)
HfstTransducer noT(hfst::internal_identity, t->get_type());
- noT.repeat_star().minimize();
+ noT.repeat_star().minimize(); // it should be safe to minimize
HfstTransducer * oneOrMoreT = contains(t);
- noT.subtract(*oneOrMoreT).minimize();
+ noT.subtract(*oneOrMoreT);
+ noT.optimize();
delete oneOrMoreT;
// return [weighted_rule - noT]
// (subtract strings that do not contain t from weighted rule)
- weighted_rule.subtract(noT).minimize();
+ weighted_rule.subtract(noT);
+ weighted_rule.optimize();
return new HfstTransducer(weighted_rule);
}
@@ -926,31 +974,37 @@ xfst_label_to_transducer(const char* input, const char* output)
{
// any_star = [?*]
HfstTransducer any_star(hfst::internal_identity, hfst::xre::format);
- any_star.repeat_star().minimize();
+ any_star.repeat_star().minimize(); // it should be safe to minimize
// any_plus = [?+]
HfstTransducer any_plus(hfst::internal_identity, hfst::xre::format);
- any_plus.repeat_plus().minimize();
+ any_plus.repeat_plus().minimize(); // it should be safe to minimize
// t1 = [?+ c ?*]
HfstTransducer * t1 = new HfstTransducer(any_plus);
- t1->concatenate(*c).minimize();
- t1->concatenate(any_star).minimize();
+ t1->concatenate(*c);
+ t1->optimize();
+ t1->concatenate(any_star);
+ t1->optimize();
// t2 = [c ?*]
HfstTransducer t2(*c);
- t2.concatenate(any_star).minimize();
+ t2.concatenate(any_star);
+ t2.optimize();
// t1 = [[?+ c ?*] & [c ?*]]
- t1->intersect(t2).minimize();
+ t1->intersect(t2);
// t3 = [[c ?+] & c]
HfstTransducer t3(*c);
- t3.concatenate(any_plus).minimize();
- t3.intersect(*c).minimize();
+ t3.concatenate(any_plus);
+ t3.optimize();
+ t3.intersect(*c);
+ t3.optimize();
// t1 = [t1 | t3]
- t1->disjunct(t3).minimize();
+ t1->disjunct(t3);
+ t1->optimize();
// cont_t1 = $[t1]
HfstTransducer * cont_t1 = contains(t1);
@@ -959,7 +1013,8 @@ xfst_label_to_transducer(const char* input, const char* output)
HfstTransducer * cont_c = contains(c);
// $[c] - $[t1]
- cont_c->subtract(*cont_t1).minimize();
+ cont_c->subtract(*cont_t1);
+ cont_c->optimize();
delete cont_t1;
return cont_c;
}
@@ -969,12 +1024,15 @@ xfst_label_to_transducer(const char* input, const char* output)
// neg_t = ~$[t]
HfstTransducer * cont_t = contains(t);
HfstTransducer neg_t(hfst::internal_identity, hfst::xre::format);
- neg_t.repeat_star().minimize();
- neg_t.subtract(*cont_t).minimize();
+ neg_t.repeat_star();
+ neg_t.optimize();
+ neg_t.subtract(*cont_t);
+ neg_t.optimize();
delete cont_t;
HfstTransducer * retval = contains_once(t);
- retval->disjunct(neg_t).minimize();
+ retval->disjunct(neg_t);
+ retval->optimize();
return retval;
}
@@ -983,7 +1041,7 @@ xfst_label_to_transducer(const char* input, const char* output)
// Merge operation creates an XreCompiler that needs this information below. Otherwise, it will overwrite all this.
struct XreConstructorArguments args(hfst::xre::definitions, hfst::xre::function_definitions, hfst::xre::function_arguments, hfst::xre::symbol_lists, hfst::xre::format);
- tr1->minimize();
+ tr1->optimize();
tr2->merge(*tr1, args);
return tr2;
}
@@ -1055,23 +1113,37 @@ void warn_about_special_symbols_in_replace(HfstTransducer * t)
if (!verbose_)
return;
- std::ostream * err = xreerrstr();
+ std::ostream * err = xreerrstr();
StringSet alphabet = t->get_alphabet();
- for (StringSet::const_iterator it = alphabet.begin();
+ for (StringSet::const_iterator it = alphabet.begin();
it != alphabet.end(); it++)
{
- if (HfstTransducer::is_special_symbol(*it) &&
+ if (HfstTransducer::is_special_symbol(*it) &&
*it != hfst::internal_epsilon &&
*it != hfst::internal_unknown &&
*it != hfst::internal_identity)
- {
+ {
*err << "warning: using special symbol '" << *it << "' in replace rule, use substitute instead" << std::endl;
}
}
xreflush(err);
}
+void check_multichar_symbol(const char * symbol)
+{
+ if (defined_multichar_symbols_ == NULL)
+ return;
+
+ if (defined_multichar_symbols_->find(std::string(symbol)) ==
+ defined_multichar_symbols_->end())
+ {
+ std::ostream * err = xreerrstr();
+ *err << "warning: multichar symbol '" << symbol << "' used but not defined" << std::endl;
+ xreflush(err);
+ }
+}
+
bool has_non_identity_pairs(const HfstTransducer * t)
{
hfst::implementations::HfstBasicTransducer basic(*t);
diff --git a/libhfst/src/parsers/xre_utils.h b/libhfst/src/parsers/xre_utils.h
index 058584b..aa45a50 100644
--- a/libhfst/src/parsers/xre_utils.h
+++ b/libhfst/src/parsers/xre_utils.h
@@ -1,10 +1,10 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
/**
@@ -18,7 +18,7 @@
#define GUARD_xre_utils_h
#include <map>
-#include "HfstTransducer.h"
+#include "HfstDataTypes.h"
namespace hfst { namespace xre {
@@ -55,6 +55,8 @@ char* strip_percents(const char *s);
char* strip_newline(char *s);
+ void count_lines(const char * s);
+
/**
* @brief add percents to string to form valid XRE symbol.
*/
@@ -65,13 +67,13 @@ char* add_percents(const char* s);
*/
char* get_quoted(const char *s);
-char* parse_quoted(const char *s);
+ char* parse_quoted(const char *s, unsigned int & length);
int* get_n_to_k(const char* s);
double get_weight(const char* s);
-/**
+/**
* @brief If \a s is of form "<foo>", free \a s and return an escaped version
* of it: "@_<foo>_@". Else, return \a s.
*/
@@ -103,7 +105,7 @@ HfstTransducer* compile_first(const std::string& xre,
hfst::ImplementationType type,
unsigned int & chars_read);
-/**
+/**
* @brief For a single-transition transducer, if the transition symbol is a name for
* transducer definition, expand the transition into the corresponding transducer.
*/
@@ -146,6 +148,8 @@ bool is_valid_function_call(const char * name, const std::vector<HfstTransducer>
/* Warn about \a symbol if it is of form "<.*>" or "all" and verbose mode is on. */
void warn_about_xfst_special_symbol(const char * symbol);
+ void check_multichar_symbol(const char * symbol);
+
bool has_non_identity_pairs(const HfstTransducer * t);
} } // namespaces
diff --git a/m4/ax_check_compile_flag.m4 b/m4/ax_check_compile_flag.m4
new file mode 100644
index 0000000..51df0c0
--- /dev/null
+++ b/m4/ax_check_compile_flag.m4
@@ -0,0 +1,74 @@
+# ===========================================================================
+# http://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+# AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS], [INPUT])
+#
+# DESCRIPTION
+#
+# Check whether the given FLAG works with the current language's compiler
+# or gives an error. (Warnings, however, are ignored)
+#
+# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on
+# success/failure.
+#
+# If EXTRA-FLAGS is defined, it is added to the current language's default
+# flags (e.g. CFLAGS) when the check is done. The check is thus made with
+# the flags: "CFLAGS EXTRA-FLAGS FLAG". This can for example be used to
+# force the compiler to issue an error when a bad flag is given.
+#
+# INPUT gives an alternative input source to AC_COMPILE_IFELSE.
+#
+# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this
+# macro in sync with AX_CHECK_{PREPROC,LINK}_FLAG.
+#
+# LICENSE
+#
+# Copyright (c) 2008 Guido U. Draheim <guidod at gmx.de>
+# Copyright (c) 2011 Maarten Bosmans <mkbosmans at gmail.com>
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# As a special exception, the respective Autoconf Macro's copyright owner
+# gives unlimited permission to copy, distribute and modify the configure
+# scripts that are the output of Autoconf when processing the Macro. You
+# need not follow the terms of the GNU General Public License when using
+# or distributing such scripts, even though portions of the text of the
+# Macro appear in them. The GNU General Public License (GPL) does govern
+# all other use of the material that constitutes the Autoconf Macro.
+#
+# This special exception to the GPL applies to versions of the Autoconf
+# Macro released by the Autoconf Archive. When you make and distribute a
+# modified version of the Autoconf Macro, you may extend this special
+# exception to the GPL to apply to your modified version as well.
+
+#serial 3
+
+AC_DEFUN([AX_CHECK_COMPILE_FLAG],
+[AC_PREREQ(2.59)dnl for _AC_LANG_PREFIX
+AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl
+AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [
+ ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS
+ _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1"
+ AC_COMPILE_IFELSE([m4_default([$5],[AC_LANG_PROGRAM()])],
+ [AS_VAR_SET(CACHEVAR,[yes])],
+ [AS_VAR_SET(CACHEVAR,[no])])
+ _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags])
+AS_IF([test x"AS_VAR_GET(CACHEVAR)" = xyes],
+ [m4_default([$2], :)],
+ [m4_default([$3], :)])
+AS_VAR_POPDEF([CACHEVAR])dnl
+])dnl AX_CHECK_COMPILE_FLAGS
diff --git a/man/Makefile.am b/man/Makefile.am
index ca043ad..a584a6d 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -15,7 +15,6 @@
## You should have received a copy of the GNU General Public License
## along with this program. If not, see <http://www.gnu.org/licenses/>.
-
hfst_tools=hfst-compare.1 hfst-compose.1 \
hfst-concatenate.1 hfst-determinize.1 \
hfst-disjunct.1 \
@@ -30,48 +29,72 @@ hfst_tools=hfst-compare.1 hfst-compose.1 \
hfst-calculate.1 hfst-compose-intersect.1 \
hfst-lookup.1 \
hfst-substitute.1 hfst-format.1 hfst-name.1 hfst-lexc.1 \
- hfst-lexc-wrapper.1 \
hfst-affix-guessify.1 hfst-edit-metadata.1 hfst-expand-equivalences.1 \
hfst-grep.1 hfst-guess.1 hfst-guessify.1 hfst-info.1 hfst-multiply.1 \
hfst-pair-test.1 hfst-prune-alphabet.1 hfst-reweight.1 hfst-shuffle.1 \
hfst-traverse.1 hfst-tokenize.1 hfst-pmatch.1 hfst-pmatch2fst.1 \
hfst-optimized-lookup.1
-hfst_xfst=hfst-xfst.1
-hfst_apertium_proc=hfst-apertium-proc.1
-hfst_foma=hfst-foma.1 # hand-written
-hfst_foma_wrapper=hfst-foma-wrapper.1 # hand-written
+hfst_xfst=hfst-xfst.1
+hfst_apertium_proc=hfst-apertium-proc.1
+
+if GENERATE_LEXC_WRAPPER
+ hfst_lexc_wrapper=hfst-lexc-wrapper.1
+else
+ hfst_lexc_wrapper=
+endif
+
+if GENERATE_FOMA_WRAPPER
+ hfst_foma=hfst-foma.1 # hand-written
+ hfst_foma_wrapper=hfst-foma-wrapper.1 # hand-written
+else
+ hfst_foma=
+ hfst_foma_wrapper=
+endif
+
hfst_tag_and_reweight=hfst-tag.1 hfst-reweight-tagger.1
-hfst_train_tagger=hfst-train-tagger.1
-hfst_train_tagger_subprograms=hfst-open-input-file-for-tagger.1 \
+
+if WANT_TRAIN_TAGGER
+ hfst_train_tagger=hfst-train-tagger.1
+ hfst_train_tagger_subprograms=hfst-open-input-file-for-tagger.1 \
hfst_tagger_compute_data_statistics.py.1 hfst-build-tagger.1 # hand-written
+else
+ hfst_train_tagger=
+ hfst_train_tagger_subprograms=
+endif
+
hfst_twolc=hfst-twolc.1
-hfst_twolc_subprograms= htwolcpre1.1 htwolcpre2.1 htwolcpre3.1 # hand-written
+
+if WANT_TWOLC_SCRIPT
+ hfst_twolc_subprograms= htwolcpre1.1 htwolcpre2.1 htwolcpre3.1 hfst-twolc-script.1 # hand-written
+else
+ hfst_twolc_subprograms=
+endif
aliases= hfst-determinise.1 hfst-expand.1 hfst-intersect.1 hfst-minimise.1 \
hfst-minus.1 hfst-sfstpl2fst.1 hfst-summarise.1 hfst-union.1 hfst-proc.1 \
- hfst-optimised-lookup.1 hfst-train-tagger-loc.1 hfst-train-tagger-system.1 \
- hfst-twolc-loc.1 hfst-twolc-system.1 hfst-proc2.1 hfst-tokenise.1
+ hfst-optimised-lookup.1 \
+ hfst-proc2.1 hfst-tokenise.1
dist_man1_MANS=$(hfst_tools) $(hfst_xfst) $(hfst_apertium_proc) $(hfst_foma) $(hfst_foma_wrapper) \
$(hfst_tag_and_reweight) $(hfst_train_tagger) $(hfst_train_tagger_subprograms) $(hfst_twolc) \
- $(hfst_twolc_subprograms) $(aliases)
+ $(hfst_twolc_subprograms) $(hfst_lexc_wrapper) $(aliases)
LN_S=ln -s -f
hfst-determinise.1:
- $(LN_S) hfst-determinize.1 hfst-determinise.1
+ $(LN_S) hfst-determinize.1 hfst-determinise.1
hfst-expand.1:
- $(LN_S) hfst-fst2strings.1 hfst-expand.1
+ $(LN_S) hfst-fst2strings.1 hfst-expand.1
hfst-intersect.1:
- $(LN_S) hfst-conjunct.1 hfst-intersect.1
+ $(LN_S) hfst-conjunct.1 hfst-intersect.1
hfst-minimise.1:
- $(LN_S) hfst-minimize.1 hfst-minimise.1
+ $(LN_S) hfst-minimize.1 hfst-minimise.1
hfst-minus.1:
- $(LN_S) hfst-subtract.1 hfst-minus.1
+ $(LN_S) hfst-subtract.1 hfst-minus.1
hfst-optimised-lookup.1:
$(LN_S) hfst-optimized-lookup.1 hfst-optimised-lookup.1
@@ -80,22 +103,10 @@ hfst-proc.1:
$(LN_S) hfst-apertium-proc.1 hfst-proc.1
hfst-sfstpl2fst.1:
- $(LN_S) hfst-calculate.1 hfst-sfstpl2fst.1
+ $(LN_S) hfst-calculate.1 hfst-sfstpl2fst.1
-hfst-summarise.1:
- $(LN_S) hfst-summarize.1 hfst-summarise.1
-
-hfst-train-tagger-loc.1:
- $(LN_S) hfst-train-tagger.1 hfst-train-tagger-loc.1
-
-hfst-train-tagger-system.1:
- $(LN_S) hfst-train-tagger.1 hfst-train-tagger-system.1
-
-hfst-twolc-loc.1:
- $(LN_S) hfst-twolc.1 hfst-twolc-loc.1
-
-hfst-twolc-system.1:
- $(LN_S) hfst-twolc.1 hfst-twolc-system.1
+hfst-summarise.1:
+ $(LN_S) hfst-summarize.1 hfst-summarise.1
hfst-union.1:
$(LN_S) hfst-disjunct.1 hfst-union.1
@@ -112,7 +123,7 @@ man:
make bootstrap && make aliases
# Generate man pages from help messages
-bootstrap:
+bootstrap:
for f in $(hfst_tools); do \
N=`echo $$f | sed 's/.1/ \-\-help/' | sed 's/hfst/..\/tools\/src\/hfst/' | sh | sed -n 2p`; \
help2man \
@@ -155,5 +166,5 @@ bootstrap:
aliases:
for f in $(aliases) ; do \
make $$f; \
- done;
+ done;
diff --git a/man/hfst-affix-guessify.1 b/man/hfst-affix-guessify.1
index 88cbbe3..6ef5c47 100644
--- a/man/hfst-affix-guessify.1
+++ b/man/hfst-affix-guessify.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-AFFIX-GUESSIFY "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-AFFIX-GUESSIFY "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-affix-guessify \- =Create weighted affix guesser from automaton
.SH SYNOPSIS
.B hfst-affix-guessify
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Create weighted affix guesser from automaton
.SS "Common options:"
@@ -25,17 +25,17 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Guesser parameters:"
.TP
-\fB\-D\fR, \fB\-\-direction\fR=\fIDIR\fR
+\fB\-D\fR, \fB\-\-direction\fR=\fI\,DIR\/\fR
set direction of guessing
.TP
-\fB\-w\fR, \fB\-\-weight\fR=\fIWEIGHT\fR
+\fB\-w\fR, \fB\-\-weight\fR=\fI\,WEIGHT\/\fR
set weight difference of affix lengths
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
@@ -44,7 +44,7 @@ DIR is either suffix or prefix, or suffix if omitted.
WEIGHT is a weight of each arc not in the known suffix or prefix being guessed, as parsed with strtod(3), or 1.0 if omitted.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-affix\-guessify home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstAffixGuessify>
@@ -52,7 +52,7 @@ hfst\-affix\-guessify home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-apertium-proc.1 b/man/hfst-apertium-proc.1
index 0485e0b..686070e 100644
--- a/man/hfst-apertium-proc.1
+++ b/man/hfst-apertium-proc.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-APERTIUM-PROC "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-APERTIUM-PROC "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-apertium-proc \- =Usage: hfst-proc [-a [-p|-C|-x] [-k]|-g|-n|-d|-t] [-W] [-n N] [-c|-w] [-z] [-v|-q|]
.SH SYNOPSIS
.B hfst-proc
-[\fI-a \fR[\fI-p|-C|-x\fR] [\fI-k\fR]\fI|-g|-n|-d|-t\fR] [\fI-W\fR] [\fI-n N\fR] [\fI-c|-w\fR] [\fI-z\fR] [\fI-v|-q|\fR]
+[\fI\,-a \/\fR[\fI\,-p|-C|-x\/\fR] [\fI\,-k\/\fR]\fI\,|-g|-n|-d|-t\/\fR] [\fI\,-W\/\fR] [\fI\,-n N\/\fR] [\fI\,-c|-w\/\fR] [\fI\,-z\/\fR] [\fI\,-v|-q|\/\fR]
.SH DESCRIPTION
.IP
transducer_file [input_file [output_file]]
@@ -49,7 +49,7 @@ Print final analysis weights (if any)
\fB\-r\fR, \fB\-\-show\-raw\-in\-cg\fR
Print the raw analysis string as sub\-reading in CG output
.TP
-\fB\-N\fR N, \fB\-\-analyses\fR=\fIN\fR
+\fB\-N\fR N, \fB\-\-analyses\fR=\fI\,N\/\fR
Output no more than N analyses
(if the transducer is weighted, the N best analyses)
.TP
@@ -86,7 +86,7 @@ case, ``superblanks'' or anything else!!!
.SH "REPORTING BUGS"
Report bugs to hfst\-bugs at helsinki.fi
.PP
-hfst\-proc 0.0 (hfst 3.9.0)
+hfst\-proc 0.0 (hfst 3.12.2)
.br
-Dec 14 2015 10:44:22
+Mar 23 2017 15:03:05
copyright (C) 2009\-2011 University of Helsinki
diff --git a/man/hfst-calculate.1 b/man/hfst-calculate.1
index 3fe7975..cae1549 100644
--- a/man/hfst-calculate.1
+++ b/man/hfst-calculate.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-SFSTPL2FST "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-SFSTPL2FST "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-sfstpl2fst \- =Compile a file written with SFST programming language into a transducer.
.SH SYNOPSIS
.B hfst-sfstpl2fst
-\fI(alias hfst-calculate) \fR[\fIOPTIONS\fR...] [\fIINFILE\fR]
+\fI\,(alias hfst-calculate) \/\fR[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Compile a file written with SFST programming language into a transducer.
.SS "Common options:"
@@ -25,41 +25,41 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "String and format options:"
.TP
-\fB\-f\fR, \fB\-\-format\fR=\fIFMT\fR
+\fB\-f\fR, \fB\-\-format\fR=\fI\,FMT\/\fR
Write result in FMT format
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
FMT must be one of the following: {foma, sfst, openfst\-tropical, openfst\-log}
.SH EXAMPLES
.IP
-echo "[a\-z]*" | hfst\-sfstpl2fst \fB\-f\fR foma > az.foma
+echo "[a\-z]*" | hfst\-sfstpl2fst \-f foma > az.foma
.IP
create a foma transducer accepting any number of consecutive
characters between a and z, inclusive
.IP
-echo "a:b (a:<>)+" > ab.sfst\-pl ; hfst\-sfstpl2fst \fB\-f\fR openfst\-tropical \e
+echo "a:b (a:<>)+" > ab.sfst\-pl ; hfst\-sfstpl2fst \-f openfst\-tropical \e
.HP
-\fB\-i\fR ab.sfst\-pl \fB\-o\fR ab.tropical
+\-i ab.sfst\-pl \-o ab.tropical
.IP
create a tropical OpenFst transducer that accepts two or more
consecutive 'a's and maps them into string "b"
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
hfst\-sfstpl2fst home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstSfstPl2Fst>
.br
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-compare.1 b/man/hfst-compare.1
index d65bd6a..38f5fa5 100644
--- a/man/hfst-compare.1
+++ b/man/hfst-compare.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-COMPARE "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-COMPARE "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-compare \- =Compare two transducers
.SH SYNOPSIS
.B hfst-compare
-[\fIOPTIONS\fR...] [\fIINFILE1 \fR[\fIINFILE2\fR]]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE1 \/\fR[\fI\,INFILE2\/\fR]]
.SH DESCRIPTION
Compare two transducers
.SS "Common options:"
@@ -25,16 +25,16 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-1\fR, \fB\-\-input1\fR=\fIINFILE1\fR
+\fB\-1\fR, \fB\-\-input1\fR=\fI\,INFILE1\/\fR
Read first input transducer from INFILE1
.TP
-\fB\-2\fR, \fB\-\-input2\fR=\fIINFILE2\fR
+\fB\-2\fR, \fB\-\-input2\fR=\fI\,INFILE2\/\fR
Read second input transducer from INFILE2
.TP
\fB\-C\fR, \fB\-\-do\-not\-convert\fR
Do not allow transducers to be converted into the same type
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write results to OUTFILE
.SS "Harmonization:"
.HP
@@ -62,7 +62,7 @@ $ hfst\-compare cat.hfst cat.hfst
cat.hfst[1] == cat.hfst[1]
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-compare home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstCompare>
@@ -70,7 +70,7 @@ hfst\-compare home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-compose-intersect.1 b/man/hfst-compose-intersect.1
index 6c020fc..d000a43 100644
--- a/man/hfst-compose-intersect.1
+++ b/man/hfst-compose-intersect.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-COMPOSE-INTERSECT "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-COMPOSE-INTERSECT "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-compose-intersect \- =Compose a lexicon with one or more rule transducers.
.SH SYNOPSIS
.B hfst-compose-intersect
-[\fIOPTIONS\fR...] [\fIINFILE1 \fR[\fIINFILE2\fR]]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE1 \/\fR[\fI\,INFILE2\/\fR]]
.SH DESCRIPTION
Compose a lexicon with one or more rule transducers.
.SS "Common options:"
@@ -25,16 +25,16 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-1\fR, \fB\-\-input1\fR=\fIINFILE1\fR
+\fB\-1\fR, \fB\-\-input1\fR=\fI\,INFILE1\/\fR
Read first input transducer from INFILE1
.TP
-\fB\-2\fR, \fB\-\-input2\fR=\fIINFILE2\fR
+\fB\-2\fR, \fB\-\-input2\fR=\fI\,INFILE2\/\fR
Read second input transducer from INFILE2
.TP
\fB\-C\fR, \fB\-\-do\-not\-convert\fR
Do not allow transducers to be converted into the same type
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write results to OUTFILE
.SS "Composition options:"
.TP
@@ -63,12 +63,12 @@ INFILE1 (the lexicon) must contain exactly one transducer.
INFILE2 (rule file) may contain several transducers.
.SH EXAMPLES
.IP
-hfst\-compose\-intersect \fB\-o\fR analyzer.hfst lexicon.hfst rules.hfst
+hfst\-compose\-intersect \-o analyzer.hfst lexicon.hfst rules.hfst
.PP
compose rules with lexicon
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-compose\-intersect home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstComposeIntersect>
@@ -76,7 +76,7 @@ hfst\-compose\-intersect home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-compose.1 b/man/hfst-compose.1
index 79e7d25..31a43b5 100644
--- a/man/hfst-compose.1
+++ b/man/hfst-compose.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-COMPOSE "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-COMPOSE "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-compose \- =Compose two transducers
.SH SYNOPSIS
.B hfst-compose
-[\fIOPTIONS\fR...] [\fIINFILE1 \fR[\fIINFILE2\fR]]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE1 \/\fR[\fI\,INFILE2\/\fR]]
.SH DESCRIPTION
Compose two transducers
.SS "Common options:"
@@ -25,23 +25,23 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-1\fR, \fB\-\-input1\fR=\fIINFILE1\fR
+\fB\-1\fR, \fB\-\-input1\fR=\fI\,INFILE1\/\fR
Read first input transducer from INFILE1
.TP
-\fB\-2\fR, \fB\-\-input2\fR=\fIINFILE2\fR
+\fB\-2\fR, \fB\-\-input2\fR=\fI\,INFILE2\/\fR
Read second input transducer from INFILE2
.TP
\fB\-C\fR, \fB\-\-do\-not\-convert\fR
Do not allow transducers to be converted into the same type
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write results to OUTFILE
.SS "Composition options:"
.TP
-\fB\-x\fR, \fB\-\-xerox\-composition\fR=\fIVALUE\fR Whether flag diacritics are treated as ordinary
+\fB\-x\fR, \fB\-\-xerox\-composition\fR=\fI\,VALUE\/\fR Whether flag diacritics are treated as ordinary
symbols in composition (default is false).
.TP
-\fB\-X\fR, \fB\-\-xfst\fR=\fIVARIABLE\fR
+\fB\-X\fR, \fB\-\-xfst\fR=\fI\,VARIABLE\/\fR
Toggle xfst compatibility option VARIABLE.
.SS "Harmonization:"
.HP
@@ -66,11 +66,11 @@ VALUE can be one of the following: [true|false], [yes|no] or [ON|OFF],
false being the default.
.SH EXAMPLES
.TP
-hfst\-compose \fB\-o\fR cat2dog.hfst cat2mouse.hfst mouse2dog.hfst
+hfst\-compose \-o cat2dog.hfst cat2mouse.hfst mouse2dog.hfst
composes two automata
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-compose home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstCompose>
@@ -78,7 +78,7 @@ hfst\-compose home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-concatenate.1 b/man/hfst-concatenate.1
index 0879986..693225b 100644
--- a/man/hfst-concatenate.1
+++ b/man/hfst-concatenate.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-CONCATENATE "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-CONCATENATE "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-concatenate \- =Concatenate two transducers
.SH SYNOPSIS
.B hfst-concatenate
-[\fIOPTIONS\fR...] [\fIINFILE1 \fR[\fIINFILE2\fR]]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE1 \/\fR[\fI\,INFILE2\/\fR]]
.SH DESCRIPTION
Concatenate two transducers
.SS "Common options:"
@@ -25,16 +25,16 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-1\fR, \fB\-\-input1\fR=\fIINFILE1\fR
+\fB\-1\fR, \fB\-\-input1\fR=\fI\,INFILE1\/\fR
Read first input transducer from INFILE1
.TP
-\fB\-2\fR, \fB\-\-input2\fR=\fIINFILE2\fR
+\fB\-2\fR, \fB\-\-input2\fR=\fI\,INFILE2\/\fR
Read second input transducer from INFILE2
.TP
\fB\-C\fR, \fB\-\-do\-not\-convert\fR
Do not allow transducers to be converted into the same type
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write results to OUTFILE
.SS "Harmonization:"
.HP
@@ -55,12 +55,12 @@ If INFILE2 has only one transducer, the operation is applied for
each transducer in INFILE1 keeping the second transducer constant.
.SH EXAMPLES
.IP
-hfst\-concatenate \fB\-o\fR catdog.hfst cat.hfst dog.hfst
+hfst\-concatenate \-o catdog.hfst cat.hfst dog.hfst
.PP
concatenates cat.hfst with dog.hfst and writes results to catdog.hfst
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-concatenate home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstConcatenate>
@@ -68,7 +68,7 @@ hfst\-concatenate home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-conjunct.1 b/man/hfst-conjunct.1
index 8f89f2b..21d2776 100644
--- a/man/hfst-conjunct.1
+++ b/man/hfst-conjunct.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-CONJUNCT "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-CONJUNCT "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-conjunct \- =Conjunct (intersect, AND) two transducers
.SH SYNOPSIS
.B hfst-conjunct
-[\fIOPTIONS\fR...] [\fIINFILE1 \fR[\fIINFILE2\fR]]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE1 \/\fR[\fI\,INFILE2\/\fR]]
.SH DESCRIPTION
Conjunct (intersect, AND) two transducers
.SS "Common options:"
@@ -25,16 +25,16 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-1\fR, \fB\-\-input1\fR=\fIINFILE1\fR
+\fB\-1\fR, \fB\-\-input1\fR=\fI\,INFILE1\/\fR
Read first input transducer from INFILE1
.TP
-\fB\-2\fR, \fB\-\-input2\fR=\fIINFILE2\fR
+\fB\-2\fR, \fB\-\-input2\fR=\fI\,INFILE2\/\fR
Read second input transducer from INFILE2
.TP
\fB\-C\fR, \fB\-\-do\-not\-convert\fR
Do not allow transducers to be converted into the same type
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write results to OUTFILE
.SS "Flag diacritics:"
.TP
@@ -55,10 +55,10 @@ If INFILE2 has only one transducer, the operation is applied for
each transducer in INFILE1 keeping the second transducer constant.
.SH EXAMPLES
.IP
-hfst\-conjunct \fB\-o\fR dog.hfst cat_or_dog.hfst dog_or_mouse.hfst
+hfst\-conjunct \-o dog.hfst cat_or_dog.hfst dog_or_mouse.hfst
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-conjunct home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstConjunct>
@@ -66,7 +66,7 @@ hfst\-conjunct home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-determinize.1 b/man/hfst-determinize.1
index 823df8f..d58c2bd 100644
--- a/man/hfst-determinize.1
+++ b/man/hfst-determinize.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-DETERMINIZE "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-DETERMINIZE "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-determinize \- =Determinize a transducer
.SH SYNOPSIS
.B hfst-determinize
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Determinize a transducer
.SS "Common options:"
@@ -25,10 +25,10 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Command-specific options:"
.TP
@@ -40,7 +40,7 @@ If OUTFILE or INFILE is missing or \-, standard streams will be used.
Format of result depends on format of INFILE
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-determinize home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstDeterminize>
@@ -48,7 +48,7 @@ hfst\-determinize home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-disjunct.1 b/man/hfst-disjunct.1
index 423459e..32d646a 100644
--- a/man/hfst-disjunct.1
+++ b/man/hfst-disjunct.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-DISJUNCT "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-DISJUNCT "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-disjunct \- =Disjunct (union, OR) two transducers
.SH SYNOPSIS
.B hfst-disjunct
-[\fIOPTIONS\fR...] [\fIINFILE1 \fR[\fIINFILE2\fR]]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE1 \/\fR[\fI\,INFILE2\/\fR]]
.SH DESCRIPTION
Disjunct (union, OR) two transducers
.SS "Common options:"
@@ -25,16 +25,16 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-1\fR, \fB\-\-input1\fR=\fIINFILE1\fR
+\fB\-1\fR, \fB\-\-input1\fR=\fI\,INFILE1\/\fR
Read first input transducer from INFILE1
.TP
-\fB\-2\fR, \fB\-\-input2\fR=\fIINFILE2\fR
+\fB\-2\fR, \fB\-\-input2\fR=\fI\,INFILE2\/\fR
Read second input transducer from INFILE2
.TP
\fB\-C\fR, \fB\-\-do\-not\-convert\fR
Do not allow transducers to be converted into the same type
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write results to OUTFILE
.PP
If OUTFILE, or either INFILE1 or INFILE2 is missing or \-,
@@ -55,10 +55,10 @@ Harmonization:
Harmonize flag diacritics.
.SH EXAMPLES
.IP
-hfst\-disjunct \fB\-o\fR cat_or_dog.hfst cat.hfst dog.hfst
+hfst\-disjunct \-o cat_or_dog.hfst cat.hfst dog.hfst
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-disjunct home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstDisjunct>
@@ -66,7 +66,7 @@ hfst\-disjunct home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-edit-metadata.1 b/man/hfst-edit-metadata.1
index 43ecc4a..85d86d2 100644
--- a/man/hfst-edit-metadata.1
+++ b/man/hfst-edit-metadata.1
@@ -1,21 +1,21 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-EDIT-METADATA "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-EDIT-METADATA "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-edit-metadata \- =Name a transducer
.SH SYNOPSIS
.B hfst-edit-metadata
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Name a transducer
.SS "Name options:"
.TP
-\fB\-a\fR, \fB\-\-add\fR=\fIANAME=VALUE\fR
+\fB\-a\fR, \fB\-\-add\fR=\fI\,ANAME=VALUE\/\fR
add or replace property ANAMEwith VALUE
.TP
-\fB\-p\fR, \fB\-\-print\fR[=\fINAME\fR]
+\fB\-p\fR, \fB\-\-print\fR[=\fI\,NAME\/\fR]
print the current PNAME
.TP
-\fB\-t\fR, \fB\-\-truncate_length\fR=\fILEN\fR
+\fB\-t\fR, \fB\-\-truncate_length\fR=\fI\,LEN\/\fR
truncate added properties' lengths to LEN
.SS "Common options:"
.TP
@@ -35,10 +35,10 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
@@ -46,7 +46,7 @@ Format of result depends on format of INFILE
If PNAME is omitted, all values are printed
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-edit\-metadata home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstEditMetadata>
@@ -54,7 +54,7 @@ hfst\-edit\-metadata home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-expand-equivalences.1 b/man/hfst-expand-equivalences.1
index df118c1..6b9471e 100644
--- a/man/hfst-expand-equivalences.1
+++ b/man/hfst-expand-equivalences.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-EXPAND-EQUIVALENCES "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-EXPAND-EQUIVALENCES "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-expand-equivalences \- =Extend transducer arcs for equivalence classes
.SH SYNOPSIS
.B hfst-expand-equivalences
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Extend transducer arcs for equivalence classes
.SS "Common options:"
@@ -25,19 +25,19 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Eqv. class extension options:"
.TP
-\fB\-f\fR, \fB\-\-from\fR=\fIISYM\fR
+\fB\-f\fR, \fB\-\-from\fR=\fI\,ISYM\/\fR
convert single symbol ISYM to allow OSYM
.TP
-\fB\-t\fR, \fB\-\-to\fR=\fIOSYM\fR
+\fB\-t\fR, \fB\-\-to\fR=\fI\,OSYM\/\fR
convert to OSYM
.TP
-\fB\-a\fR, \fB\-\-acx\fR=\fIACXFILE\fR
+\fB\-a\fR, \fB\-\-acx\fR=\fI\,ACXFILE\/\fR
read extensions in acx format from ACXFILE
.TP
-\fB\-T\fR, \fB\-\-tsv\fR=\fITSVFILE\fR
+\fB\-T\fR, \fB\-\-tsv\fR=\fI\,TSVFILE\/\fR
read extensions in tsv format from TSVFILE
.TP
-\fB\-l\fR, \fB\-\-level\fR=\fILEVEL\fR
+\fB\-l\fR, \fB\-\-level\fR=\fI\,LEVEL\/\fR
perform extensions on LEVEL of fsa
.PP
Either ACXFILE, TSVFILE or both ISYM and OSYM must be specified.
@@ -49,14 +49,14 @@ hfst\-expand\-equivalences \fB\-o\fR rox.hfst \fB\-a\fR romanian.acx ro.hfst
extend romanian charequivalences
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
hfst\-expand\-equivalences home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstExpandEquivalences>
.br
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-format.1 b/man/hfst-format.1
index 7877110..647da5c 100644
--- a/man/hfst-format.1
+++ b/man/hfst-format.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-FORMAT "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-FORMAT "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-format \- =determine HFST transducer format
.SH SYNOPSIS
.B hfst-format
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
determine HFST transducer format
.SS "Common options:"
@@ -25,10 +25,10 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Tool-specific options:"
.TP
@@ -44,7 +44,7 @@ If OUTFILE or INFILE is missing or \-, standard streams will be used.
Format of result depends on format of INFILE
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-format home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstFormat>
@@ -54,7 +54,7 @@ General help using HFST software:
.br
Transducers in (null) are of type SFST (1.4 compatible)
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-fst2fst.1 b/man/hfst-fst2fst.1
index 5375354..ca2a28d 100644
--- a/man/hfst-fst2fst.1
+++ b/man/hfst-fst2fst.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-FST2FST "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-FST2FST "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-fst2fst \- =Convert transducers between binary formats
.SH SYNOPSIS
.B hfst-fst2fst
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Convert transducers between binary formats
.SS "Common options:"
@@ -25,14 +25,14 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Conversion options:"
.TP
-\fB\-f\fR, \fB\-\-format\fR=\fIFMT\fR
+\fB\-f\fR, \fB\-\-format\fR=\fI\,FMT\/\fR
Write result in FMT format
.TP
\fB\-b\fR, \fB\-\-use\-backend\-format\fR
@@ -71,7 +71,7 @@ optimized\-lookup\-weighted, optimized\-lookup\-unweighted }.
Note that xfsm format is always written in native format without HFST wrappers.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-fst2fst home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstFst2Fst>
@@ -79,7 +79,7 @@ hfst\-fst2fst home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-fst2strings.1 b/man/hfst-fst2strings.1
index 889e2ca..20cfe31 100644
--- a/man/hfst-fst2strings.1
+++ b/man/hfst-fst2strings.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-FST2STRINGS "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-FST2STRINGS "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-fst2strings \- =Display the strings recognized by a transducer
.SH SYNOPSIS
.B hfst-fst2strings
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Display the strings recognized by a transducer
.SS "Common options:"
@@ -25,16 +25,16 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Fst2strings options:"
.TP
-\fB\-n\fR, \fB\-\-max\-strings\fR=\fINSTR\fR
+\fB\-n\fR, \fB\-\-max\-strings\fR=\fI\,NSTR\/\fR
print at most NSTR strings
.TP
-\fB\-N\fR, \fB\-\-nbest\fR=\fINBEST\fR
+\fB\-N\fR, \fB\-\-nbest\fR=\fI\,NBEST\/\fR
print at most NBEST best strings
.TP
-\fB\-r\fR, \fB\-\-random\fR=\fINRAND\fR
+\fB\-r\fR, \fB\-\-random\fR=\fI\,NRAND\/\fR
print at most NRAND random strings
.TP
-\fB\-c\fR, \fB\-\-cycles\fR=\fINCYC\fR
+\fB\-c\fR, \fB\-\-cycles\fR=\fI\,NCYC\/\fR
follow cycles at most NCYC times
.TP
\fB\-w\fR, \fB\-\-print\-weights\fR
@@ -43,33 +43,33 @@ display the weight for each string
\fB\-S\fR, \fB\-\-print\-separator\fR
print separator "\-\-" after each transducer
.TP
-\fB\-e\fR, \fB\-\-epsilon\-format\fR=\fIEPS\fR
+\fB\-e\fR, \fB\-\-epsilon\-format\fR=\fI\,EPS\/\fR
print epsilon as EPS
.TP
-\fB\-X\fR, \fB\-\-xfst\fR=\fIVARIABLE\fR
+\fB\-X\fR, \fB\-\-xfst\fR=\fI\,VARIABLE\/\fR
toggle xfst compatibility option VARIABLE
.SS "Ignore paths if:"
.TP
-\fB\-b\fR, \fB\-\-beam\fR=\fIB\fR
+\fB\-b\fR, \fB\-\-beam\fR=\fI\,B\/\fR
output string weight not within B from the weight
of the best output string
.TP
-\fB\-l\fR, \fB\-\-max\-in\-length\fR=\fIMIL\fR
+\fB\-l\fR, \fB\-\-max\-in\-length\fR=\fI\,MIL\/\fR
input string longer than MIL
.TP
-\fB\-L\fR, \fB\-\-max\-out\-length\fR=\fIMOL\fR
+\fB\-L\fR, \fB\-\-max\-out\-length\fR=\fI\,MOL\/\fR
output string longer than MOL
.TP
-\fB\-p\fR, \fB\-\-in\-prefix\fR=\fIOPREFIX\fR
+\fB\-p\fR, \fB\-\-in\-prefix\fR=\fI\,OPREFIX\/\fR
input string not beginning with IPREFIX
.TP
-\fB\-P\fR, \fB\-\-out\-prefix\fR=\fIOPREFIX\fR
+\fB\-P\fR, \fB\-\-out\-prefix\fR=\fI\,OPREFIX\/\fR
output string not beginning with OPREFIX
.TP
-\fB\-u\fR, \fB\-\-in\-exclude\fR=\fIIXSTR\fR
+\fB\-u\fR, \fB\-\-in\-exclude\fR=\fI\,IXSTR\/\fR
input string containing IXSTR
.TP
-\fB\-U\fR, \fB\-\-out\-exclude\fR=\fIOXST\fR
+\fB\-U\fR, \fB\-\-out\-exclude\fR=\fI\,OXST\/\fR
output string containing OXSTR
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
@@ -92,7 +92,7 @@ generates all forms of lexical.hfst
Does not work correctly for hfst optimized lookup format.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-fst2strings home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstFst2Strings>
@@ -100,7 +100,7 @@ hfst\-fst2strings home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-fst2txt.1 b/man/hfst-fst2txt.1
index 4fcc16a..bc51fb1 100644
--- a/man/hfst-fst2txt.1
+++ b/man/hfst-fst2txt.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-FST2TXT "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-FST2TXT "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-fst2txt \- =Print transducer in AT&T, dot, prolog or pckimmo format
.SH SYNOPSIS
.B hfst-fst2txt
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Print transducer in AT&T, dot, prolog or pckimmo format
.SS "Common options:"
@@ -25,10 +25,10 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Text format options:"
.TP
@@ -38,7 +38,7 @@ If weights are printed in all cases
\fB\-D\fR, \fB\-\-do\-not\-print\-weights\fR
If weights are not printed in any case
.TP
-\fB\-f\fR, \fB\-\-format\fR=\fITFMT\fR
+\fB\-f\fR, \fB\-\-format\fR=\fI\,TFMT\/\fR
Print output in TFMT format [default=att]
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
@@ -47,7 +47,7 @@ if and only if the transducer is in weighted format.
TFMT is one of {att, dot, prolog, pckimmo}.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-fst2txt home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstFst2Txt>
@@ -55,7 +55,7 @@ hfst\-fst2txt home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-grep.1 b/man/hfst-grep.1
index 237accc..b5abef0 100644
--- a/man/hfst-grep.1
+++ b/man/hfst-grep.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-GREP "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-GREP "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-grep \- =Search for PATTERN in each FILE or standard input.
.SH SYNOPSIS
.B hfst-grep
-[\fIOPTIONS\fR...] \fIPATTERN \fR[\fIFILE\fR...]
+[\fI\,OPTIONS\/\fR...] \fI\,PATTERN \/\fR[\fI\,FILE\/\fR...]
.SH DESCRIPTION
Search for PATTERN in each FILE or standard input.
Pattern is, by default, a Xerox regular expression (XRE).
@@ -26,7 +26,7 @@ Only print fatal erros and requested output
\fB\-s\fR, \fB\-\-silent\fR
Alias of \fB\-\-quiet\fR
.TP
-\fB\-9\fR, \fB\-\-format\fR=\fITYPE\fR
+\fB\-9\fR, \fB\-\-format\fR=\fI\,TYPE\/\fR
compile expressions to TYPE automata
.SS "Regexp selection and interpretation:"
.TP
@@ -45,10 +45,10 @@ PATTERN is a Perl regular expression
\fB\-X\fR, \fB\-\-xerox\-regexp\fR
PATTERN is a Xerox regulare expression
.TP
-\fB\-e\fR, \fB\-\-regexp\fR=\fIPATTERN\fR
+\fB\-e\fR, \fB\-\-regexp\fR=\fI\,PATTERN\/\fR
use PATTERN for matching
.TP
-\fB\-f\fR, \fB\-\-file\fR=\fIFILE\fR
+\fB\-f\fR, \fB\-\-file\fR=\fI\,FILE\/\fR
obtain PATTERN from FILE
.TP
\fB\-I\fR, \fB\-\-ignore\-case\fR
@@ -71,7 +71,7 @@ suppress error messages
select non\-matching lines
.SS "Output control:"
.TP
-\fB\-m\fR, \fB\-\-max\-count\fR=\fINUM\fR
+\fB\-m\fR, \fB\-\-max\-count\fR=\fI\,NUM\/\fR
stop after NUM matches\en \fB\-b\fR, \fB\-\-byte\-offset\fR print the byte offset with output lines
.TP
\fB\-n\fR, \fB\-\-line\-number\fR
@@ -86,40 +86,40 @@ print the filename for each match
\fB\-h\fR, \fB\-\-no\-filename\fR
suppress the prefixing filename on output
.TP
-\fB\-\-label\fR=\fILABEL\fR
+\fB\-\-label\fR=\fI\,LABEL\/\fR
print LABEL as filename for standard input
.TP
\fB\-o\fR, \fB\-\-only\-matching\fR
show only the part of a line matching PATTERN
.TP
-\fB\-\-binary\-files\fR=\fITYPE\fR
+\fB\-\-binary\-files\fR=\fI\,TYPE\/\fR
assume that binary files are TYPE;
TYPE is `binary', `text', or `without\-match'
.TP
\fB\-a\fR, \fB\-\-text\fR
-equivalent to \fB\-\-binary\-files\fR=\fItext\fR
+equivalent to \fB\-\-binary\-files\fR=\fI\,text\/\fR
.TP
-\fB\-d\fR, \fB\-\-directories\fR=\fIACTION\fR
+\fB\-d\fR, \fB\-\-directories\fR=\fI\,ACTION\/\fR
how to handle directories;
ACTION is `read', `recurse', or `skip'
.TP
-\fB\-D\fR, \fB\-\-devices\fR=\fIACTION\fR
+\fB\-D\fR, \fB\-\-devices\fR=\fI\,ACTION\/\fR
how to handle devices, FIFOs and sockets;
ACTION is `read' or `skip'
.TP
\fB\-R\fR, \fB\-r\fR, \fB\-\-recursive\fR
-equivalent to \fB\-\-directories\fR=\fIrecurse\fR
+equivalent to \fB\-\-directories\fR=\fI\,recurse\/\fR
.TP
-\fB\-\-include\fR=\fIFILE_PATTERN\fR
+\fB\-\-include\fR=\fI\,FILE_PATTERN\/\fR
search only files that match FILE_PATTERN
.TP
-\fB\-\-exclude\fR=\fIFILE_PATTERN\fR
+\fB\-\-exclude\fR=\fI\,FILE_PATTERN\/\fR
skip files and directories matching FILE_PATTERN
.TP
-\fB\-\-exclude\-from\fR=\fIFILE\fR
+\fB\-\-exclude\-from\fR=\fI\,FILE\/\fR
skip files matching any file pattern from FILE
.TP
-\fB\-\-exclude\-dir\fR=\fIPATTERN\fR
+\fB\-\-exclude\-dir\fR=\fI\,PATTERN\/\fR
directories that match PATTERN will be skipped
.TP
\fB\-L\fR, \fB\-\-files\-without\-match\fR
@@ -138,18 +138,18 @@ make tabs line up (if needed)
print 0 byte after FILE name
.SS "Context control:"
.TP
-\fB\-B\fR, \fB\-\-before\-context\fR=\fINUM\fR
+\fB\-B\fR, \fB\-\-before\-context\fR=\fI\,NUM\/\fR
print NUM lines of leading context
.TP
-\fB\-A\fR, \fB\-\-after\-context\fR=\fINUM\fR
+\fB\-A\fR, \fB\-\-after\-context\fR=\fI\,NUM\/\fR
print NUM lines of trailing context
.TP
-\fB\-C\fR, \fB\-\-context\fR=\fINUM\fR
+\fB\-C\fR, \fB\-\-context\fR=\fI\,NUM\/\fR
print NUM lines of output context
.HP
-\fB\-\-color\fR[=\fIWHEN\fR],
+\fB\-\-color\fR[=\fI\,WHEN\/\fR],
.TP
-\fB\-\-colour\fR[=\fIWHEN\fR]
+\fB\-\-colour\fR[=\fI\,WHEN\/\fR]
use markers to highlight the matching strings;
WHEN is `always', `never', or `auto'
.TP
@@ -160,14 +160,14 @@ do not strip CR characters at EOL (MSDOS)
report offsets as if CRs were not there (MSDOS)
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
hfst\-grep home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstGrep>
.br
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-guess.1 b/man/hfst-guess.1
index a67520c..6a9d6fd 100644
--- a/man/hfst-guess.1
+++ b/man/hfst-guess.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-GUESS "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-GUESS "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-guess \- =Use a guesser (and generator) to guess analyses or inflectional
.SH SYNOPSIS
.B hfst-guess
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Use a guesser (and generator) to guess analyses or inflectional
paradigms of unknown words.
@@ -26,10 +26,10 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Guesser options:"
.TP
@@ -64,7 +64,7 @@ increase load time.
If OUTFILE or INFILE is missing or \-, standard streams will be used.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-guess home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstGuess>
@@ -72,7 +72,7 @@ hfst\-guess home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-guessify.1 b/man/hfst-guessify.1
index c90bc4c..33e761c 100644
--- a/man/hfst-guessify.1
+++ b/man/hfst-guessify.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-GUESSIFY "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-GUESSIFY "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-guessify \- =Compile a morphological analyzer into a guesser and generator.
.SH SYNOPSIS
.B hfst-guessify
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Compile a morphological analyzer into a guesser and generator.
.SS "Common options:"
@@ -25,10 +25,10 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Guesser options:"
.TP
@@ -56,7 +56,7 @@ analyses of unknown word forms, \fB\-d\fR has no effect on load time.
If OUTFILE or INFILE is missing or \-, standard streams will be used.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-guessify home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstGuessify>
@@ -64,7 +64,7 @@ hfst\-guessify home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-head.1 b/man/hfst-head.1
index 18cb6c4..5229c92 100644
--- a/man/hfst-head.1
+++ b/man/hfst-head.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-HEAD "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-HEAD "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-head \- =Get first transducers from an archive
.SH SYNOPSIS
.B hfst-head
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Get first transducers from an archive
.SS "Common options:"
@@ -25,14 +25,14 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Archive options:"
.TP
-\fB\-n\fR, \fB\-\-n\-first\fR=\fI[\-]K\fR
+\fB\-n\fR, \fB\-\-n\-first\fR=\fI\,[\-]K\/\fR
print the first K transducers;
with the leading `\-', print all but last K transducers
.PP
@@ -42,7 +42,7 @@ K must be an integer, as parsed by strtoul base 10, and not 0.
If K is omitted default is 1.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-head home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHead>
@@ -50,7 +50,7 @@ hfst\-head home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-info.1 b/man/hfst-info.1
index e312fe9..212f6be 100644
--- a/man/hfst-info.1
+++ b/man/hfst-info.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-INFO "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-INFO "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-info \- =show or test HFST versions and features
.SH SYNOPSIS
.B hfst-info
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
show or test HFST versions and features
.SS "Common options:"
@@ -25,23 +25,23 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Test features:"
.TP
-\fB\-a\fR, \fB\-\-atleast\-version\fR=\fIMVER\fR
+\fB\-a\fR, \fB\-\-atleast\-version\fR=\fI\,MVER\/\fR
require at least MVER version of HFST
.TP
-\fB\-e\fR, \fB\-\-exact\-version\fR=\fIEVER\fR
+\fB\-e\fR, \fB\-\-exact\-version\fR=\fI\,EVER\/\fR
require exactly EVER version of HFST
.TP
-\fB\-m\fR, \fB\-\-max\-version\fR=\fIUVER\fR
+\fB\-m\fR, \fB\-\-max\-version\fR=\fI\,UVER\/\fR
require at most UVER version of HFST
.TP
-\fB\-f\fR, \fB\-\-requirefeature\fR=\fIFEAT\fR
+\fB\-f\fR, \fB\-\-requirefeature\fR=\fI\,FEAT\/\fR
require named FEAT support from HFST
.PP
MVER, EVER or UVER version vectors must be composed of one to three full stop separated runs of digits.
FEAT should be name of feature supported by HFST, such as SFST, foma or openfst
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-info home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstInfo>
@@ -49,7 +49,7 @@ hfst\-info home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-invert.1 b/man/hfst-invert.1
index f4bb18b..0e371a1 100644
--- a/man/hfst-invert.1
+++ b/man/hfst-invert.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-INVERT "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-INVERT "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-invert \- =Invert a transducer
.SH SYNOPSIS
.B hfst-invert
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Invert a transducer
.SS "Common options:"
@@ -25,17 +25,17 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
Format of result depends on format of INFILE
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-invert home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstInvert>
@@ -43,7 +43,7 @@ hfst\-invert home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-lexc-compiler.1 b/man/hfst-lexc-compiler.1
deleted file mode 100644
index 6f64319..0000000
--- a/man/hfst-lexc-compiler.1
+++ /dev/null
@@ -1,57 +0,0 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.38.4.
-.TH HFST-LEXC-COMPILER "1" "March 2011" "HFST" "User Commands"
-.SH NAME
-hfst-lexc-compiler \- manual page for hfst-lexc-compiler 0.1 (hfst 3.0)
-.SH SYNOPSIS
-.B hfst-lexc-compiler
-[\fIOPTIONS\fR...] [\fIINFILE1\fR...]]
-.SH DESCRIPTION
-Compile lexc files into transducer
-.SS "Common options:"
-.TP
-\fB\-h\fR, \fB\-\-help\fR
-Print help message
-.TP
-\fB\-V\fR, \fB\-\-version\fR
-Print version info
-.TP
-\fB\-v\fR, \fB\-\-verbose\fR
-Print verbosely while processing
-.TP
-\fB\-q\fR, \fB\-\-quiet\fR
-Do not print output
-.TP
-\fB\-s\fR, \fB\-\-silent\fR
-Alias of \fB\-\-quiet\fR
-.SS "Input/Output options:"
-.TP
-\fB\-f\fR, \fB\-\-format\fR=\fIFORMAT\fR
-compile into FORMAT transducer
-.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
-write result into OUTFILE
-.PP
-If INFILE or OUTFILE are omitted or \-, standard streams will be used
-FORMAT must be one of supported transducer formats, such as openfst\-tropical, sfst, foma, etc.
-.SH EXAMPLES
-.TP
-hfst\-lexc\-compiler \fB\-o\fR cat.hfst cat.lexc
-Compile single\-file lexicon
-.TP
-hfst\-lexc\-compiler \fB\-o\fR L.hfst Root.lexc 2.lexc 3.lexc
-Compile multi\-file lexicon
-.SH "REPORTING BUGS"
-Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
-.PP
-hfst\-lexc\-compiler home page:
-<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstLexcCompiler>
-.br
-General help using HFST software:
-<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
-.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
-License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
-.br
-This is free software: you are free to change and redistribute it.
-There is NO WARRANTY, to the extent permitted by law.
diff --git a/man/hfst-lexc.1 b/man/hfst-lexc.1
index 333b0d1..71428e0 100644
--- a/man/hfst-lexc.1
+++ b/man/hfst-lexc.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-LEXC "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-LEXC "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-lexc \- =Compile lexc files into transducer
.SH SYNOPSIS
.B hfst-lexc
-[\fIOPTIONS\fR...] [\fIINFILE1\fR...]]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE1\/\fR...]]
.SH DESCRIPTION
Compile lexc files into transducer
.SS "Common options:"
@@ -25,10 +25,10 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-f\fR, \fB\-\-format\fR=\fIFORMAT\fR
+\fB\-f\fR, \fB\-\-format\fR=\fI\,FORMAT\/\fR
compile into FORMAT transducer
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
write result into OUTFILE
.SS "Lexc options:"
.TP
@@ -48,10 +48,10 @@ if \fB\-\-withFlags\fR is used, minimize the number of flags
if \fB\-\-withFlags\fR and \fB\-\-minimizeFlags\fR are used, rename
flags (for testing)
.TP
-\fB\-x\fR, \fB\-\-xerox\-composition\fR=\fIVALUE\fR Whether flag diacritics are treated as ordinary
+\fB\-x\fR, \fB\-\-xerox\-composition\fR=\fI\,VALUE\/\fR Whether flag diacritics are treated as ordinary
symbols in composition (default is true).
.TP
-\fB\-X\fR, \fB\-\-xfst\fR=\fIVARIABLE\fR
+\fB\-X\fR, \fB\-\-xfst\fR=\fI\,VARIABLE\/\fR
toggle xfst compatibility option VARIABLE.
.TP
\fB\-W\fR, \fB\-\-Werror\fR
@@ -64,10 +64,10 @@ VALUEs recognized are {true,ON,yes} and {false,OFF,no}.
Xfst variables are {flag\-is\-epsilon (default OFF)}.
.SH EXAMPLES
.TP
-hfst\-lexc \fB\-o\fR cat.hfst cat.lexc
+hfst\-lexc \-o cat.hfst cat.lexc
Compile single\-file lexicon
.TP
-hfst\-lexc \fB\-o\fR L.hfst Root.lexc 2.lexc 3.lexc
+hfst\-lexc \-o L.hfst Root.lexc 2.lexc 3.lexc
Compile multi\-file lexicon
.SS "Using weights:"
.IP
@@ -79,7 +79,7 @@ Using weights has an effect only if FORMAT is weighted, i.e.
{ openfst\-tropical, openfst\-log, optimized\-lookup\-weighted }.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-lexc home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstLexc>
@@ -87,7 +87,7 @@ hfst\-lexc home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-lookup.1 b/man/hfst-lookup.1
index 9a3f4f1..fb61337 100644
--- a/man/hfst-lookup.1
+++ b/man/hfst-lookup.1
@@ -1,12 +1,16 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-LOOKUP "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-LOOKUP "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-lookup \- =perform transducer lookup (apply)
.SH SYNOPSIS
.B hfst-lookup
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
perform transducer lookup (apply)
+NOTE: hfst\-lookup does lookup from left to right as opposed to xfst and foma
+.IP
+lookup which is carried out from right to left. In order to do lookup
+in a similar way as xfst and foma, use 'hfst\-flookup' instead.
.SS "Common options:"
.TP
\fB\-h\fR, \fB\-\-help\fR
@@ -25,43 +29,51 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output to OUTFILE
.HP
-\fB\-p\fR, \fB\-\-pipe\-mode\fR[=\fISTREAM\fR] Control input and output streams
+\fB\-p\fR, \fB\-\-pipe\-mode\fR[=\fI\,STREAM\/\fR] Control input and output streams
.SS "Lookup options:"
.TP
-\fB\-I\fR, \fB\-\-input\-strings\fR=\fISFILE\fR
+\fB\-I\fR, \fB\-\-input\-strings\fR=\fI\,SFILE\/\fR
Read lookup strings from SFILE
.TP
-\fB\-O\fR, \fB\-\-output\-format\fR=\fIOFORMAT\fR
+\fB\-O\fR, \fB\-\-output\-format\fR=\fI\,OFORMAT\/\fR
Use OFORMAT printing results sets
.TP
-\fB\-e\fR, \fB\-\-epsilon\-format\fR=\fIEPS\fR
+\fB\-e\fR, \fB\-\-epsilon\-format\fR=\fI\,EPS\/\fR
Print epsilon as EPS
.TP
-\fB\-F\fR, \fB\-\-input\-format\fR=\fIIFORMAT\fR
+\fB\-F\fR, \fB\-\-input\-format\fR=\fI\,IFORMAT\/\fR
Use IFORMAT parsing input
.TP
\fB\-x\fR, \fB\-\-statistics\fR
Print statistics
.TP
-\fB\-X\fR, \fB\-\-xfst\fR=\fIVARIABLE\fR
+\fB\-X\fR, \fB\-\-xfst\fR=\fI\,VARIABLE\/\fR
Toggle xfst VARIABLE
.TP
-\fB\-c\fR, \fB\-\-cycles\fR=\fIINT\fR
+\fB\-c\fR, \fB\-\-cycles\fR=\fI\,INT\/\fR
How many times to follow input epsilon cycles
+(only for non\-lookup\-optimized transducers)
+.TP
+\fB\-n\fR, \fB\-\-max\-number\fR=\fI\,INT\/\fR
+Maximum number of results printed for each input
+(only for lookup\-optimized transducers)
.TP
-\fB\-b\fR, \fB\-\-beam\fR=\fIB\fR
+\fB\-b\fR, \fB\-\-beam\fR=\fI\,B\/\fR
Output only analyses whose weight is within B from
the best analysis
.TP
-\fB\-t\fR, \fB\-\-time\-cutoff\fR=\fIS\fR
+\fB\-t\fR, \fB\-\-time\-cutoff\fR=\fI\,S\/\fR
Limit search after having used S seconds per input
-(currently only works in optimized\-lookup mode
+(only for lookup\-optimized transducers)
+.TP
+\fB\-C\fR, \fB\-\-cascade\fR=\fI\,CASCADE\/\fR
+How multiple transducers in input are handled
.TP
\fB\-P\fR, \fB\-\-progress\fR
Show neat progress bar if possible
@@ -80,21 +92,23 @@ S must be a non\-negative float. The default, 0.0, indicates no cutoff.
If the input contains several transducers, a set containing
results from all transducers is printed for each input string.
.PP
+CASCADE must be one of { union, priority\-union, composition }.
+If not specified, defaults to {union}.
+.PP
STREAM can be { input, output, both }. If not given, defaults to {both}.
If input file is not specified with \fB\-I\fR, input is read interactively line by
-line from the user. If you redirect input from a file, use \fB\-\-pipe\-mode\fR=\fIinput\fR.
-\fB\-\-pipe\-mode\fR=\fIoutput\fR is ignored on non\-windows platforms.
+line from the user. If you redirect input from a file, use \fB\-\-pipe\-mode\fR=\fI\,input\/\fR.
+\fB\-\-pipe\-mode\fR=\fI\,output\/\fR is ignored on non\-windows platforms.
.SS "Todo:"
.IP
-For optimized lookup format, only strings that pass flag diacritic checks
-are printed and flag diacritic symbols are not printed.
-Support VARIABLE 'print\-space' for optimized lookup format
+Support \fB\-\-xfst\fR=\fI\,obey\-flags\/\fR for optimized lookup format.
+Support \fB\-\-cycles\fR for optimized lookup format.
.SS "Known bugs:"
.IP
\&'quote\-special' quotes spaces that come from 'print\-space'
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-lookup home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstLookup>
@@ -102,7 +116,7 @@ hfst\-lookup home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-minimize.1 b/man/hfst-minimize.1
index 5770566..779b83f 100644
--- a/man/hfst-minimize.1
+++ b/man/hfst-minimize.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-MINIMIZE "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-MINIMIZE "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-minimize \- =Minimize a transducer
.SH SYNOPSIS
.B hfst-minimize
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Minimize a transducer
.SS "Common options:"
@@ -25,10 +25,10 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Command-specific options:"
.TP
@@ -40,7 +40,7 @@ If OUTFILE or INFILE is missing or \-, standard streams will be used.
Format of result depends on format of INFILE
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-minimize home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstMinimize>
@@ -48,7 +48,7 @@ hfst\-minimize home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-multiply.1 b/man/hfst-multiply.1
index 10c3216..677a943 100644
--- a/man/hfst-multiply.1
+++ b/man/hfst-multiply.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-MULTIPLY "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-MULTIPLY "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-multiply \- =Use first transducer of an archive repeatedly
.SH SYNOPSIS
.B hfst-multiply
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Use first transducer of an archive repeatedly
.SS "Common options:"
@@ -25,14 +25,14 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Archive options:"
.TP
-\fB\-n\fR, \fB\-\-n\-last\fR=\fINUMBER\fR
+\fB\-n\fR, \fB\-\-n\-last\fR=\fI\,NUMBER\/\fR
Duplicate each transducer NUMBER times
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
@@ -40,7 +40,7 @@ Format of result depends on format of INFILE
NUMBER must be a positive integer as parsed by strtoul base 10
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-multiply home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstDuplicate>
@@ -48,7 +48,7 @@ hfst\-multiply home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-name.1 b/man/hfst-name.1
index addd522..d1e81fd 100644
--- a/man/hfst-name.1
+++ b/man/hfst-name.1
@@ -1,21 +1,21 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-NAME "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-NAME "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-name \- =Name a transducer
.SH SYNOPSIS
.B hfst-name
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Name a transducer
.SS "Name options:"
.TP
-\fB\-n\fR, \fB\-\-name\fR=\fINAME\fR
+\fB\-n\fR, \fB\-\-name\fR=\fI\,NAME\/\fR
Name the transducer NAME
.TP
\fB\-p\fR, \fB\-\-print\-name\fR
Only print the current name
.TP
-\fB\-t\fR, \fB\-\-truncate_length\fR=\fILEN\fR
+\fB\-t\fR, \fB\-\-truncate_length\fR=\fI\,LEN\/\fR
Truncate name length to LEN
.SS "Common options:"
.TP
@@ -35,17 +35,17 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
Format of result depends on format of INFILE
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-name home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstName>
@@ -53,7 +53,7 @@ hfst\-name home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-optimized-lookup.1 b/man/hfst-optimized-lookup.1
index 8baaf0e..8121020 100644
--- a/man/hfst-optimized-lookup.1
+++ b/man/hfst-optimized-lookup.1
@@ -1,12 +1,16 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-OPTIMIZED-LOOKUP "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-OPTIMIZED-LOOKUP "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-optimized-lookup \- =Usage: hfst-optimized-lookup [OPTIONS] TRANSDUCER
.SH SYNOPSIS
.B hfst-optimized-lookup
-[\fIOPTIONS\fR] \fITRANSDUCER\fR
+[\fI\,OPTIONS\/\fR] \fI\,TRANSDUCER\/\fR
.SH DESCRIPTION
Run a transducer on standard input (one word per line) and print analyses
+NOTE: hfst\-optimized\-lookup does lookup from left to right as opposed to xfst
+.IP
+and foma lookup which is carried out from right to left. In order to do
+lookup in a similar way as xfst and foma, invert the transducer first.
.TP
\fB\-h\fR, \fB\-\-help\fR
Print this help message
@@ -33,15 +37,15 @@ Print final analysis weights (if any)
\fB\-u\fR, \fB\-\-unique\fR
Suppress duplicate analyses
.TP
-\fB\-n\fR N, \fB\-\-analyses\fR=\fIN\fR
+\fB\-n\fR N, \fB\-\-analyses\fR=\fI\,N\/\fR
Output no more than N analyses
(if the transducer is weighted, the N best analyses)
.TP
-\fB\-b\fR, \fB\-\-beam\fR=\fIB\fR
+\fB\-b\fR, \fB\-\-beam\fR=\fI\,B\/\fR
Output only analyses whose weight is within B from
the best analysis
.TP
-\fB\-t\fR, \fB\-\-time\-cutoff\fR=\fIS\fR
+\fB\-t\fR, \fB\-\-time\-cutoff\fR=\fI\,S\/\fR
Limit search after having used S seconds per input
.TP
\fB\-x\fR, \fB\-\-xerox\fR
@@ -52,7 +56,7 @@ Be as fast as possible.
(with this option enabled \fB\-u\fR and \fB\-n\fR don't work and
output won't be ordered by weight).
.TP
-\fB\-p\fR, \fB\-\-pipe\-mode\fR[=\fISTREAM\fR]
+\fB\-p\fR, \fB\-\-pipe\-mode\fR[=\fI\,STREAM\/\fR]
Control input and output streams.
.PP
N must be a positive integer. B must be a non\-negative float.
@@ -61,12 +65,12 @@ Options \fB\-n\fR and \fB\-b\fR are combined with AND, i.e. they both restrict t
.PP
STREAM can be { input, output, both }. If not given, defaults to {both}.
Input is read interactively line by line from the user. If you redirect input
-from a file, use \fB\-\-pipe\-mode\fR=\fIinput\fR. \fB\-\-pipe\-mode\fR=\fIoutput\fR is ignored on non\-windows
+from a file, use \fB\-\-pipe\-mode\fR=\fI\,input\/\fR. \fB\-\-pipe\-mode\fR=\fI\,output\/\fR is ignored on non\-windows
platforms.
.SH "REPORTING BUGS"
Report bugs to hfst\-bugs at helsinki.fi
.PP
hfst\-optimized\-lookup 1.2
.br
-Dec 10 2015 17:39:54
+Mar 23 2017 15:04:12
copyright (C) 2009 University of Helsinki
diff --git a/man/hfst-pair-test.1 b/man/hfst-pair-test.1
index 7fc10bd..cf9d4bc 100644
--- a/man/hfst-pair-test.1
+++ b/man/hfst-pair-test.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-PAIR-TEST "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-PAIR-TEST "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-pair-test \- =pair test for a twolc rule file.
.SH SYNOPSIS
.B hfst-pair-test
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
pair test for a twolc rule file.
.SS "Common options:"
@@ -25,10 +25,10 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input rule file from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write test output to OUTFILE
.TP
\fB\-N\fR \fB\-\-negative\-test\fR
@@ -40,7 +40,7 @@ In xerox mode, test cases are harvested
from a twolc source file.
.SS "Pair test options:"
.TP
-\fB\-I\fR, \fB\-\-input\-strings\fR=\fISFILE\fR
+\fB\-I\fR, \fB\-\-input\-strings\fR=\fI\,SFILE\/\fR
Read pair test strings from
SFILE
.PP
@@ -77,23 +77,23 @@ the strings that are allowed are printed.
.PP
In Xerox mode, the input should be a twolc file. Tests consist of
two lines: an input form and an output form. The test cases are
-specialized comments prefixed with either '!???' or '!$' depeding on
+specialized comments prefixed with either '!!???' or '!!$' depeding on
whether the pair should succeed or fail. An example of a positive
test:
.PP
-!??? earlYer
-!??? earlier
+!!??? earlYer
+!!??? earlier
.PP
An example of a negative test:
.PP
-!$ earlYer
-!$ earlyer
+!!$ earlYer
+!!$ earlyer
.PP
In silent mode (\fB\-s\fR), the program won't print anything. Only the
exit code tells whether the test was successful or not.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-pair\-test home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstPairTest>
@@ -101,7 +101,7 @@ hfst\-pair\-test home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-pmatch.1 b/man/hfst-pmatch.1
index 6441473..939218d 100644
--- a/man/hfst-pmatch.1
+++ b/man/hfst-pmatch.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-PMATCH "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-PMATCH "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-pmatch \- =perform matching/lookup on text streams
.SH SYNOPSIS
.B hfst-pmatch
-[\fIOPTIONS\fR...] \fITRANSDUCER\fR
+[\fI\,OPTIONS\/\fR...] \fI\,TRANSDUCER\/\fR
.SH DESCRIPTION
perform matching/lookup on text streams
.SS "Common options:"
@@ -23,23 +23,48 @@ Only print fatal erros and requested output
.TP
\fB\-s\fR, \fB\-\-silent\fR
Alias of \fB\-\-quiet\fR
+.SS "Input/Output options:"
+.TP
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
+Read input transducer from INFILE
+.TP
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
+Write output transducer to OUTFILE
.TP
\fB\-n\fR \fB\-\-newline\fR
Newline as input separator (default is blank line)
.TP
-\fB\-x\fR \fB\-\-extract\-tags\fR
+\fB\-x\fR \fB\-\-extract\-patterns\fR
Only print tagged parts in output
.TP
\fB\-l\fR \fB\-\-locate\fR
Only print locations of matches
.TP
+\fB\-c\fR \fB\-\-count\-patterns\fR
+Print the total number of matches when done
+.TP
+\fB\-\-delete\-patterns\fR
+Replace matches with opening tags
+.TP
+\fB\-\-no\-mark\-patterns\fR
+Don't tag matched patterns
+.TP
+\fB\-\-max\-context\fR
+Upper limit to context length allowed
+.TP
+\fB\-\-max\-recursion\fR
+Upper limit for recursion
+.TP
+\fB\-t\fR, \fB\-\-time\-cutoff\fR=\fI\,S\/\fR
+Limit search after having used S seconds per input
+.TP
\fB\-p\fR \fB\-\-profile\fR
Produce profiling data
.PP
Use standard streams for input and output.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-pmatch home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstPmatch>
@@ -47,7 +72,7 @@ hfst\-pmatch home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-pmatch2fst.1 b/man/hfst-pmatch2fst.1
index 529e6cf..3ff8d71 100644
--- a/man/hfst-pmatch2fst.1
+++ b/man/hfst-pmatch2fst.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-PMATCH2FST "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-PMATCH2FST "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-pmatch2fst \- =Compile regular expressions into transducer(s)
.SH SYNOPSIS
.B hfst-pmatch2fst
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Compile regular expressions into transducer(s)
.IP
@@ -27,14 +27,14 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "String and format options:"
.TP
-\fB\-e\fR, \fB\-\-epsilon\fR=\fIEPS\fR
+\fB\-e\fR, \fB\-\-epsilon\fR=\fI\,EPS\/\fR
Map EPS as zero
.TP
\fB\-\-flatten\fR
@@ -51,7 +51,7 @@ UppercaseAlpha Alpha* LC({professor}) EndTag(ProfName);" | hfst\-pmatch2fst
create matcher that tags "professor Chomsky" as "professor <ProfName>Chomsky</ProfName>"
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-pmatch2fst home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//Pmatch2Fst>
@@ -59,7 +59,7 @@ hfst\-pmatch2fst home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-project.1 b/man/hfst-project.1
index c117061..8ff6927 100644
--- a/man/hfst-project.1
+++ b/man/hfst-project.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-PROJECT "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-PROJECT "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-project \- =Project (extract a level) transducer
.SH SYNOPSIS
.B hfst-project
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Project (extract a level) transducer
.SS "Common options:"
@@ -25,14 +25,14 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Projection options:"
.TP
-\fB\-p\fR, \fB\-\-project\fR=\fILEVEL\fR
+\fB\-p\fR, \fB\-\-project\fR=\fI\,LEVEL\/\fR
project extracting tape LEVEL
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
@@ -40,7 +40,7 @@ Format of result depends on format of INFILE
LEVEL must be one of upper, input, first, analysis or lower, output, second, generation
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-project home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstProject>
@@ -48,7 +48,7 @@ hfst\-project home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-prune-alphabet.1 b/man/hfst-prune-alphabet.1
index 45a8b82..9644203 100644
--- a/man/hfst-prune-alphabet.1
+++ b/man/hfst-prune-alphabet.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-PRUNE-ALPHABET "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-PRUNE-ALPHABET "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-prune-alphabet \- =Prune the alphabet of a transducer
.SH SYNOPSIS
.B hfst-prune-alphabet
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Prune the alphabet of a transducer
.SS "Common options:"
@@ -25,10 +25,10 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Alphabet pruning options:"
.TP
@@ -43,7 +43,7 @@ If OUTFILE or INFILE is missing or \-, standard streams will be used.
Format of result depends on format of INFILE
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-prune\-alphabet home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstPruneAlphabet>
@@ -51,7 +51,7 @@ hfst\-prune\-alphabet home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-push-weights.1 b/man/hfst-push-weights.1
index 549c654..760c2e3 100644
--- a/man/hfst-push-weights.1
+++ b/man/hfst-push-weights.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-PUSH-WEIGHTS "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-PUSH-WEIGHTS "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-push-weights \- =Push weights of transducer
.SH SYNOPSIS
.B hfst-push-weights
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Push weights of transducer
.SS "Common options:"
@@ -25,14 +25,14 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Push options:"
.TP
-\fB\-p\fR, \fB\-\-push\fR=\fIDIRECTION\fR
+\fB\-p\fR, \fB\-\-push\fR=\fI\,DIRECTION\/\fR
push to DIRECTION
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
@@ -40,7 +40,7 @@ Format of result depends on format of INFILE
DIRECTION must be one of start, initial, begin or end, final
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-push\-weights home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstPush>
@@ -48,7 +48,7 @@ hfst\-push\-weights home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-regexp2fst.1 b/man/hfst-regexp2fst.1
index eaba95c..f50eb5e 100644
--- a/man/hfst-regexp2fst.1
+++ b/man/hfst-regexp2fst.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-REGEXP2FST "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-REGEXP2FST "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-regexp2fst \- =Compile (weighted) regular expressions into transducer(s)
.SH SYNOPSIS
.B hfst-regexp2fst
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Compile (weighted) regular expressions into transducer(s)
Common options:
@@ -25,14 +25,14 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "String and format options:"
.TP
-\fB\-f\fR, \fB\-\-format\fR=\fIFMT\fR
+\fB\-f\fR, \fB\-\-format\fR=\fI\,FMT\/\fR
Write result in FMT format
.TP
\fB\-j\fR, \fB\-\-disjunct\fR
@@ -45,15 +45,15 @@ Input is line separated (default)
\fB\-S\fR, \fB\-\-semicolon\fR
Input is semicolon separated
.TP
-\fB\-e\fR, \fB\-\-epsilon\fR=\fIEPS\fR
+\fB\-e\fR, \fB\-\-epsilon\fR=\fI\,EPS\/\fR
Map EPS as zero, i.e. epsilon.
.TP
-\fB\-x\fR, \fB\-\-xerox\-composition\fR=\fIVALUE\fR Whether flag diacritics are treated as ordinary
+\fB\-x\fR, \fB\-\-xerox\-composition\fR=\fI\,VALUE\/\fR Whether flag diacritics are treated as ordinary
symbols in composition (default is false).
.TP
-\fB\-X\fR, \fB\-\-xfst\fR=\fIVARIABLE\fR
+\fB\-X\fR, \fB\-\-xfst\fR=\fI\,VARIABLE\/\fR
Toggle xfst compatibility option VARIABLE.
-.SS "Harmonization:"
+.SS "Harmonization and optimization options:"
.TP
\fB\-H\fR, \fB\-\-do\-not\-harmonize\fR
Do not expand '?' symbols.
@@ -63,6 +63,9 @@ Harmonize flag diacritics.
.TP
\fB\-E\fR, \fB\-\-encode\-weights\fR
Encode weights when minimizing (default is false).
+.TP
+\fB\-M\fR, \fB\-\-do\-not\-minimize\fR
+Determinize result instead of minimizing it.
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
FMT must be one of the following: {foma, sfst, openfst\-tropical, openfst\-log}.
@@ -81,13 +84,13 @@ echo " c:d a:o::3 t:g " | hfst\-regexp2fst
same but with weight 3
in the middle
.TP
-echo " cat ; dog ; "3" " | hfst\-regexp2fst \fB\-S\fR
+echo " cat ; dog ; "3" " | hfst\-regexp2fst \-S
create transducers
.IP
"cat" and "dog" and "3"
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-regexp2fst home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//Regexp2Fst>
@@ -95,7 +98,7 @@ hfst\-regexp2fst home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-remove-epsilons.1 b/man/hfst-remove-epsilons.1
index bdf65ab..f032ffc 100644
--- a/man/hfst-remove-epsilons.1
+++ b/man/hfst-remove-epsilons.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-REMOVE-EPSILONS "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-REMOVE-EPSILONS "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-remove-epsilons \- =Remove epsilons from a transducer
.SH SYNOPSIS
.B hfst-remove-epsilons
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Remove epsilons from a transducer
.SS "Common options:"
@@ -25,17 +25,17 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
Format of result depends on format of INFILE
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-remove\-epsilons home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstRemoveEpsilons>
@@ -43,7 +43,7 @@ hfst\-remove\-epsilons home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-repeat.1 b/man/hfst-repeat.1
index 8310782..17c1baf 100644
--- a/man/hfst-repeat.1
+++ b/man/hfst-repeat.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-REPEAT "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-REPEAT "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-repeat \- =Repeat transducer
.SH SYNOPSIS
.B hfst-repeat
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Repeat transducer
.SS "Common options:"
@@ -25,17 +25,17 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Repetition options:"
.TP
-\fB\-f\fR, \fB\-\-from\fR=\fIFNUM\fR
+\fB\-f\fR, \fB\-\-from\fR=\fI\,FNUM\/\fR
repeat at least FNUM times
.TP
-\fB\-t\fR, \fB\-\-to\fR=\fITNUM\fR
+\fB\-t\fR, \fB\-\-to\fR=\fI\,TNUM\/\fR
repeat at most TNUM times
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
@@ -45,7 +45,7 @@ if FNUM is omitted it defaults to 0, if TNUM is omitted it defaults to Inf
FNUM must be less than TNUM
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-repeat home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstRepeat>
@@ -53,7 +53,7 @@ hfst\-repeat home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-reverse.1 b/man/hfst-reverse.1
index 7300812..57864ff 100644
--- a/man/hfst-reverse.1
+++ b/man/hfst-reverse.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-REVERSE "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-REVERSE "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-reverse \- =Reverse a transducer
.SH SYNOPSIS
.B hfst-reverse
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Reverse a transducer
.SS "Common options:"
@@ -25,17 +25,17 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
Format of result depends on format of INFILE
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-reverse home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstReverse>
@@ -43,7 +43,7 @@ hfst\-reverse home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-reweight-tagger.1 b/man/hfst-reweight-tagger.1
index 5ce1354..51604ed 100644
--- a/man/hfst-reweight-tagger.1
+++ b/man/hfst-reweight-tagger.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-REWEIGHT-TAGGER "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-REWEIGHT-TAGGER "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-reweight-tagger \- =Reweight a tagger accoring to a configuration file
.SH SYNOPSIS
.B hfst-reweight-tagger
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Reweight a tagger accoring to a configuration file
hfst_tagger_config.
@@ -26,14 +26,14 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-reweight\-tagger home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstReweightTagger>
@@ -41,7 +41,7 @@ hfst\-reweight\-tagger home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-reweight.1 b/man/hfst-reweight.1
index cbe1fab..e4bc3ab 100644
--- a/man/hfst-reweight.1
+++ b/man/hfst-reweight.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-REWEIGHT "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-REWEIGHT "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-reweight \- =Reweight transducer weights simply
.SH SYNOPSIS
.B hfst-reweight
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Reweight transducer weights simply
.SS "Common options:"
@@ -25,36 +25,36 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Reweighting options:"
.TP
-\fB\-a\fR, \fB\-\-addition\fR=\fIAVAL\fR
+\fB\-a\fR, \fB\-\-addition\fR=\fI\,AVAL\/\fR
add AVAL to matching weights
.TP
-\fB\-b\fR, \fB\-\-multiplier\fR=\fIBVAL\fR
+\fB\-b\fR, \fB\-\-multiplier\fR=\fI\,BVAL\/\fR
multiply matching weights by BVAL
.TP
-\fB\-F\fR, \fB\-\-function\fR=\fIFNAME\fR
+\fB\-F\fR, \fB\-\-function\fR=\fI\,FNAME\/\fR
operate matching weights by FNAME
.TP
-\fB\-l\fR, \fB\-\-lower\-bound\fR=\fILVAL\fR
+\fB\-l\fR, \fB\-\-lower\-bound\fR=\fI\,LVAL\/\fR
match weights greater than LVAL
.TP
-\fB\-u\fR, \fB\-\-upper\-bound\fR=\fIUVAL\fR
+\fB\-u\fR, \fB\-\-upper\-bound\fR=\fI\,UVAL\/\fR
match weights less than UVAL
.TP
-\fB\-I\fR, \fB\-\-input\-symbol\fR=\fIISYM\fR
+\fB\-I\fR, \fB\-\-input\-symbol\fR=\fI\,ISYM\/\fR
match arcs with input symbol ISYM
.TP
-\fB\-O\fR, \fB\-\-output\-symbol\fR=\fIOSYM\fR
+\fB\-O\fR, \fB\-\-output\-symbol\fR=\fI\,OSYM\/\fR
match arcs with output symbol OSYM
.TP
-\fB\-S\fR, \fB\-\-symbol\fR=\fISYM\fR
-match arcs havins symbol SYM
+\fB\-S\fR, \fB\-\-symbol\fR=\fI\,SYM\/\fR
+match arcs with input or output symbol SYM or both
.TP
\fB\-e\fR, \fB\-\-end\-states\-only\fR
match end states only, no arcs
@@ -62,7 +62,7 @@ match end states only, no arcs
\fB\-A\fR, \fB\-\-arcs\-only\fR
match arcs only, no end states
.TP
-\fB\-T\fR, \fB\-\-tsv\-file\fR=\fITFILE\fR
+\fB\-T\fR, \fB\-\-tsv\-file\fR=\fI\,TFILE\/\fR
read reweighting rules from TFILE
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
@@ -70,6 +70,7 @@ Format of result depends on format of INFILE
If AVAL, BVAL or FNAME are omitted, they default to neutral elements of addition, multiplication or identity function.
If LVAL or UVAL are omitted, they default to minimum and maximum values of the weight structure.
If ISYM, OSYM or SYM are omitted, they default to a value that matches all arcs.
+Only one ISYM, OSYM and SYM can be given.
.PP
Float values are parsed with strtod(3) and integers strtoul(3).
The functions allowed for FNAME are <cmath> float functions with parameter count of 1 and a matching return value:
@@ -88,7 +89,7 @@ Weights are by default modified for all arcs and end states,
unless option \fB\-\-end\-states\-only\fR or \fB\-\-arcs\-only\fR is used.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-reweight home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstReweight>
@@ -96,7 +97,7 @@ hfst\-reweight home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-shuffle.1 b/man/hfst-shuffle.1
index 98f5bf4..bc86d64 100644
--- a/man/hfst-shuffle.1
+++ b/man/hfst-shuffle.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-SHUFFLE "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-SHUFFLE "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-shuffle \- =Shuffle two transducers
.SH SYNOPSIS
.B hfst-shuffle
-[\fIOPTIONS\fR...] [\fIINFILE1 \fR[\fIINFILE2\fR]]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE1 \/\fR[\fI\,INFILE2\/\fR]]
.SH DESCRIPTION
Shuffle two transducers
.SS "Common options:"
@@ -25,16 +25,16 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-1\fR, \fB\-\-input1\fR=\fIINFILE1\fR
+\fB\-1\fR, \fB\-\-input1\fR=\fI\,INFILE1\/\fR
Read first input transducer from INFILE1
.TP
-\fB\-2\fR, \fB\-\-input2\fR=\fIINFILE2\fR
+\fB\-2\fR, \fB\-\-input2\fR=\fI\,INFILE2\/\fR
Read second input transducer from INFILE2
.TP
\fB\-C\fR, \fB\-\-do\-not\-convert\fR
Do not allow transducers to be converted into the same type
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write results to OUTFILE
.PP
If OUTFILE, or either INFILE1 or INFILE2 is missing or \-,
@@ -49,10 +49,10 @@ If INFILE2 has only one transducer, the operation is applied for
each transducer in INFILE1 keeping the second transducer constant.
.SH EXAMPLES
.IP
-hfst\-shuffle \fB\-o\fR shuffled.hfst cat.hfst dog.hfst
+hfst\-shuffle \-o shuffled.hfst cat.hfst dog.hfst
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-shuffle home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstShuffle>
@@ -60,7 +60,7 @@ hfst\-shuffle home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-split.1 b/man/hfst-split.1
index a358278..d1e3cd9 100644
--- a/man/hfst-split.1
+++ b/man/hfst-split.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-SPLIT "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-SPLIT "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-split \- =Extract transducers from archive with systematic file names
.SH SYNOPSIS
.B hfst-split
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Extract transducers from archive with systematic file names
.SS "Common options:"
@@ -25,13 +25,13 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-p\fR, \fB\-\-prefix\fR=\fIPRE\fR
+\fB\-p\fR, \fB\-\-prefix\fR=\fI\,PRE\/\fR
Use the prefix PRE in naming output files
.TP
-\fB\-e\fR, \fB\-\-extension\fR=\fIEXT\fR
+\fB\-e\fR, \fB\-\-extension\fR=\fI\,EXT\/\fR
Use the extension EXT in naming output files
.PP
If INFILE is omitted or \-, stdin is used.
@@ -47,7 +47,7 @@ This command creates files "rule1.tr" (equivalent to transducer_a)
and "rule2.tr" (equivalent to transducer_b).
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-split home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstSplit>
@@ -55,7 +55,7 @@ hfst\-split home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-strings2fst.1 b/man/hfst-strings2fst.1
index a974cfd..c7af5e2 100644
--- a/man/hfst-strings2fst.1
+++ b/man/hfst-strings2fst.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-STRINGS2FST "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-STRINGS2FST "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-strings2fst \- =Compile string pairs and pair-strings into transducer(s)
.SH SYNOPSIS
.B hfst-strings2fst
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Compile string pairs and pair\-strings into transducer(s)
.SS "Common options:"
@@ -25,14 +25,14 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input strings from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "String and format options:"
.TP
-\fB\-f\fR, \fB\-\-format\fR=\fIFMT\fR
+\fB\-f\fR, \fB\-\-format\fR=\fI\,FMT\/\fR
Write result in FMT format
.TP
\fB\-j\fR, \fB\-\-disjunct\-strings\fR
@@ -55,10 +55,10 @@ Input is in pairstring format
\fB\-S\fR, \fB\-\-has\-spaces\fR
Input has spaces between symbols/symbol pairs
.TP
-\fB\-e\fR, \fB\-\-epsilon\fR=\fIEPS\fR
+\fB\-e\fR, \fB\-\-epsilon\fR=\fI\,EPS\/\fR
Interpret string EPS as epsilon.
.TP
-\fB\-m\fR, \fB\-\-multichar\-symbols\fR=\fIFILE\fR
+\fB\-m\fR, \fB\-\-multichar\-symbols\fR=\fI\,FILE\/\fR
Strings that must be tokenized as one symbol.
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
@@ -77,17 +77,17 @@ by a tabulator. The weight cannot be zero.
echo "cat:dog" | hfst\-strings2fst
create cat:dog fst
.TP
-echo "c:da:ot:g" | hfst\-strings2fst \fB\-p\fR
+echo "c:da:ot:g" | hfst\-strings2fst \-p
same as pairstring
.TP
-echo "c:d a:o t:g" | hfst\-strings2fst \fB\-p\fR \fB\-S\fR
+echo "c:d a:o t:g" | hfst\-strings2fst \-p \-S
same as pairstring with spaces
.TP
-echo "c a t:d o g" | hfst\-strings2fst \fB\-S\fR
+echo "c a t:d o g" | hfst\-strings2fst \-S
same with spaces
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-strings2fst home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//Strings2Fst>
@@ -95,7 +95,7 @@ hfst\-strings2fst home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-substitute.1 b/man/hfst-substitute.1
index d41dbf9..140e9ed 100644
--- a/man/hfst-substitute.1
+++ b/man/hfst-substitute.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-SUBSTITUTE "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-SUBSTITUTE "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-substitute \- =Relabel transducer arcs
.SH SYNOPSIS
.B hfst-substitute
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Relabel transducer arcs
.SS "Common options:"
@@ -25,23 +25,23 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Relabeling options:"
.TP
-\fB\-f\fR, \fB\-\-from\-label\fR=\fIFLABEL\fR
+\fB\-f\fR, \fB\-\-from\-label\fR=\fI\,FLABEL\/\fR
replace FLABEL
.TP
-\fB\-t\fR, \fB\-\-to\-label\fR=\fITLABEL\fR
+\fB\-t\fR, \fB\-\-to\-label\fR=\fI\,TLABEL\/\fR
replace with TLABEL
.TP
-\fB\-T\fR, \fB\-\-to\-transducer\fR=\fITFILE\fR
+\fB\-T\fR, \fB\-\-to\-transducer\fR=\fI\,TFILE\/\fR
replace with transducer read from TFILE
.TP
-\fB\-F\fR, \fB\-\-from\-file\fR=\fILABELFILE\fR
+\fB\-F\fR, \fB\-\-from\-file\fR=\fI\,LABELFILE\/\fR
read replacements from LABELFILE
.TP
\fB\-R\fR, \fB\-\-in\-order\fR
@@ -66,20 +66,20 @@ LABELFILE is a 2 column tsv file where col 1 is FLABEL
and col 2 gives TLABEL specifications.
.SH EXAMPLES
.IP
-hfst\-substitute \fB\-i\fR tr.hfst \fB\-o\fR tr_relabeled.hfst \fB\-f\fR 'a' \fB\-t\fR 'A'
+hfst\-substitute \-i tr.hfst \-o tr_relabeled.hfst \-f 'a' \-t 'A'
.IP
relabel all symbols 'a' with 'A'
.IP
-hfst\-substitute \fB\-i\fR tr.hfst \fB\-o\fR tr_relabeled.hfst \fB\-f\fR 'a:b' \fB\-t\fR 'A:B'
+hfst\-substitute \-i tr.hfst \-o tr_relabeled.hfst \-f 'a:b' \-t 'A:B'
.IP
relabel all arcs 'a:b' with 'A:B'
.IP
-hfst\-substitute \fB\-i\fR tr.hfst \fB\-o\fR tr_relabeled.hfst \fB\-f\fR 'a:b' \fB\-T\fR repl.hfst
+hfst\-substitute \-i tr.hfst \-o tr_relabeled.hfst \-f 'a:b' \-T repl.hfst
.IP
replace all arcs 'a:b' with transducer repl.hfst
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-substitute home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstSubstitute>
@@ -87,7 +87,7 @@ hfst\-substitute home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-subtract.1 b/man/hfst-subtract.1
index ef720dc..4eaf5c9 100644
--- a/man/hfst-subtract.1
+++ b/man/hfst-subtract.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-SUBTRACT "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-SUBTRACT "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-subtract \- =Subtract (minus) two transducers
.SH SYNOPSIS
.B hfst-subtract
-[\fIOPTIONS\fR...] [\fIINFILE1 \fR[\fIINFILE2\fR]]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE1 \/\fR[\fI\,INFILE2\/\fR]]
.SH DESCRIPTION
Subtract (minus) two transducers
.SS "Common options:"
@@ -25,16 +25,16 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-1\fR, \fB\-\-input1\fR=\fIINFILE1\fR
+\fB\-1\fR, \fB\-\-input1\fR=\fI\,INFILE1\/\fR
Read first input transducer from INFILE1
.TP
-\fB\-2\fR, \fB\-\-input2\fR=\fIINFILE2\fR
+\fB\-2\fR, \fB\-\-input2\fR=\fI\,INFILE2\/\fR
Read second input transducer from INFILE2
.TP
\fB\-C\fR, \fB\-\-do\-not\-convert\fR
Do not allow transducers to be converted into the same type
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write results to OUTFILE
.SS "Flag diacritics:"
.TP
@@ -55,11 +55,11 @@ If INFILE2 has only one transducer, the operation is applied for
each transducer in INFILE1 keeping the second transducer constant.
.SH EXAMPLES
.TP
-hfst\-subtract \fB\-o\fR catdog.hfst cat.hfst dog.hfst
+hfst\-subtract \-o catdog.hfst cat.hfst dog.hfst
subtracts transducers
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-subtract home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstSubtract>
@@ -67,7 +67,7 @@ hfst\-subtract home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-summarize.1 b/man/hfst-summarize.1
index c249763..031108d 100644
--- a/man/hfst-summarize.1
+++ b/man/hfst-summarize.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-SUMMARIZE "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-SUMMARIZE "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-summarize \- =Calculate the properties of a transducer
.SH SYNOPSIS
.B hfst-summarize
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Calculate the properties of a transducer
.SS "Common options:"
@@ -25,14 +25,14 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Summarize options:"
.TP
-\fB\-S\fR, \fB\-\-print\-symbol\-pair\-statistics\fR=\fIN\fR
+\fB\-S\fR, \fB\-\-print\-symbol\-pair\-statistics\fR=\fI\,N\/\fR
Print info about symbol pairs that occur
at most N times (default is infinity)
.PP
@@ -43,7 +43,7 @@ The parameter \fB\-\-verbose\fR gives more extensive information on
the properties of a transducer.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-summarize home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstSummarize>
@@ -51,7 +51,7 @@ hfst\-summarize home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-tag.1 b/man/hfst-tag.1
index 021c9d1..a73b371 100644
--- a/man/hfst-tag.1
+++ b/man/hfst-tag.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-TAG "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-TAG "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-tag \- =Tag a text file given from stdin using an hfst tagger.
.SH SYNOPSIS
.B hfst-tag
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Tag a text file given from stdin using an hfst tagger.
.SS "Common options:"
@@ -25,14 +25,14 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-tag home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstTag>
@@ -40,7 +40,7 @@ hfst\-tag home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-tail.1 b/man/hfst-tail.1
index e328db3..85ffb62 100644
--- a/man/hfst-tail.1
+++ b/man/hfst-tail.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-TAIL "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-TAIL "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-tail \- =Get last transducers from an archive
.SH SYNOPSIS
.B hfst-tail
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Get last transducers from an archive
.SS "Common options:"
@@ -25,14 +25,14 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Archive options:"
.TP
-\fB\-n\fR, \fB\-\-n\-last\fR=\fI[\fR+]K
+\fB\-n\fR, \fB\-\-n\-last\fR=\fI\,[\/\fR+]K
Print the last K transducers;
use +K to print transducers starting from the Kth
.PP
@@ -42,7 +42,7 @@ K must be an integer, as parsed by strtoul base 10, and not 0.
if K is omitted, it defaults to +1 (all except the first)
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-tail home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstTail>
@@ -50,7 +50,7 @@ hfst\-tail home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-tokenize.1 b/man/hfst-tokenize.1
index 79821ad..4517e7d 100644
--- a/man/hfst-tokenize.1
+++ b/man/hfst-tokenize.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.46.4.
-.TH HFST-TOKENIZE "1" "January 2016" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-TOKENIZE "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-tokenize \- =perform matching/lookup on text streams
.SH SYNOPSIS
.B hfst-tokenize
-[\fI\,--segment | --xerox | --cg\/\fR] [\fI\,OPTIONS\/\fR...] \fI\,RULESET\/\fR
+[\fI\,--segment | --xerox | --cg | --giella-cg\/\fR] [\fI\,OPTIONS\/\fR...] \fI\,RULESET\/\fR
.SH DESCRIPTION
perform matching/lookup on text streams
.SS "Common options:"
@@ -24,39 +24,59 @@ Only print fatal erros and requested output
\fB\-s\fR, \fB\-\-silent\fR
Alias of \fB\-\-quiet\fR
.TP
-\fB\-n\fR \fB\-\-newline\fR
+\fB\-n\fR, \fB\-\-newline\fR
Newline as input separator (default is blank line)
.TP
-\fB\-a\fR \fB\-\-print\-all\fR
+\fB\-a\fR, \fB\-\-print\-all\fR
Print nonmatching text
.TP
-\fB\-w\fR \fB\-\-print\-weight\fR
+\fB\-w\fR, \fB\-\-print\-weight\fR
Print weights
.TP
-\fB\-\-tokenize\-multichar\fR
-Tokenize multicharacter symbols
+\fB\-m\fR, \fB\-\-tokenize\-multichar\fR Tokenize multicharacter symbols
(by default only one utf\-8 character is tokenized at a time
regardless of what is present in the alphabet)
.TP
-\fB\-t\fR, \fB\-\-time\-cutoff\fR=\fI\,S\/\fR
+\fB\-tS\fR, \fB\-\-time\-cutoff\fR=\fI\,S\/\fR
Limit search after having used S seconds per input
.TP
-\fB\-\-segment\fR
+\fB\-lN\fR, \fB\-\-weight\-classes\fR=\fI\,N\/\fR
+Output no more than N best weight classes
+(where analyses with equal weight constitute a class
+.TP
+\fB\-u\fR, \fB\-\-unique\fR
+Remove duplicate analyses
+.TP
+\fB\-z\fR, \fB\-\-segment\fR
Segmenting / tokenization mode (default)
.TP
-\fB\-\-xerox\fR
+\fB\-i\fR, \fB\-\-space\-separated\fR
+Tokenization with one sentence per line, space\-separated tokens
+.TP
+\fB\-x\fR, \fB\-\-xerox\fR
Xerox output
.TP
-\fB\-\-cg\fR
-cg output
+\fB\-c\fR, \fB\-\-cg\fR
+Constraint Grammar output
+.TP
+\fB\-S\fR, \fB\-\-superblanks\fR
+Ignore contents of unescaped [] (cf. apertium\-destxt); flush on NUL
+.TP
+\fB\-g\fR, \fB\-\-giella\-cg\fR
+CG format used in Giella infrastructe (implies \fB\-l2\fR,
+treats @PMATCH_INPUT_MARK@ as subreading separator,
+expects tags to start or end with +, flush on NUL)
+.TP
+\fB\-C\fR \fB\-\-conllu\fR
+CoNLL\-U format
.TP
-\fB\-\-finnpos\fR
+\fB\-f\fR, \fB\-\-finnpos\fR
FinnPos output
.PP
Use standard streams for input and output (for now).
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-tokenize home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstTokenize>
@@ -64,7 +84,7 @@ hfst\-tokenize home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-tool.1.template b/man/hfst-tool.1.template
index 4afa3f0..b57a2dc 100644
--- a/man/hfst-tool.1.template
+++ b/man/hfst-tool.1.template
@@ -85,7 +85,7 @@ Frobnicating diagonal automata will lead to inconclusive results. Don't do it.
.SH BUGS
You cannot frobnicate automata of weighted backend formats.
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-train-tagger-system.1 b/man/hfst-train-tagger-system.1
deleted file mode 120000
index d893136..0000000
--- a/man/hfst-train-tagger-system.1
+++ /dev/null
@@ -1 +0,0 @@
-hfst-open-input-file-for-tagger.1
\ No newline at end of file
diff --git a/man/hfst-traverse.1 b/man/hfst-traverse.1
index 4d68585..72488b7 100644
--- a/man/hfst-traverse.1
+++ b/man/hfst-traverse.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-TRAVERSE "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-TRAVERSE "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-traverse \- =Walk through the transducer arc by arc
.SH SYNOPSIS
.B hfst-traverse
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Walk through the transducer arc by arc
.SS "Common options:"
@@ -25,17 +25,17 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
Format of result depends on format of INFILE
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-traverse home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstDeterminize>
@@ -43,7 +43,7 @@ hfst\-traverse home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-twolc-system.1 b/man/hfst-twolc-system.1
deleted file mode 120000
index f1c9142..0000000
--- a/man/hfst-twolc-system.1
+++ /dev/null
@@ -1 +0,0 @@
-hfst-twolc.1
\ No newline at end of file
diff --git a/man/hfst-twolc.1 b/man/hfst-twolc.1
index 19b5c8b..991ec95 100644
--- a/man/hfst-twolc.1
+++ b/man/hfst-twolc.1
@@ -1,13 +1,13 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-TWOLC "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-TWOLC "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-twolc \- =
.SH SYNOPSIS
.B hfst-twolc
-[\fIOPTIONS\fR...] \fIINFILE\fR
+[\fI\,OPTIONS\/\fR...] \fI\,INFILE\/\fR
.SH DESCRIPTION
Usage: hfst\-twolc [OPTIONS...] \fB\-i\fR INFILE
-Usage: hfst\-twolc [OPTIONS...] \fB\-\-input\fR=\fIINFILE\fR
+Usage: hfst\-twolc [OPTIONS...] \fB\-\-input\fR=\fI\,INFILE\/\fR
Usage: cat INFILE | hfst\-twolc [OPTIONS...]
An input file has to be given either using the option \fB\-i\fR or
\fB\-\-input\fR, as the last commandline argument or from STDIN.
@@ -37,10 +37,10 @@ Do not print output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "TwolC grammar options:"
.TP
@@ -49,7 +49,7 @@ Resolve left\-arrow conflicts.
.HP
\fB\-D\fR, \fB\-\-dont\-resolve\-right\fR Don't resolve right\-arrow conflicts.
.TP
-\fB\-f\fR, \fB\-\-format\fR=\fIFORMAT\fR
+\fB\-f\fR, \fB\-\-format\fR=\fI\,FORMAT\/\fR
Store result in format FORMAT.
.PP
Format may be one of openfst\-log, openfst\-tropical, foma or sfst.
@@ -57,7 +57,7 @@ Format may be one of openfst\-log, openfst\-tropical, foma or sfst.
By default format is openfst\-tropical. By default right arrow
conflicts are resolved and left arrow conflicts are not resolved.
.PP
-hfst\-twolc 0 (hfst 3.8.3)
+hfst\-twolc 0 (hfst 3.12.2)
Copyright \(co 2010 University of Helsinki,
License GPLv3: GNU GPL version 3
<http://gnu.org/licenses/gpl.html>
diff --git a/man/hfst-txt2fst.1 b/man/hfst-txt2fst.1
index 77fd78a..2c08121 100644
--- a/man/hfst-txt2fst.1
+++ b/man/hfst-txt2fst.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-TXT2FST "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-TXT2FST "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-txt2fst \- =Convert AT&T or prolog format into a binary transducer
.SH SYNOPSIS
.B hfst-txt2fst
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+[\fI\,OPTIONS\/\fR...] [\fI\,INFILE\/\fR]
.SH DESCRIPTION
Convert AT&T or prolog format into a binary transducer
.SS "Common options:"
@@ -25,17 +25,17 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+\fB\-i\fR, \fB\-\-input\fR=\fI\,INFILE\/\fR
Read input transducer from INFILE
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+\fB\-o\fR, \fB\-\-output\fR=\fI\,OUTFILE\/\fR
Write output transducer to OUTFILE
.SS "Text and format options:"
.TP
-\fB\-f\fR, \fB\-\-format\fR=\fIFMT\fR
+\fB\-f\fR, \fB\-\-format\fR=\fI\,FMT\/\fR
Write result using FMT as backend format
.TP
-\fB\-e\fR, \fB\-\-epsilon\fR=\fIEPS\fR
+\fB\-e\fR, \fB\-\-epsilon\fR=\fI\,EPS\/\fR
Interpret string EPS as epsilon in att format
.TP
\fB\-p\fR, \fB\-\-prolog\fR
@@ -56,7 +56,7 @@ Space in transition symbols must be escaped as '@_SPACE_@' when using
att format.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-txt2fst home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstTxt2Fst>
@@ -64,7 +64,7 @@ hfst\-txt2fst home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/man/hfst-xfst.1 b/man/hfst-xfst.1
index de3c42a..3b56dff 100644
--- a/man/hfst-xfst.1
+++ b/man/hfst-xfst.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-XFST "1" "December 2015" "HFST" "User Commands"
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3.
+.TH HFST-XFST "1" "March 2017" "HFST" "User Commands"
.SH NAME
hfst-xfst \- =Compile XFST scripts or execute XFST commands interactively
.SH SYNOPSIS
.B hfst-xfst
-[\fIOPTIONS\fR...]
+[\fI\,OPTIONS\/\fR...]
.SH DESCRIPTION
Compile XFST scripts or execute XFST commands interactively
.SS "Common options:"
@@ -25,19 +25,19 @@ Only print fatal erros and requested output
Alias of \fB\-\-quiet\fR
.SS "Xfst-specific options:"
.TP
-\fB\-e\fR, \fB\-\-execute\fR=\fICMD\fR
+\fB\-e\fR, \fB\-\-execute\fR=\fI\,CMD\/\fR
Execute command CMD on startup
.TP
-\fB\-f\fR, \fB\-\-format\fR=\fIFMT\fR
+\fB\-f\fR, \fB\-\-format\fR=\fI\,FMT\/\fR
Write result using FMT as backend format
.TP
-\fB\-F\fR, \fB\-\-scriptfile\fR=\fIFILE\fR
+\fB\-F\fR, \fB\-\-scriptfile\fR=\fI\,FILE\/\fR
Read commands from FILE, and quit
.TP
-\fB\-l\fR, \fB\-\-startupfile\fR=\fIFILE\fR
+\fB\-l\fR, \fB\-\-startupfile\fR=\fI\,FILE\/\fR
Read commands from FILE on startup
.HP
-\fB\-p\fR, \fB\-\-pipe\-mode\fR[=\fISTREAM\fR] Control input and output streams
+\fB\-p\fR, \fB\-\-pipe\-mode\fR[=\fI\,STREAM\/\fR] Control input and output streams
.TP
\fB\-r\fR, \fB\-\-no\-readline\fR
Do not use readline library for input
@@ -53,11 +53,11 @@ Input files are always treated as UTF\-8.
.PP
STREAM can be { input, output, both }. If not given, defaults to {both}.
If input file is not specified with \fB\-F\fR, input is read interactively line by
-line from the user. If you redirect input from a file, use \fB\-\-pipe\-mode\fR=\fIinput\fR.
-\fB\-\-pipe\-mode\fR=\fIoutput\fR is ignored on non\-windows platforms.
+line from the user. If you redirect input from a file, use \fB\-\-pipe\-mode\fR=\fI\,input\/\fR.
+\fB\-\-pipe\-mode\fR=\fI\,output\/\fR is ignored on non\-windows platforms.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
-<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+<https://github.com/hfst/hfst/issues>
.PP
hfst\-xfst home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstXfst2Fst>
@@ -65,7 +65,7 @@ hfst\-xfst home page:
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
.SH COPYRIGHT
-Copyright \(co 2010 University of Helsinki,
+Copyright \(co 2017 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/python/Makefile.am b/python/Makefile.am
index 4c00a4c..daad914 100644
--- a/python/Makefile.am
+++ b/python/Makefile.am
@@ -1,12 +1,18 @@
-# Copyright (c) 2016 University of Helsinki
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
+# Copyright (c) 2016 University of Helsinki
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
# version 3 of the License, or (at your option) any later version.
-# See the file COPYING included with this distribution for more
+# See the file COPYING included with this distribution for more
# information.
SUBDIRS = test
-EXTRA_DIST = libhfst.i README setup.py
+EXTRA_DIST = libhfst.i docstrings.i README setup.py \
+hfst_extensions.cc hfst_file_extensions.cc hfst_lexc_extensions.cc hfst_sfst_extensions.cc \
+hfst_lookup_extensions.cc hfst_pmatch_extensions.cc hfst_prolog_extensions.cc \
+hfst_regex_extensions.cc hfst_rules_extensions.cc hfst_xfst_extensions.cc \
+hfst/__init__.py hfst/exceptions/__init__.py hfst/sfst_rules/__init__.py \
+hfst/xerox_rules/__init__.py
+
AUTOMAKE_OPTIONS = subdir-objects
diff --git a/python/README b/python/README
index a29eb4c..7fba5d2 100644
--- a/python/README
+++ b/python/README
@@ -1,59 +1,122 @@
- ------------------------------------------------
- *** THIS IS THE NEW PYTHON WRAPPER FOR HFST ***
- *** IT IS STILL UNDER CONSTRUCTION ***
- ------------------------------------------------
+This folder contains source code for SWIG bindings for using HFST library with
+Python. The bindings work with Python version 3.
-This is the SWIG binding for using HFST library with python.
+============
+REQUIREMENTS
+============
+
+The requirements for HFST C++ library are given in README of parent directory.
+The setup script has been tested with the following version combinations of
+swig and python: swig2.0.4/python3.2mu and swig3.0/python 3.4m.
+
+
+=====================
+BUILDING THE BINDINGS
+=====================
Building and installation is done using the standard Python module distutils.
-Once you've successfully built libhfst, use the script setup.py to build the
-Python extension eg. like so:
+Once you've successfully built and installed HFST library (located in the
+parent directory, if you are compiling from source), use the script setup.py
+to build the Python extension eg. like so:
- python setup.py build_ext --inplace
+ python3 setup.py build_ext --inplace
+If you have only built HFST library but not installed it (or have an earlier
+version of HFST library installed) and want to build the bindings, you either
+have to add full path to ../libhfst/src/.libs to LD_LIBRARY_PATH or hard-code
+the library path by modifying setup.py:
-NOTE: if you already have hfst installed, it is possible that SWIG will use
-the installed version of hfst library instead of the default located in
-"../libhfst/src/.libs/". To avoid this, add the full path to hfst library to
-LD_LIBRARY_PATH before building the Python extension, e.g. by executing
+ extra_link_arguments = []
+ # If you wish to link to the local HFST library, replace the above with:
+ # extra_link_arguments = ["-Wl,-rpath=" + absolute_libhfst_src_path + "/.libs"]
- LD_LIBRARY_PATH="path/to/hfst-top-dir/libhfst/src/.libs/:"$LD_LIBRARY_PATH
+=================
+RUNNING THE TESTS
+=================
-If you wish to install the extension, run
+The folder 'test' contains tests for the bindings. See test/README for more
+information.
- python setup.py install
+=======================
+INSTALLING THE BINDINGS
+=======================
-For further information, consult
+If you wish to install the extension, run
- python setup.py --help
- python setup.py --help-commands
+ python3 setup.py install
-and the distutils documentation.
+The same that was said about linking to HFST C++ library in 'Building the
+bindings' above, applies also to installing.
-NOTE: Some version combinations of SWIG and Python make HFST exception classes
-subclasses of Python's _object instead of Exception. Then you will get an
-error like
+For further information, consult
- TypeError: catching classes that do not inherit from BaseException is not allowed
+ python3 setup.py --help
+ python3 setup.py --help-commands
-If this is the case, run
+and the distutils documentation.
- sed -i 's/class HfstException(_object):/class HfstException(Exception):/' libhfst.py
-after build/installation to be able to use HfstException and its subclasses in Python.
+==================
+USING THE BINDINGS
+==================
+
+The bindings should be usable after installation in Python with command:
+ import hfst
-If you want to use Python bindings locally, add path/to/hfst-top-dir/python to
-PYTHONPATH, e.g. by executing
+NOTE: If you want to use Python bindings locally, either add the absolute path
+to this folder to PYTHONPATH, e.g. by executing
PYTHONPATH="path/to/hfst-top-dir/python:"$PYTHONPATH
-If this does not work, you can do the following in Python before importing libhfst:
+or do the following in Python before importing hfst:
import sys
sys.path.insert(1, 'path/to/hfst-top-dir/python')
+
+====================
+THE PYTHON INTERFACE
+====================
+
+C++ side functions and classes are wrapped with SWIG under module 'libhfst'. It
+is possible to use this module directly, but there is a package named 'hfst'
+which encapsulates the libhfst module in a more user-friendly manner. The
+structure of the package is
+
+ * hfst
+ * hfst.exceptions
+ * hfst.sfst_rules
+ * hfst.xerox_rules
+
+The module hfst.exceptions contains HfstException and its subclasses. The
+modules hfst.sfst_rules and hfst.xerox_rules contain functions that create
+transducers implementing replace and two-level rules. All other functions and
+classes are in module hfst.
+
+For documentation and examples, see <https://hfst.github.io/python/index.html>.
+
+
+==========
+KNOWN BUGS
+==========
+
+Some version combinations of SWIG and Python make HFST exception classes
+subclasses of Python's _object instead of Exception. Then you will get an
+error like
+
+ TypeError: catching classes that do not inherit from BaseException is not allowed
+
+If this is the case, run
+
+ sed -i 's/class HfstException(_object):/class HfstException(Exception):/' libhfst.py
+
+after build/installation to be able to use HfstException and its subclasses in
+Python.
+
+Python version 3.4 requires swig 3.0, else HFST exception classes will not be
+handled correctly by Python.
diff --git a/python/config.h b/python/config.h
new file mode 100644
index 0000000..e69de29
diff --git a/python/doc/Doxyfile b/python/doc/Doxyfile
index ed5f7fe..6dd80cc 100644
--- a/python/doc/Doxyfile
+++ b/python/doc/Doxyfile
@@ -31,7 +31,7 @@ PROJECT_NAME = "HFST - Helsinki Finite-State Transducer Technology - P
# This could be handy for archiving the generated documentation or
# if some version control system is used.
-PROJECT_NUMBER = "version 3.9.1"
+PROJECT_NUMBER = "version 3.12.1"
# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute)
# base path where the generated documentation will be put.
@@ -568,7 +568,7 @@ WARN_LOGFILE =
# directories like "/usr/src/myproject". Separate the files or directories
# with spaces.
-INPUT = "libhfst.py" # api_documentation.dox
+INPUT = "hfst/__init__.py" "hfst/exceptions/__init__.py" "hfst/sfst_rules/__init__.py" "hfst/xerox_rules/__init__.py"
# This tag can be used to specify the character encoding of the source files
# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is
diff --git a/python/doc/README b/python/doc/README
new file mode 100644
index 0000000..1c7aa6f
--- /dev/null
+++ b/python/doc/README
@@ -0,0 +1,17 @@
+
+This directory contains documentation for hfst package in Doxygen format.
+Run 'doxygen Doxyfile' to generate the documentation (it will be written in
+directory html which is created when doxygen is run) and copy everything under
+directory 'html' to HFST github repository (hfst.github.io/python/VERSION/,
+where VERSION is the version number of HFST, e.g. '3.12.0', or 'next_release'
+for upcoming releases). The documentation will be visible on page
+<https://hfst.github.io/python/VERSION/index.html>. Also remember to update
+the redirection link in hfst.github.io/python/index.html to point to the
+documentation of newest release.
+
+Scripts doxy2swig.py and xml2docstrings.sh have been used for creating hfst
+doxygen documentation semiautomatically from docstrings in ../docstrings.i,
+../libhfst.py and __init__.py files in directory ../hfst/. Currently,
+changes in documentation must be done twice (in 'hfst/python/doc/' as well as
+in 'hfst/python/') so that they are visible both on github pages and in hfst
+python package (via 'help' command).
diff --git a/python/doc/libhfst.py b/python/doc/hfst/__init__.py
similarity index 59%
rename from python/doc/libhfst.py
rename to python/doc/hfst/__init__.py
index b3cff7a..0a24eec 100644
--- a/python/doc/libhfst.py
+++ b/python/doc/hfst/__init__.py
@@ -1,22 +1,27 @@
-## @package libhfst
+## @package hfst
# HFST API for Python
## @mainpage
#
# HFST - The Helsinki Finite-State Transducer technology is intended for creating and manipulating weighted or unweighted synchronic transducers implementing regular relations.
# UTF-8 is chosen as the character encoding used in HFST software. Currently, HFST has been implemented using the
-# <a href="http://www.ims.uni-stuttgart.de/projekte/gramotron/SOFTWARE/SFST.html">SFST</a>,
-# <a href="http://www.openfst.org">OpenFst</a> and
+# <a href="http://www.ims.uni-stuttgart.de/projekte/gramotron/SOFTWARE/SFST.html">SFST</a>,
+# <a href="http://www.openfst.org">OpenFst</a> and
# <a href="http://code.google.com/p/foma/">foma</a> software libraries.
# Other versions may be added in some future release. SFST and foma implementations are unweighted and OpenFst implementation is weighted.
#
# Part of HFST interface has been implemented for HFST's own two transducer formats, HfstBasicTransducer and optimized lookup format.
-# The previous is useful for accessing individual states and transitions of a transducer, converting between transducer formats and storing transducers in an implementation-independent format.
+# The previous is useful for accessing individual states and transitions of a transducer, converting between transducer formats and storing transducers in an implementation-independent format.
# The latter is used for fast lookup of strings in a transducer.
#
# All back-end implementations - SFST, OpenFst and foma - work according to the same interface, so it is possible to compile the same piece of code using different back-end libraries.
# There are some differences related to weights, as only OpenFst supports them.
#
+# HFST is written in C++, but there is a Python interface available which is documented on these pages.
+# The Python API is basically a wrapper around the C++ API with some additional code and modifications.
+# The C++ API has developed around the HFST command line tools, but the Python version is intended to be used as such and has been designed to be more user-friendly.
+#
+#
# For a quick start to the HFST interface with examples, see <a href="QuickStart.html">here</a>.
#
# The examples given in this documentation use <a href="http://www.fsmbook.com/">Xerox transducer notation</a>.
@@ -25,15 +30,15 @@
#
# \section features Features
#
-# - Create transducers and apply \link libhfst.HfstTransducer operations\endlink on them
+# - Create transducers and apply \link hfst.HfstTransducer operations\endlink on them
#
-# - Create transducers \link libhfst.HfstBasicTransducer from scratch\endlink
+# - Create transducers \link hfst.HfstBasicTransducer from scratch\endlink
#
-# - \link libhfst.HfstBasicTransducer.states Iterate\endlink through a transducer's states and transitions
+# - \link hfst.HfstBasicTransducer.states Iterate\endlink through a transducer's states and transitions
#
-# - Create transducers by \link libhfst.HfstTokenizer tokenizing\endlink UTF-8 strings with multicharacter symbols
+# - Create transducers by \link hfst.HfstTokenizer tokenizing\endlink UTF-8 strings with multicharacter symbols
#
-# - Apply two-level, replace, restriction and coercion \link libhfst.two_level_if rules\endlink
+# - Apply \link hfst.xerox_rules.replace replace\endlink, \link hfst.sfst_rules.two_level_if two-level\endlink, restriction and coercion rules
#
#
# \section tutorial Tutorial
@@ -44,359 +49,19 @@
#
# \section download_hfst Download
#
-# - <a href="https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstDownloads">Download and install</a> the HFST interface and command line tools
+# - <a href="https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstPython">Download and install</a> HFST Python API
#
#
#\section links Links
#
-# - <a href="https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstCommandLineToolFunctionalities">Examples</a> of programs implemented using the HFST interface
+# - <a href="https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstHome">Documentation</a> for the HFST project
#
-# - <a href="https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstHome">Documentation</a> of the HFST project
+# - Documentation for releases <a href="https://hfst.github.io/python/next_release/index.html">3.12.2 (upcoming)</a>, <a href="https://hfst.github.io/python/3.12.1/index.html">3.12.1</a>, <a href="https://hfst.github.io/python/3.12.0/index.html">3.12.0</a> and <a href="https://hfst.github.io/python/3.11.0/index.html">3.11.0</a>.
#
# <br>
-
-## Base class for HfstExceptions. Holds its own name and the file and line number where it was thrown.
-class HfstException:
- ## The name of the exception.
- self.name
- ## The source file where the exception originated.
- self.file
- ## The line of the source file where the exception originated.
- self.line
-
-## Two or more HfstTransducers are not of the same type. Same as HfstTransducerTypeMismatchException ???
-class HfstTransducerTypeMismatchException(HfstException):
- pass
-
-## The library required by the implementation type requested is not linked to HFST.
-class ImplementationTypeNotAvailableException(HfstException):
- pass
-
-## Function has not been implemented (yet).
-class FunctionNotImplementedException(HfstException):
- pass
-
-## Flag diacritics encountered on one but not the other side of a transition.
-class FlagDiacriticsAreNotIdentitiesException(HfstException):
- pass
-
-## The input is not in valid prolog format.
-class NotValidPrologFormatException(HfstException):
- pass
-
-## Stream cannot be read.
-class StreamNotReadableException(HfstException):
- pass
-
-## Stream cannot be written.
-#
-# Thrown by #libhfst.HfstOutputStream.write and #libhfst.HfstTransducer.write_att
-class StreamCannotBeWrittenException(HfstException):
- pass
-
-## Stream is closed.
-#
-# Thrown by #libhfst.HfstTransducer.write_att
-# #libhfst.HfstOutputStream.write
-#
-# An example:
-#
-# \verbatim
-# try:
-# tr = libhfst.regex('foo')
-# outstr = libhfst.HfstOutputStream(filename='testfile')
-# outstr.close()
-# outstr.write(tr)
-# except libhfst.StreamIsClosedException:
-# print("Could not write transducer: stream to file was closed.")
-# \endverbatim
-class StreamIsClosedException(HfstException):
- pass
-
-## The stream is at end.
-#
-# Thrown by
-# #libhfst.HfstTransducer
-# #libhfst.HfstInputStream.__init__
-class EndOfStreamException(HfstException):
- pass
-
-## Transducer is cyclic.
-#
-# Thrown by #libhfst.HfstTransducer.extract_paths. An example
-# \verbatim
-# transducer = libhfst.regex('[a:b]*')
-# try:
-# results = transducer.extract_paths(output='text')
-# print("The transducer has %i paths:" % len(results))
-# print(results)
-# except libhfst.TransducerIsCyclicException:
-# print("The transducer is cyclic and has an infinite number of paths. Some of them:")
-# results = transducer.extract_paths(output='text', max_cycles=5)
-# print(results)
-# \endverbatim
-class TransducerIsCyclicException(HfstException):
- pass
-
-## The stream does not contain transducers.
-#
-# Thrown by
-# #libhfst.HfstTransducer
-# #libhfst.HfstInputStream.__init__
-#
-# An example.
-# \verbatim
-# f = open('foofile', 'w')
-# f.write('This is an ordinary text file.\n')
-# f.close()
-# try:
-# instr = libhfst.HfstInputStream('foofile')
-# tr = instr.read()
-# print(tr)
-# instr.close()
-# except libhfst.NotTransducerStreamException:
-# print("Could not print transducer: the file does not contain binary transducers.")
-# \endverbatim
-class NotTransducerStreamException(HfstException):
- pass
-
-## The stream is not in valid AT&T format.
-#
-# An example:
-# \verbatim
-# f = open('testfile1.att', 'w')
-# f.write('0 1 a b\n\
-# 1 2 c\n\
-# 2\n')
-# f.close()
-# f = libhfst.hfst_open('testfile1.att', 'r')
-# try:
-# tr = libhfst.read_att(f)
-# except libhfst.NotValidAttFormatException:
-# print('Could not read file: it is not in valid ATT format.')
-# f.close()
-# \endverbatim
-# thrown by
-# #libhfst.HfstTransducer.__init__
-class NotValidAttFormatException(HfstException):
- pass
-
-## The input is not in valid LexC format.
-class NotValidLexcFormatException(HfstException):
- pass
-
-## State is not final (and cannot have a final weight).
-#
-# An example :
-#
-# \verbatim
-# tr = libhfst.HfstBasicTransducer()
-# tr.add_state(1)
-# # An exception is thrown as state number 1 is not final
-# try:
-# w = tr.get_final_weight(1)
-# except libhfst.StateIsNotFinalException:
-# print("State is not final.")
-# \endverbatim
-#
-# You should use function #libhfst.HfstBasicTransducer.is_final_state if you are not sure whether a
-# state is final.
-#
-# Thrown by #libhfst.HfstBasicTransducer get_final_weight.
-class StateIsNotFinalException(HfstException):
- pass
-
-## Transducers given as rule context are not automata.
-# @see libhfst.HfstTransducer.is_automaton()
-class ContextTransducersAreNotAutomataException(HfstException):
- pass
-
-## Transducers are not automata.
-#
-# Example:
-# \verbatim
-# tr1 = libhfst.regex('foo:bar')
-# tr2 = libhfst.regex('bar:baz')
-# try:
-# tr1.cross_product(tr2)
-# except libhfst.TransducersAreNotAutomataException:
-# print('Transducers must be automata in cross product.')
-# \endverbatim
-# This exception is thrown by
-# #libhfst.HfstTransducer.cross_product
-# when either input transducer does not have equivalent input and
-# output symbols in all its transitions.
-class TransducersAreNotAutomataException(HfstException):
- pass
-
-## The state number argument is not valid.
-#
-# An example :
-#
-# \verbatim
-# tr = libhfst.HfstBasicTransducer()
-# tr.add_state(1)
-# try:
-# w = tr.get_final_weight(2)
-# except libhfst.StateIndexOutOfBoundsException:
-# print('State number 2 does not exist')
-# \endverbatim
-class StateIndexOutOfBoundsException(HfstException):
- pass
-
-## Transducer has a malformed HFST header.
-#
-# Thrown by #libhfst.HfstTransducer.__init__ #libhfst.HfstInputStream
-class TransducerHeaderException(HfstException):
- pass
-
-## An OpenFst transducer does not have an input symbol table.
-#
-# When converting from OpenFst to tropical or log HFST, the OpenFst transducer
-# must have at least an input symbol table. If the output symbol table
-# is missing, it is assumed to be equivalent to the input symbol table.
-#
-# Thrown by libhfst.HfstTransducer.__init__
-class MissingOpenFstInputSymbolTableException(HfstException):
- pass
-
-## Two or more transducers do not have the same type.
-#
-# This can happen if (1) the calling and called transducer in a binary
-# operation, (2) two transducers in a pair of transducers,
-# (3) two consecutive transducers coming from an HfstInputStream or
-# (4) two transducers in a function taking two or more transducers as
-# arguments do not have the same type.
-#
-# An example:
-# \verbatim
-# libhfst.set_default_fst_type(libhfst.TROPICAL_OPENFST_TYPE)
-# tr1 = libhfst.regex('foo')
-# tr2 = libhfst.regex('bar')
-# tr2.convert(libhfst.FOMA_TYPE)
-# try:
-# tr1.disjunct(tr2)
-# except libhfst.TransducerTypeMismatchException:
-# print('The implementation types of transducers must be the same.')
-# \endverbatim
-class TransducerTypeMismatchException(HfstException):
- pass
-
-## The set of transducer pairs is empty.
-#
-# Thrown by rule functions.
-class EmptySetOfContextsException(HfstException):
- pass
-
-## The type of a transducer is not specified.
-#
-# This exception is thrown when an implementation type argument
-# is ERROR_TYPE.
-class SpecifiedTypeRequiredException(HfstException):
- pass
-
-## An error happened probably due to a bug in the HFST code.
-class HfstFatalException(HfstException):
- pass
-
-## Transducer has wrong type.
-#
-# This exception suggests that an HfstTransducer has not been properly
-# initialized, probably due to a bug in the HFST library. Alternatively
-# the default constructor of HfstTransducer has been called at some point.
-#
-# @see #libhfst.HfstTransducer.__init__
-class TransducerHasWrongTypeException(HfstException):
- pass
-
-## String is not valid utf-8.
-#
-# This exception suggests that an input string is not valid utf8.
-#
-class IncorrectUtf8CodingException(HfstException):
- pass
-
-## An argument string is an empty string.
-# A transition symbol cannot be an empty string.
-class EmptyStringException(HfstException):
- pass
-
-## A bug in the HFST code.
-class SymbolNotFoundException(HfstException):
- pass
-
-## A piece of metadata in an HFST header is not supported.
-class MetadataException(HfstException):
- pass
-
-## An SFST transducer, unweighted.
-SFST_TYPE = _libhfst.SFST_TYPE
-
-## An OpenFst transducer with tropical weights.
-TROPICAL_OPENFST_TYPE = _libhfst.TROPICAL_OPENFST_TYPE
-
-## An OpenFst transducer with logarithmic weights (limited support).
-LOG_OPENFST_TYPE = _libhfst.LOG_OPENFST_TYPE
-
-## A foma transducer, unweighted.
-FOMA_TYPE = _libhfst.FOMA_TYPE
-
-## An HFST optimized lookup transducer, unweighted.
-HFST_OL_TYPE = _libhfst.HFST_OL_TYPE
-
-## An HFST optimized lookup transducer with weights.
-HFST_OLW_TYPE = _libhfst.HFST_OLW_TYPE
-
-## HFST2 header present, conversion required.
-HFST2_TYPE = _libhfst.HFST2_TYPE
-
-## Format left open by e.g. default constructor.
-UNSPECIFIED_TYPE = _libhfst.UNSPECIFIED_TYPE
-
-## Type not recognised. This type might be returned by a function if an error occurs.
-ERROR_TYPE = _libhfst.ERROR_TYPE
-
-## Push weights toward initial state.
-# @see #libhfst.HfstTransducer.push_weights
-TO_INITIAL_STATE = _libhfst.TO_INITIAL_STATE
-
-## Push weights toward final state(s).
-# @see #libhfst.HfstTransducer.push_weights
-TO_FINAL_STATE = _libhfst.TO_FINAL_STATE
-
-
-## A wrapper class for file operations.
-# @see libhfst.hfst_open, libhfst.hfst_stdin, libhfst.hfst_stdout, libhfst.hfst_stderr
-class HfstFile:
- ## Close the file.
- def close(self):
- pass
- ## Write string to the file.
- # @param str String to be written.
- def write(self, str):
- pass
- ## Whether the file is at end.
- def is_eof(self):
- pass
-
-## Open file named \a filename in mode \a mode.
-# @param filename The name of the file.
-# @param mode Mode in which the file is opened.
-def hfst_open(filename, mode):
- pass
-## Get file that points to standard input.
-def hfst_stdin():
- pass
-## Get file that points to standard output.
-def hfst_stdout():
- pass
-## Get file that points to standard error stream.
-def hfst_stderr():
- pass
-
## Set the default implementation type.
-# @param impl An libhfst.ImplementationType.
+# @param impl An hfst.ImplementationType.
#
# Set the implementation type (SFST_TYPE, TROPICAL_OPENFST_TYPE, FOMA_TYPE) that is
# used by default by all operations that create transducers. The default value is
@@ -404,22 +69,22 @@ def hfst_stderr():
def set_default_fst_type(impl):
pass
## Get default transducer implementation type.
-# If the default type is not set, it defaults to libhfst.TROPICAL_OPENFST_TYPE
+# If the default type is not set, it defaults to hfst.ImplementationType.TROPICAL_OPENFST_TYPE
def get_default_fst_type():
pass
## Get a string representation of transducer implementation type \a type.
-# @param type An libhfst.ImplementationType.
+# @param type An hfst.ImplementationType.
def fst_type_to_string(type):
pass
## The string for epsilon symbol.
# An example:
# \verbatim
-# fsm = libhfst.HfstBasicTransducer()
+# fsm = hfst.HfstBasicTransducer()
# fsm.add_state(1)
# fsm.set_final_weight(1, 2.0)
-# fsm.add_transition(0, 1, "foo", libhfst.EPSILON)
-# if not libhfst.HfstTransducer(fsm).compare(libhfst.regex('foo:0::2.0')):
+# fsm.add_transition(0, 1, "foo", hfst.EPSILON)
+# if not hfst.HfstTransducer(fsm).compare(hfst.regex('foo:0::2.0')):
# raise RuntimeError('')
# \endverbatim
# @note In regular expressions, "0" is used for the epsilon.
@@ -428,12 +93,12 @@ EPSILON='@_EPSILON_SYMBOL_@'
## The string for unknown symbol.
# An example:
# \verbatim
-# fsm = libhfst.HfstBasicTransducer()
+# fsm = hfst.HfstBasicTransducer()
# fsm.add_state(1)
# fsm.set_final_weight(1, -0.5)
-# fsm.add_transition(0, 1, "foo", libhfst.UNKNOWN)
+# fsm.add_transition(0, 1, "foo", hfst.UNKNOWN)
# fsm.add_transition(0, 1, "foo", "foo")
-# if not libhfst.HfstTransducer(fsm).compare(libhfst.regex('foo:?::-0.5')):
+# if not hfst.HfstTransducer(fsm).compare(hfst.regex('foo:?::-0.5')):
# raise RuntimeError('')
# \endverbatim
# @note In regular expressions, "?" on either or both sides of a transition is used for the unknown symbol.
@@ -442,11 +107,11 @@ UNKNOWN='@_UNKNOWN_SYMBOL_@'
## The string for identity symbol.
# An example:
# \verbatim
-# fsm = libhfst.HfstBasicTransducer()
+# fsm = hfst.HfstBasicTransducer()
# fsm.add_state(1)
# fsm.set_final_weight(1, 1.5)
-# fsm.add_transition(0, 1, libhfst.IDENTITY, libhfst.IDENTITY)
-# if not libhfst.HfstTransducer(fsm).compare(libhfst.regex('?::1.5')):
+# fsm.add_transition(0, 1, hfst.IDENTITY, hfst.IDENTITY)
+# if not hfst.HfstTransducer(fsm).compare(hfst.regex('?::1.5')):
# raise RuntimeError('')
# \endverbatim
# @note In regular expressions, a single "?" is used for the identity symbol.
@@ -460,7 +125,7 @@ IDENTITY='@_IDENTITY_SYMBOL_@'
# \verbatim
# One unweighted identity path:
# 'foo' -> [f o o]
-# Weighted path: a tuple of string and number, e.g.
+# Weighted path: a tuple of string and number, e.g.
# ('foo',1.4)
# ('bar',-3)
# ('baz',0)
@@ -475,16 +140,59 @@ IDENTITY='@_IDENTITY_SYMBOL_@'
def fst(arg):
pass
+## Get a transducer (automaton) where each transition symbol pair isymbol:osymbol of \a fst is replaced with a transition isymbolosymbol:isymbolosymbol, adding \a separator between isymbol and osymbol.
+# @param fst The transducer.
+# @param separator The separator symbol inserted between input and output symbols.
+#
+# Examples
+# \verbatim
+# import hfst
+# foo2bar = hfst.fst({'foo':'bar'})
+# \endverbatim
+# creates a transducer [f:b o:a o:r]. Calling
+# \verbatim
+# foobar = hfst.fst_to_fsa(foo2bar)
+# \endverbatim
+# will create the transducer [fb:fb oa:oa or:or] and
+# \verbatim
+# foobar = hfst.fst_to_fsa(foo2bar, '^')
+# \endverbatim
+# the transducer [f^b:f^b o^a:o^a o^r:o^r].
+# @see hfst.fsa_to_fst
+def fst_to_fsa(fst, separator=''):
+ pass
+
+
+## Get a transducer where each transition isymbolSosymbol:isymbolSosymbol of \a fsa is replaced a transition isymbol:osymbol, if \a separator is S.
+# @param fsa The transducer. Must be an automaton, i.e. for each transition, the input and output symbols must be the same. Else, a TransducerIsNotAutomatonException is thrown.
+# @param separator The symbol separating input and output symbol parts in \a fsa. If it is the empty string, length of each symbol in \a fsa (excluding special symbols of form "@...@") must be exactly 2. Else, a RuntimeError is thrown.
+
+#
+# Examples:
+# \verbatim
+# import hfst
+# foo2bar = hfst.fst({'foo':'bar'}) # creates transducer [f:b o:a o:r]
+# foobar = hfst.fst_to_fsa(foo2bar, '^')
+# \endverbatim
+# creates the transducer [f^b:f^b o^a:o^a o^r:o^r]. Then calling
+# \verbatim
+# foo2bar = hfst.fsa_to_fst(foobar, '^')
+# \endverbatim
+# will create again the original transducer [f:b o:a o:r].
+# @see hfst.fst_to_fsa
+def fsa_to_fst(fsa, separator=''):
+ pass
+
## Get a transducer that recognizes the concatenation of symbols or symbol pairs in \a arg.
# @param arg The symbols or symbol pairs that form the path to be recognized.
#
-# Example
+# Example
# \verbatim
-# import libhfst
-# tok = libhfst.HfstTokenizer()
+# import hfst
+# tok = hfst.HfstTokenizer()
# tok.add_multichar_symbol('foo')
# tok.add_multichar_symbol('bar')
-# tr = libhfst.tokenized_fst(tok.tokenize('foobar', 'foobaz'))
+# tr = hfst.tokenized_fst(tok.tokenize('foobar', 'foobaz'))
# \endverbatim
# will create the transducer [foo:foo bar:b 0:a 0:z]
def tokenized_fst(arg, weight=0):
@@ -503,82 +211,225 @@ def epsilon_fst(weight=0):
## Get a transducer as defined by regular expression \a regexp.
# @param regexp The regular expression defined with <a href="http://www.fsmbook.com/">Xerox transducer notation</a>.
-# @param kvargs Argumnets recognized are: error.
+# @param kwargs Arguments recognized are: error.
# @param error Where warnings and errors are printed. Possible values are sys.stdout, sys.stderr (the default), a StringIO or None, indicating a quiet mode.
-def regex(regexp, **kvargs):
+#
+# Regular expression operators:
+#
+# \verbatim
+#
+# ~ complement
+# \ term complement
+# & intersection
+# - minus
+#
+# $. contains once
+# $? contains optionally
+# $ contains once or more
+# ( ) optionality
+#
+# + Kleene plus
+# * Kleene star
+#
+# ./. ignore internally (not yet implemented)
+# / ignoring
+#
+# | union
+#
+# <> shuffle
+# < before
+# > after
+#
+# .o. composition
+# .O. lenient composition
+# .m>. merge right
+# .<m. merge left
+# .x. cross product
+# .P. input priority union
+# .p. output priority union
+# .-u. input minus
+# .-l. output minus
+# `[ ] substitute
+#
+# ^n,k catenate from n to k times, inclusive
+# ^>n catenate more than n times
+# ^>n catenate less than n times
+# ^n catenate n times
+#
+# .r reverse
+# .i invert
+# .u input side
+# .l output side
+#
+# \\\ left quotient
+#
+# Two-level rules:
+#
+# \<= left restriction
+# <=> left and right arrow
+# <= left arrow
+# => right arrow
+#
+# Replace rules:
+#
+# -> replace right
+# (->) optionally replace right
+# <- replace left
+# (<-) optionally replace left
+# <-> replace left and right
+# (<->) optionally replace left and right
+# @-> left-to-right longest match
+# @> left-to-right shortest match
+# ->@ right-to-left longest match
+# >@ right-to-left shortest match
+#
+# Rule contexts, markers and separators:
+#
+# || match contexts on input sides
+# // match left context on output side and right context on input side
+# \\ match left context on input side and right context on output side
+# \/ match contexts on output sides
+# _ center marker
+# ... markup marker
+# ,, rule separator in parallel rules
+# , context separator
+# [. .] match epsilons only once
+#
+# Read from file:
+#
+# @bin" " read binary transducer
+# @txt" " read transducer in att text format
+# @stxt" " read spaced text
+# @pl" " read transducer in prolog text format
+# @re" " read regular expression
+#
+# Symbols:
+#
+# .#. word boundary symbol in replacements, restrictions
+# 0 the epsilon
+# ? any token
+# % escape character
+# { } concatenate symbols
+# " " quote symbol
+#
+# : pair separator
+# :: weight
+#
+# ; end of expression
+# ! starts a comment until end of line
+# # starts a comment until end of line
+# \endverbatim
+def regex(regexp, **kwargs):
+ pass
+
+## Compile sfst file \a filename into a transducer.
+# @param filename The name of the sfst file.
+# @param kwargs Arguments recognized are: verbose, output.
+# @param verbose Whether sfst file is processed in verbose mode, defaults to False.
+# @param output TODO: Where output is printed. Possible values are sys.stdout, sys.stderr, a StringI0, sys.stderr being the default.
+# @return On success the resulting transducer, else None.
+def compile_sfst_file(filename, **kwargs):
pass
## Compile lexc file \a filename into a transducer.
# @param filename The name of the lexc file.
-# @param kvargs Arguments recognized are: verbosity, with_flags, output.
+# @param kwargs Arguments recognized are: verbosity, with_flags, output.
# @param verbosity The verbosity of the compiler, defaults to 0 (silent). Possible values are: 0, 1, 2.
# @param with_flags Whether lexc flags are used when compiling, defaults to False.
# @param output Where output is printed. Possible values are sys.stdout, sys.stderr, a StringIO, sys.stderr being the default?
-def compile_lexc_file(filename, **kvargs):
+# @return On success the resulting transducer, else None.
+def compile_lexc_file(filename, **kwargs):
pass
## Compile (is 'run' a better term?) xfst file \a filename.
# @param filename The name of the xfst file.
-# @param kvargs Arguments recognized are: verbosity, quit_on_fail, output, type.
+# @param kwargs Arguments recognized are: verbosity, quit_on_fail, output, type.
# @param verbosity The verbosity of the compiler, defaults to 0 (silent). Possible values are: 0, 1, 2.
# @param quit_on_fail Whether the script is exited on any error, defaults to True.
# @param output Where output is printed. Possible values are sys.stdout, sys.stderr, a StringIO, sys.stderr being the default?
-# @param type Implementation type of the compiler, defaults to libhfst.get_default_fst_type().
-def compile_xfst_file(filename, **kvargs):
+# @param type Implementation type of the compiler, defaults to hfst.get_default_fst_type().
+# @return On success 0, else an integer greater than 0.
+def compile_xfst_file(filename, **kwargs):
+ pass
+
+## Compile pmatch expressions as defined in \a filename and return a tuple of transducers.
+#
+# An example:
+#
+# If we have a file named streets.txt that contains:
+#
+# define CapWord UppercaseAlpha Alpha* ;
+# define StreetWordFr [{avenue} | {boulevard} | {rue}] ;
+# define DeFr [ [{de} | {du} | {des} | {de la}] Whitespace ] | [{d'} | {l'}] ;
+# define StreetFr StreetWordFr (Whitespace DeFr) CapWord+ ;
+# regex StreetFr EndTag(FrenchStreetName) ;
+#
+# we can run:
+#
+# defs = hfst.compile_pmatch_file('streets.txt')
+# const = hfst.PmatchContainer(defs)
+# assert cont.match("Je marche seul dans l'avenue des Ternes.") == "Je marche seul dans l'<FrenchStreetName>avenue des Ternes</FrenchStreetName>."
+def compile_pmatch_file(filename):
+ pass
+
+## Compile a pmatch expression into a tuple of transducers.
+# @param expr A string defining how pmatch is done.
+# @see hfst.compile_pmatch_file
+def compile_pmatch_expression(expr):
pass
## Start interactive xfst compiler.
-# @param kvargs Arguments recognized are: type, quit_on_fail.
+# @param kwargs Arguments recognized are: type, quit_on_fail.
# @param quit_on_fail Whether the compiler exits on any error, defaults to False.
-# @param type Implementation type of the compiler, defaults to libhfst.get_default_fst_type().
-def start_xfst(**kvargs):
+# @param type Implementation type of the compiler, defaults to hfst.get_default_fst_type().
+def start_xfst(**kwargs):
pass
## Read AT&T input from the user and return a transducer.
-# @return An HfstTransducer whose type is libhfst.get_default_fst_type().
-# Read one AT&T line at a time from standard input and create an equivalent transducer.
+# @return An HfstTransducer whose type is hfst.get_default_fst_type().
+# Read one AT&T line at a time from standard input and finally return an equivalent transducer.
# An empty line signals the end of input.
def read_att_input():
pass
## Read a multiline string \a att and return a transducer.
# @param att A string in AT&& format that defines the transducer.
-# @return An HfstTransducer whose type is libhfst.get_default_fst_type().
+# @return An HfstTransducer whose type is hfst.get_default_fst_type().
# Read \a att and create a transducer as defined in it.
def read_att_string(att):
pass
## Read next transducer from AT&T file pointed by \a f. \a epsilonstr defines the symbol used for epsilon in the file.
-# @param f A file pointer (#libhfst.HfstFile) to the file.
+# @param f A python file
# @param epsilonstr How epsilon is represented in the file. By default, "@_EPSILON_SYMBOL_@" and "@0@" are both recognized.
#
# If the file contains several transducers, they must be separated by "--" lines.
# In AT&T format, the transition lines are of the form:
- #
- # \verbatim
- # [0-9]+[\w]+[0-9]+[\w]+[^\w]+[\w]+[^\w]([\w]+(-)[0-9]+(\.[0-9]+))
+ #
+ # \verbatim
+ # [0-9]+[\w]+[0-9]+[\w]+[^\w]+[\w]+[^\w]([\w]+(-)[0-9]+(\.[0-9]+))
# \endverbatim
- #
+ #
# and final state lines:
- #
+ #
# \verbatim
# [0-9]+[\w]+([\w]+(-)[0-9]+(\.[0-9]+))
# \endverbatim
- #
- # If several transducers are listed in the same file, they are separated by lines of
- # two consecutive hyphens "--". If the weight
+ #
+ # If several transducers are listed in the same file, they are separated by lines of
+ # two consecutive hyphens "--". If the weight
# \verbatim
# ([\w]+(-)[0-9]+(\.[0-9]+))
# \endverbatim
# is missing, the transition or final state is given a zero weight.
- #
+ #
# NOTE: If transition symbols contains spaces, they must be escaped
# as '\@_SPACE_\@' because spaces are used as field separators.
# Both '\@0\@' and '\@_EPSILON_SYMBOL_\@' are always interpreted as
# epsilons.
- #
- #
- # An example:
+ #
+ #
+ # An example:
# \verbatim
# 0 1 foo bar 0.3
# 1 0.5
@@ -589,65 +440,178 @@ def read_att_string(att):
# 0 0.0
# 0 0 a <eps> 0.2
# \endverbatim
- #
- # The example lists four transducers in AT&T format:
+ #
+ # The example lists four transducers in AT&T format:
# one transducer accepting the string pair <'foo','bar'>, one
- # epsilon transducer, one empty transducer and one transducer
+ # epsilon transducer, one empty transducer and one transducer
# that accepts any number of 'a's and produces an empty string
- # in all cases. The transducers can be read with the following commands (from a file named
+ # in all cases. The transducers can be read with the following commands (from a file named
# 'testfile.att'):
# \verbatim
# transducers = []
- # ifile = libhfst.hfst_open('testfile2.att', 'r')
+ # ifile = open('testfile.att', 'r')
# try:
- # while (not ifile.is_eof()):
- # t = libhfst.read_att(ifile, '<eps>')
+ # while (True):
+ # t = hfst.read_att_transducer(ifile, '<eps>')
# transducers.append(t)
# print("read one transducer")
- # except libhfst.NotValidAttFormatException:
+ # except hfst.exceptions.NotValidAttFormatException as e:
# print("Error reading transducer: not valid AT&T format.")
+ # except hfst.exceptions.EndOfStreamException as e:
+ # pass
# ifile.close()
# print("Read %i transducers in total" % len(transducers))
# \endverbatim
- #
- # Epsilon will be represented as libhfst.EPSILON in the resulting transducer.
- # The argument \a epsilon_symbol only denotes how epsilons are represented
+ #
+ # Epsilon will be represented as hfst.EPSILON in the resulting transducer.
+ # The argument \a epsilon_symbol only denotes how epsilons are represented
# in \a ifile.
- #
- # @throws NotValidAttFormatException
+ #
+ # @bug Empty transducers are in theory represented as empty strings in AT&T format.
+ # However, this sometimes results in them getting interpreted as end-of-file.
+ # To avoid this, use an empty line instead, i.e. a single newline character.
+ #
+ # @throws NotValidAttFormatException
# @throws StreamNotReadableException
# @throws StreamIsClosedException
# @throws EndOfStreamException
# @see #write_att
-def read_att(f, epsilonstr=libhfst.EPSILON):
+def read_att_transducer(f, epsilonstr=hfst.EPSILON):
pass
-## Read next transducer from prolog file pointed by \a f.
-# @param f A file pointer (#libhfst.HfstFile) to the file.
+## A class for reading input in AT&T text format and converting it into transducer(s).
+#
+# An example that reads AT&T input from file 'testfile.att' where epsilon is represented as "<eps>"
+# and creates the corresponding transducers and prints them. If the input cannot be parsed, a message showing the
+# invalid line in AT&T input is printed and reading is stopped.
+#
+# \verbatim
+# with open('testfile.att', 'r') as f:
+# try:
+# r = hfst.AttReader(f, "<eps>")
+# for tr in r:
+# print(tr)
+# except hfst.exceptions.NotValidAttFormatException as e:
+# print(e.what())
+# \endverbatim
+class AttReader:
+ ## Create an AttReader that reads input from file \a f where the epsilon is represented as \a epsilonstr.
+ # @param f A python file.
+ # @param epsilonstr How epsilon is represented in the file. By default, "@_EPSILON_SYMBOL_@" and "@0@" are both recognized.
+ def __init__(self, f, epsilonstr=EPSILON):
+ pass
+ ## Read next transducer.
+ # Read next transducer description in AT&T format and return a corresponding transducer.
+ # @throws hfst.exceptions.NotValidAttFormatException
+ # @throws hfst.exceptions.EndOfStreamException
+ def read(self):
+ pass
+ ## An iterator to the reader.
+ # Needed for 'for ... in' statement.
+ # \verbatim
+ # for transducer in att_reader:
+ # print(transducer)
+ # \endverbatim
+ def __iter__(self):
+ pass
+ ## Return next element (for python version 2).
+ # Needed for 'for ... in' statement.
+ # \verbatim
+ # for transducer in att_reader:
+ # print(transducer)
+ # \endverbatim
+ # @throws StopIteration
+ def next(self):
+ pass
+ ## Return next element (for python version 3).
+ # Needed for 'for ... in' statement.
+ # \verbatim
+ # for transducer in att_reader:
+ # print(transducer)
+ # \endverbatim
+ # @throws StopIteration
+ def __next__(self):
+ pass
+
+# Read next transducer from prolog file pointed by \a f.
+# @param f A python file.
#
# If the file contains several transducers, they must be separated by empty lines.
-def read_prolog(f):
+def read_prolog_transducer(f):
pass
-## A simple transducer class with tropical weights.
+## A class for reading input in prolog text format and converting it into transducer(s).
#
-# An example of creating an HfstBasicTransducer [foo:bar baz:baz]
+# An example that reads prolog input from file 'testfile.prolog'
+# and creates the corresponding transducers and prints them. If the input cannot be parsed, a message showing the
+# invalid line in prolog input is printed and reading is stopped.
+#
+# \verbatim
+# with open('testfile.prolog', 'r') as f:
+# try:
+# r = hfst.PrologReader(f)
+# for tr in r:
+# print(tr)
+# except hfst.exceptions.NotValidPrologFormatException as e:
+# print(e.what())
+# \endverbatim
+class PrologReader:
+ ## Create a PrologReader that reads input from file \a f.
+ # @param f A python file.
+ def __init__(self, f):
+ pass
+ ## Read next transducer.
+ # Read next transducer description in prolog format and return a corresponding transducer.
+ # @throws hfst.exceptions.NotValidPrologFormatException
+ # @throws hfst.exceptions.EndOfStreamException
+ def read(self):
+ pass
+ ## An iterator to the reader.
+ # Needed for 'for ... in' statement.
+ # \verbatim
+ # for transducer in prolog_reader:
+ # print(transducer)
+ # \endverbatim
+ def __iter__(self):
+ pass
+ ## Return next element (for python version 2).
+ # Needed for 'for ... in' statement.
+ # \verbatim
+ # for transducer in prolog_reader:
+ # print(transducer)
+ # \endverbatim
+ # @throws StopIteration
+ def next(self):
+ pass
+ ## Return next element (for python version 3).
+ # Needed for 'for ... in' statement.
+ # \verbatim
+ # for transducer in prolog_reader:
+ # print(transducer)
+ # \endverbatim
+ # @throws StopIteration
+ def __next__(self):
+ pass
+
+## A simple transducer class with tropical weights.
+#
+# An example of creating an HfstBasicTransducer [foo:bar baz:baz]
# with weight 0.4 from scratch:
#
# \verbatim
# # Create an empty transducer
-# # The transducer has initially one start state (number zero)
+# # The transducer has initially one start state (number zero)
# # that is not final
-# fsm = libhfst.HfstBasicTransducer()
+# fsm = hfst.HfstBasicTransducer()
# # Add two states to the transducer
# fsm.add_state(1)
# fsm.add_state(2)
# # Create a transition [foo:bar] leading to state 1 with weight 0.1
-# tr = libhfst.HfstBasicTransition(1, 'foo', 'bar', 0.1)
+# tr = hfst.HfstBasicTransition(1, 'foo', 'bar', 0.1)
# # and add it to state zero
# fsm.add_transition(0, tr)
-# # Add a transition [baz:baz] with weight 0 from state 1 to state 2
-# fsm.add_transition(1, libhfst.HfstBasicTransition(2, 'baz', 'baz', 0.0))
+# # Add a transition [baz:baz] with weight 0 from state 1 to state 2
+# fsm.add_transition(1, hfst.HfstBasicTransition(2, 'baz', 'baz', 0.0))
# # Set state 2 as final with weight 0.3
# fsm.set_final_weight(2, 0.3)
# \endverbatim
@@ -665,7 +629,7 @@ def read_prolog(f):
# print('%i %f' % (state, fsm.get_final_weight(state)) )
# \endverbatim
#
-# @see #libhfst.HfstBasicTransition
+# @see #hfst.HfstBasicTransition
class HfstBasicTransducer:
# OMIT:
@@ -678,7 +642,7 @@ class HfstBasicTransducer:
# void write_in_prolog_format(std::ostream & os, const std::string & name, bool write_weights=true);
# static HfstBasicTransducer read_in_prolog_format(std::istream &is, FILE *file, unsigned int & linecount);
# static HfstBasicTransducer read_in_prolog_format(std::istream &is, unsigned int & linecount);
- # void write_in_att_format(std::ostream &os, bool write_weights=true);
+ # void write_in_att_format(std::ostream &os, bool write_weights=true);
# void write_in_att_format_number(FILE *file, bool write_weights=true);
# static HfstBasicTransducer read_in_att_format(std::istream &is, FILE *file, std::string epsilon_symbol, unsigned int & linecount);
# static HfstBasicTransducer read_in_att_format(std::istream &is, std::string epsilon_symbol, unsigned int & linecount);
@@ -689,20 +653,20 @@ class HfstBasicTransducer:
# HfstBasicTransducer &substitute_symbols(const HfstSymbolSubstitutions &substitutions);
# HfstBasicTransducer &substitute_symbol_pairs(const HfstSymbolPairSubstitutions &substitutions);
# HfstBasicTransducer &substitute(const HfstSymbolPair &sp, const HfstSymbolPairSet &sps);
- # HfstBasicTransducer &substitute(const HfstSymbolPair &old_pair, const HfstSymbolPair &new_pair);
+ # HfstBasicTransducer &substitute(const HfstSymbolPair &old_pair, const HfstSymbolPair &new_pair);
# HfstBasicTransducer &substitute(const HfstSymbolPair &sp, const HfstBasicTransducer &graph);
# TODO?:
# static HfstBasicTransducer intersect(HfstBasicTransducer & graph1, HfstBasicTransducer & graph2);
# HfstBasicTransducer &complete();
# std::vector<std::set<HfstState> > topsort(SortDistance dist) const;
- # std::vector<unsigned int> path_sizes();
+ # std::vector<unsigned int> path_sizes();
# bool is_lookup_infinitely_ambiguous(const HfstOneLevelPath & s);
- # bool is_lookup_infinitely_ambiguous(const StringVector & s);
+ # bool is_lookup_infinitely_ambiguous(const StringVector & s);
# void insert_transducer(HfstState state1, HfstState state2, const HfstBasicTransducer & graph);
## The length of the longest path in transducer.
- #
+ #
# Length of a path means number of arcs on that path.
def longest_path_size(self):
pass
@@ -723,15 +687,15 @@ class HfstBasicTransducer:
def is_lookup_infinitely_ambiguous(self, str):
pass
- ## Lookup tokenized input \a input in the transducer minding flag diacritics.
# @param str A list/tuple of strings to look up.
- # @param kvargs infinite_cutoff=-1, max_weight=None
- # @param infinite_cutoff Defaults to -1, i.e. infinite.
- # @param max_weight Defaults to None, i.e. infinity.
- def lookup_fd(self, input, **kvargs):
+ # @param kwargs max_epsilon_loops=-1, max_weight=None, obey_flags=False
+ # @param max_epsilon_loops How many times epsilon input loops are followed. Defaults to -1, i.e. infinitely.
+ # @param max_weight What is the maximum weight of a result allowed. Defaults to None, i.e. infinity.
+ # @param obey_flags Whether flag diacritic constraints are obeyed. Defaults to False.
+ def lookup(self, input, **kwargs):
pass
- ## Add a new state to this transducer and return its number.
+ ## Add a new state to this transducer and return its number.
# @return The next (smallest) free state number.
def add_state(self):
pass
@@ -746,24 +710,24 @@ class HfstBasicTransducer:
def add_state(self, state):
pass
- ## The states of the transducer
+ ## The states of the transducer
# @return A tuple of state numbers.
#
# An example:
- # /verbatim
+ # \verbatim
# for state in fsm.states():
# for arc in fsm.transitions(state):
# print('%i ' % (state), end='')
# print(arc)
# if fsm.is_final_state(state):
# print('%i %f' % (state, fsm.get_final_weight(state)) )
- # /endverbatim
+ # \endverbatim
def states(self):
- pass
+ pass
## The states and transitions of the transducer.
# @return A tuple of tuples of HfstBasicTransitions.
- # @see libhfst.HfstBasicTransducer.__enumerate__
+ # @see hfst.HfstBasicTransducer.__enumerate__
def states_and_transitions(self):
pass
@@ -782,7 +746,7 @@ class HfstBasicTransducer:
## Add a transition \a transition to state \a state, \a add_symbols_to_alphabet defines whether the transition symbols are added to the alphabet.
# @param state The number of the state where the transition is added. If it does not exist, it is created.
- # @param transition A libhfst.HfstBasicTransition that is added to \a state.
+ # @param transition A hfst.HfstBasicTransition that is added to \a state.
# @param add_symbols_to_alphabet Whether the transition symbols are added to the alphabet of the transducer. (In special cases this is not wanted.)
def add_transition(self, state, transition, add_symbols_to_alphabet=True):
pass
@@ -799,7 +763,7 @@ class HfstBasicTransducer:
## Remove transition \a transition from state \a s.
# @param s The state which \a transition belongs to.
# @param transition The transition to be removed.
- # @param remove_symbols_from_alphabet (?)
+ # @param remove_symbols_from_alphabet Whether
def remove_transition(self, s, transition, remove_symbols_from_alphabet=False):
pass
@@ -810,11 +774,11 @@ class HfstBasicTransducer:
#
# There is no way to test whether a graph is a trie, so the use
# of this function is probably limited to fast construction of a lexicon.
- # Here is an example:
- #
+ # Here is an example:
+ #
# \verbatim
- # lexicon = libhfst.HfstBasicTransducer()
- # tok = libhfst.HfstTokenizer()
+ # lexicon = hfst.HfstBasicTransducer()
+ # tok = hfst.HfstTokenizer()
# lexicon.disjunct(tok.tokenize('dog'), 0.3)
# lexicon.disjunct(tok.tokenize('cat'), 0.5)
# lexicon.disjunct(tok.tokenize('elephant'), 1.6)
@@ -822,8 +786,8 @@ class HfstBasicTransducer:
def disjunct(self, stringpairpath, weight):
pass
- ## The symbols in the alphabet of the transducer.
- # The symbols do not necessarily occur in any transitions of the transducer.
+ ## The symbols in the alphabet of the transducer.
+ # The symbols do not necessarily occur in any transitions of the transducer.
# Epsilon, unknown and identity symbols are always included in the alphabet.
# @return A tuple of strings.
def get_alphabet(self):
@@ -831,11 +795,11 @@ class HfstBasicTransducer:
## Get the final weight of state \a state in this transducer.
# @param state The number of the state. If it does not exist, a StateIsNotFinalException is thrown.
- # @throws libhfst.StateIsNotFinalException.
+ # @throws hfst.exceptions.StateIsNotFinalException.
def get_final_weight(self, state):
pass
- ## Get the biggest state number in use.
+ ## Get the biggest state number in use.
# @return The biggest state number in use.
def get_max_state(self):
pass
@@ -857,8 +821,8 @@ class HfstBasicTransducer:
# \endverbatim
# when harmonized.
#
- # The symbol '?' means libhfst.UNKNOWN in either or both sides of a transition
- # (transitions of type [?:x], [x:?] and [?:?]). The transition [?] means libhfst.IDENTITY.
+ # The symbol '?' means hfst.UNKNOWN in either or both sides of a transition
+ # (transitions of type [?:x], [x:?] and [?:?]). The transition [?] means hfst.IDENTITY.
#
# @note This function is always called for all transducer arguments of functions
# that take two or more graphs as their arguments, unless otherwise said.
@@ -866,11 +830,21 @@ class HfstBasicTransducer:
pass
## Create a transducer with one initial state that has state number zero and is not a final state, i.e. create an empty transducer.
+ #
+ # \verbatim
+ # tr = hfst.HfstBasicTransducer()
+ # \endverbatim
def __init__(self):
pass
## Create a transducer equivalent to \a transducer.
- # @param transducer The transducer to be copied, #libhfst.HfstBasicTransducer or #libhfst.HfstTransducer.
+ # @param transducer The transducer to be copied, #hfst.HfstBasicTransducer or #hfst.HfstTransducer.
+ #
+ # \verbatim
+ # tr = hfst.regex('foo') # creates an HfstTransducer
+ # TR = hfst.HfstBasicTransducer(tr)
+ # TR2 = hfst.HfstBasicTransducer(TR)
+ # \endverbatim
def __init__(self, transducer):
pass
@@ -895,7 +869,7 @@ class HfstBasicTransducer:
pass
## Write this transducer in AT&T format to file \a f, \a write_weights defines whether weights are written.
- def write_att(f, bool write_weights=True):
+ def write_att(self, f, bool write_weights=True):
pass
## Insert freely any number of \a symbol_pair in the transducer with weight \a weight.
@@ -909,12 +883,12 @@ class HfstBasicTransducer:
def insert_freely(self, transducer):
pass
- ## Whether state \a state is final.
+ ## Whether state \a state is final.
# @param state The state whose finality is returned.
def is_final_state(self, state):
pass
- ## Get the transitions of state \a state in this transducer.
+ ## Get the transitions of state \a state in this transducer.
# If the state does not exist, a @a StateIndexOutOfBoundsException is thrown.
# @return A tuple of HfstBasicTransitions.
#
@@ -929,32 +903,32 @@ class HfstBasicTransducer:
def transitions(self, state):
pass
- ## Remove all symbols that do not occur in transitions of the transducer from its alphabet.
+ ## Remove all symbols that do not occur in transitions of the transducer from its alphabet.
# Epsilon, unknown and identity symbols are always included in the alphabet.
def prune_alphabet(self):
pass
- ## Get a list of all symbols used in the transitions of this transducer.
+ ## Get a list of all symbols used in the transitions of this transducer.
def symbols_used(self):
pass
- ## Get a list of all input/output symbol pairs used in the transitions of this transducer.
+ ## Get a list of all input/output symbol pairs used in the transitions of this transducer.
def get_transition_pairs(self):
pass
- ## Remove symbol \a symbol from the alphabet of the graph.
+ ## Remove symbol \a symbol from the alphabet of the graph.
# @note Use with care, removing symbols that occur in the transitions of the graph can have unexpected results.
# @param symbol The string to be removed.
def remove_symbol_from_alphabet(self, symbol):
pass
- ## Remove symbols \a symbols from the alphabet of the graph.
+ ## Remove symbols \a symbols from the alphabet of the graph.
# @note Use with care, removing symbols that occur in the transitions of the graph can have unexpected results.
# @param symbols A tuple of strings to be removed.
def remove_symbols_from_alphabet(self, symbols):
pass
- ## Set the final weight of state \a state in this transducer to \a weight.
+ ## Set the final weight of state \a state in this transducer to \a weight.
# If the state does not exist, it is created.
def set_final_weight(self, state, weight):
pass
@@ -967,8 +941,8 @@ class HfstBasicTransducer:
## Substitute symbols or transitions in the transducer.
#
# @param s The symbol or transition to be substituted. Can also be a dictionary of substitutions, if S == None.
- # @param S The symbol, transition, a tuple of transitions or a transducer (libhfst.HfstBasicTransducer) that substitutes \a s.
- # @param kvargs Arguments recognized are 'input' and 'output', their values can be False or True, True being the default.
+ # @param S The symbol, transition, a tuple of transitions or a transducer (hfst.HfstBasicTransducer) that substitutes \a s.
+ # @param kwargs Arguments recognized are 'input' and 'output', their values can be False or True, True being the default.
# These arguments are valid only if \a s and \a S are strings, else they are ignored.
# @param input Whether substitution is performed on input side, defaults to True. Valid only if \a s and \a S are strings.
# @param output Whether substitution is performed on output side, defaults to True. Valid only if \a s and \a S are strings.
@@ -982,23 +956,23 @@ class HfstBasicTransducer:
# (5) substitute(dict): perform several symbol-to-symbol substitutions
# (6) substitute(dict): perform several transition-to-transition substitutions
#
- # Examples:
+ # Examples:
#
# (1) tr.substitute('a', 'A', input=True, output=False): substitute lowercase a:s with uppercase ones
- # (2) tr.substitute(('a','b'),('A','B')): substitute transitions that map lowercase a into lowercase b with transitions that map uppercase a into uppercase b
+ # (2) tr.substitute(('a','b'),('A','B')): substitute transitions that map lowercase a into lowercase b with transitions that map uppercase a into uppercase b
# (3) tr.substitute(('a','b'), (('A','B'),('a','B'),('A','b'))): change either or both sides of a transition [a:b] to uppercase
- # (4) tr.substitute(('a','b'), libhfst.regex('[a:b]+')) change [a:b] transition into one or more consecutive [a:b] transitions
+ # (4) tr.substitute(('a','b'), hfst.regex('[a:b]+')) change [a:b] transition into one or more consecutive [a:b] transitions
# (5) tr.substitute({'a':'A', 'b':'B', 'c':'C'}) change lowercase a, b and c into their uppercase variants
# (6) tr.substitute( {('a','a'):('A','A'), ('b','b'):('B','B'), ('c','c'):('C','C')} ): change lowercase a, b and c into their uppercase variants
#
# In case (4), epsilon transitions are used to attach copies of transducer \a S between the SOURCE and TARGET state of each transition that is substituted.
# The transition itself is deleted, but its weight is copied to the epsilon transition leading from SOURCE to the initial state of \a S.
# Each final state of \a S is made non-final and an epsilon transition leading to TARGET is attached to it. The final weight is copied to the epsilon transition.
- #
- def substitute(self, s, S=None, **kvargs):
+ #
+ def substitute(self, s, S=None, **kwargs):
pass
- ## Return an enumeration of the states and transitions of the transducer.
+ ## Return an enumeration of the states and transitions of the transducer.
#
# \verbatim
# for state, arcs in enumerate(fsm):
@@ -1020,7 +994,7 @@ class HfstBasicTransducer:
pass
## A transition class that consists of a target state, input and output symbols and a a tropical weight.
-# @see libhfst.HfstBasicTransducer
+# @see hfst.HfstBasicTransducer
class HfstBasicTransition:
## Create an HfstBasicTransition leading to target state \a state with input symbol \a input, output symbol \a output and weight \a weight.
@@ -1029,6 +1003,10 @@ class HfstBasicTransition:
# @param output The output string.
# @param weight The weight.
# @throws EmptyStringException
+ #
+ # \verbatim
+ # transition = hfst.HfstBasicTransition(1, 'foo', 'bar', 0.5)
+ # \endverbatim
def __init__(self, state, input, output, weight):
pass
@@ -1048,31 +1026,36 @@ class HfstBasicTransition:
def get_weight(self):
pass
+ ## Set the weight of the transition to \a weight.
+ # @param weight The weight.
+ def set_weight(self, weight):
+ pass
+
## A string representation of the transition.
#
# @return "target_state input_symbol output_symbol, weight"
def __str__(self)
## A synchronous finite-state transducer.
-#
+#
# \section argument_handling Argument handling
-#
-# Transducer functions modify their calling object and return
-# a reference to the calling object after modification,
+#
+# Transducer functions modify their calling object and return
+# a reference to the calling object after modification,
# unless otherwise mentioned.
# Transducer arguments are usually not modified.
# \verbatim
# # transducer is reversed
# transducer.reverse()
# # transducer2 is not modified, but a copy of it is disjuncted with
-# # transducer1
-# transducer1.disjunct(transducer2)
+# # transducer1
+# transducer1.disjunct(transducer2)
# # a chain of functions is possible
-# transducer.reverse().determinize().reverse().determinize()
+# transducer.reverse().determinize().reverse().determinize()
# \endverbatim
-#
+#
# \section implementation_types Implementation types
-#
+#
# Currently, an HfstTransducer has three implementation types that are well supported.
# When an HfstTransducer is created, its type is defined with an argument.
# For functions that take a transducer as an argument, the type of the calling transducer
@@ -1081,16 +1064,16 @@ class HfstBasicTransition:
# # this will cause a TransducerTypeMismatchException:
# tropical_transducer.disjunct(foma_transducer)
# # this works, but weights are lost in the conversion
-# tropical_transducer.convert(libhfst.SFST_TYPE).disjunct(sfst_transducer)
+# tropical_transducer.convert(hfst.ImplementationType.SFST_TYPE).disjunct(sfst_transducer)
# # this works, information is not lost
-# tropical_transducer.disjunct(sfst_transducer.convert(libhfst.TROPICAL_OPENFST_TYPE))
+# tropical_transducer.disjunct(sfst_transducer.convert(hfst.ImplementationType.TROPICAL_OPENFST_TYPE))
# \endverbatim
-#
+#
# \section creating_transducers Creating transducers
-#
-# With HfstTransducer constructors it is possible to create empty,
+#
+# With HfstTransducer constructors it is possible to create empty,
# epsilon, one-transition and single-path transducers.
-# Transducers can also be created from scratch with #libhfst.HfstBasicTransducer
+# Transducers can also be created from scratch with #hfst.HfstBasicTransducer
# and converted to an HfstTransducer.
# More complex transducers can be combined from simple ones with various functions.
class HfstTransducer:
@@ -1099,12 +1082,12 @@ class HfstTransducer:
def is_implementation_type_available(type):
pass
- ## Create an uninitialized transducer (use with care).
- #
- # @note This constructor leaves the backend implementation variable
- # uninitialized. An uninitialized transducer is likely to cause a
- # TransducerHasWrongTypeException at some point unless it is given
- # a value at some point.
+ ## Create an empty transducer.
+ #
+ # \verbatim
+ # tr = hfst.HfstTransducer()
+ # assert(tr.compare(hfst.empty_fst()))
+ # \endverbatim
def __init__(self):
pass
@@ -1114,8 +1097,8 @@ class HfstTransducer:
# An example:
#
# \verbatim
- # tr1 = libhfst.regex('foo bar foo')
- # tr2 = libhfst.HfstTransducer(tr1)
+ # tr1 = hfst.regex('foo bar foo')
+ # tr2 = hfst.HfstTransducer(tr1)
# tr2.substitute('foo','FOO')
# tr1.concatenate(tr2)
# \endverbatim
@@ -1126,25 +1109,35 @@ class HfstTransducer:
# @param t An HfstBasicTransducer.
# @param type The type of the resulting transducer.
# If you want to use the default type, you can just call
- # libhfst.HfstTransducer(fsm)
+ # hfst.HfstTransducer(fsm)
def __init__(self, t, type):
pass
+ ## Return a deep copy of the transducer.
+ #
+ # \verbatim
+ # tr = hfst.regex('[foo:bar::0.3]*')
+ # TR = tr.copy()
+ # assert(tr.compare(TR))
+ # \endverbatim
+ def copy(self):
+ pass
+
## Rename the transducer \a name.
# @param name The name of the transducer.
- # @see #get_name
+ # @see #get_name
def set_name(self, name):
pass
- ## Get the name of the transducer.
- # @see #set_name
+ ## Get the name of the transducer.
+ # @see #set_name
def get_name(self):
pass
## An AT&T representation of the transducer.
# Defined for print command. An example:
# \verbatim
- # >>> print(libhfst.regex('[foo:bar::2]+'))
+ # >>> print(hfst.regex('[foo:bar::2]+'))
# 0 1 foo bar 2.000000
# 1 1 foo bar 2.000000
# 1 0.000000
@@ -1181,27 +1174,27 @@ class HfstTransducer:
def get_properties(self):
pass
- ## Get the alphabet of the transducer.
- #
- # The alphabet is defined as the set of symbols known to the transducer.
+ ## Get the alphabet of the transducer.
+ #
+ # The alphabet is defined as the set of symbols known to the transducer.
# @return A tuple of strings.
def get_alphabet(self):
pass
- ## Explicitly insert \a symbol to the alphabet of the transducer.
+ ## Explicitly insert \a symbol to the alphabet of the transducer.
# @param symbol The symbol (string) to be inserted.
#
# @note Usually this function is not needed since new symbols are
- # added to the alphabet by default.
+ # added to the alphabet by default.
def insert_to_alphabet(self, symbol):
pass
## Remove \a symbol from the alphabet of the transducer.
# @param symbol The symbol (string) to be removed.
- #
+ #
# @pre \a symbol does not occur in any transition of the transducer.
# @note Use with care, removing a symbol that occurs in a transition
- # of the transducer can have unexpected results.
+ # of the transducer can have unexpected results.
def remove_from_alphabet(self, symbol):
pass
@@ -1220,8 +1213,8 @@ class HfstTransducer:
pass
## Whether each transition in the transducer has equivalent input and output symbols.
- # @note Transition with libhfst.UNKNOWN on both sides IS NOT a transition with equivalent input and output symbols.
- # @note Transition with libhfst.IDENTITY on both sides IS a transition with equivalent input and output symbols.
+ # @note Transition with hfst.UNKNOWN on both sides IS NOT a transition with equivalent input and output symbols.
+ # @note Transition with hfst.IDENTITY on both sides IS a transition with equivalent input and output symbols.
def is_automaton(self):
pass
@@ -1230,30 +1223,30 @@ class HfstTransducer:
pass
## The implementation type of the transducer.
- # @return libhfst.ImplementationType
+ # @return hfst.ImplementationType
def get_type(self):
pass
## Whether this transducer and \a another are equivalent.
# @param another The compared transducer.
# @pre \a self and \a another must have the same implementation type.
- #
+ #
# Two transducers are equivalent iff they accept the same input/output
- # string pairs with the same weights and the same alignments.
+ # string pairs with the same weights and the same alignments.
# @note For weighted transducers, the function often returns false negatives
- # due to weight precision issues.
+ # due to weight precision issues.
def compare(self, another):
pass
- ## Remove all <i>epsilon:epsilon</i> transitions from the transducer so that the resulting transducer is equivalent to the original one.
+ ## Remove all <i>epsilon:epsilon</i> transitions from the transducer so that the resulting transducer is equivalent to the original one.
def remove_epsilons(self):
pass
## Determinize the transducer.
- #
+ #
# Determinizing a transducer yields an equivalent transducer that has
# no state with two or more transitions whose input:output symbol
- # pairs are the same.
+ # pairs are the same.
def determinize(self):
pass
@@ -1266,58 +1259,58 @@ class HfstTransducer:
pass
## Write the transducer in binary format to \a ostr.
- # @param ostr A libhfst.HfstOutputStream where the transducer is written.
+ # @param ostr A hfst.HfstOutputStream where the transducer is written.
def write(self, ostr):
pass
## Write the transducer in AT&T format to file \a f, \a write_weights defined whether weights are written.
- # @param f An libhfst.HfstFile where the transducer is written.
+ # @param f A python file where transducer is written.
# @param write_weights Whether weights are written.
def write_att(self, f, write_weights=True):
pass
## Write the transducer in prolog format with name \a name to file \a f, \a write_weights defined whether weights are written.
- # @param f An libhfst.HfstFile where the transducer is written.
+ # @param f A python file where the transducer is written.
# @param name The name of the transducer that must be given in a prolog file.
# @param write_weights Whether weights are written.
- def write_prolog(f, name, write_weights=True):
+ def write_prolog(self, f, name, write_weights=True):
pass
## Minimize the transducer.
- #
- # Minimizing a transducer yields an equivalent transducer with
- # the smallest number of states.
- #
- # @bug OpenFst's minimization algorithm seems to add epsilon transitions to weighted transducers?
+ #
+ # Minimizing a transducer yields an equivalent transducer with
+ # the smallest number of states.
+ #
+ # @bug OpenFst's minimization algorithm seems to add epsilon transitions to weighted transducers?
def minimize(self):
pass
- ## Extract \a n best paths of the transducer.
- #
- # In the case of a weighted transducer (#TROPICAL_OPENFST_TYPE or
- # #LOG_OPENFST_TYPE), best paths are defined as paths with
+ ## Extract \a n best paths of the transducer.
+ #
+ # In the case of a weighted transducer (#hfst.ImplementationType.TROPICAL_OPENFST_TYPE or
+ # #hfst.ImplementationType.LOG_OPENFST_TYPE), best paths are defined as paths with
# the lowest weight.
- # In the case of an unweighted transducer (#SFST_TYPE or #FOMA_TYPE),
+ # In the case of an unweighted transducer (#hfst.ImplementationType.SFST_TYPE or #hfst.ImplementationType.FOMA_TYPE),
# the function returns random paths.
- #
- # This function is not implemented for #FOMA_TYPE or #SFST_TYPE.
- # If this function is called by an HfstTransducer of type #FOMA_TYPE
- # or #SFST_TYPE, it is converted to #TROPICAL_OPENFST_TYPE,
- # paths are extracted and it is converted back to #FOMA_TYPE or
- # #SFST_TYPE. If HFST is not linked to OpenFst library, an
- # ImplementationTypeNotAvailableException is thrown.
+ #
+ # This function is not implemented for #hfst.ImplementationType.FOMA_TYPE or #hfst.ImplementationType.SFST_TYPE.
+ # If this function is called by an HfstTransducer of type #hfst.ImplementationType.FOMA_TYPE
+ # or #hfst.ImplementationType.SFST_TYPE, it is converted to #hfst.ImplementationType.TROPICAL_OPENFST_TYPE,
+ # paths are extracted and it is converted back to #hfst.ImplementationType.FOMA_TYPE or
+ # #hfst.ImplementationType.SFST_TYPE. If HFST is not linked to OpenFst library, an
+ # #hfst.exceptions.ImplementationTypeNotAvailableException is thrown.
def n_best(self, n):
pass
- ## A concatenation of N transducers where N is any number from zero to infinity.
+ ## A concatenation of N transducers where N is any number from zero to infinity.
def repeat_star(self):
pass
- ## A concatenation of N transducers where N is any number from one to infinity.
+ ## A concatenation of N transducers where N is any number from one to infinity.
def repeat_plus(self):
pass
- ## A concatenation of \a n transducers.
+ ## A concatenation of \a n transducers.
def repeat_n(self, n):
pass
@@ -1333,33 +1326,33 @@ class HfstTransducer:
def repeat_n_to_k(self, n, k):
pass
- ## Disjunct the transducer with an epsilon transducer.
+ ## Disjunct the transducer with an epsilon transducer.
def optionalize(self):
pass
- ## Swap the input and output symbols of each transition in the transducer.
+ ## Swap the input and output symbols of each transition in the transducer.
def invert(self):
pass
- ## Reverse the transducer.
- #
- # A reverted transducer accepts the string 'n(0) n(1) ... n(N)'
+ ## Reverse the transducer.
+ #
+ # A reverted transducer accepts the string 'n(0) n(1) ... n(N)'
# iff the original
- # transducer accepts the string 'n(N) n(N-1) ... n(0)'
+ # transducer accepts the string 'n(N) n(N-1) ... n(0)'
def reverse(self):
pass
- ## Extract the input language of the transducer.
- #
- # All transition symbol pairs <i>isymbol:osymbol</i> are changed
- # to <i>isymbol:isymbol</i>.
+ ## Extract the input language of the transducer.
+ #
+ # All transition symbol pairs <i>isymbol:osymbol</i> are changed
+ # to <i>isymbol:isymbol</i>.
def input_project(self):
pass
## Extract the output language of the transducer.
- #
- # All transition symbol pairs <i>isymbol:osymbol</i> are changed
- # to <i>osymbol:osymbol</i>.
+ #
+ # All transition symbol pairs <i>isymbol:osymbol</i> are changed
+ # to <i>osymbol:osymbol</i>.
def output_project(self):
pass
@@ -1376,85 +1369,85 @@ class HfstTransducer:
## Compose this transducer with the intersection of
# transducers in \a v. If \a invert is true, then compose the
# intersection of the transducers in \a v with this transducer.
- #
- # The algorithm used by this function is faster than intersecting
- # all transducers one by one and then composing this transducer
- # with the intersection.
- #
+ #
+ # The algorithm used by this function is faster than intersecting
+ # all transducers one by one and then composing this transducer
+ # with the intersection.
+ #
# @pre The transducers in \a v are deterministic and epsilon-free.
# @param v A tuple of transducers.
# @param invert Whether the intersection of the transducers in \a v is composed with this transducer.
def compose_intersect(self, v, invert=False):
pass
- ## Concatenate this transducer with \a another.
+ ## Concatenate this transducer with \a another.
def concatenate(self, another):
pass
- ## Disjunct this transducer with \a another.
+ ## Disjunct this transducer with \a another.
def disjunct(self, another):
pass
- ## Intersect this transducer with \a another.
+ ## Intersect this transducer with \a another.
def intersect(self, another):
pass
- ## Subtract transducer \a another from this transducer.
+ ## Subtract transducer \a another from this transducer.
def subtract(self, another):
pass
## Alias for subtract.
- # @see libhfst.HfstTransducer.subtract
+ # @see hfst.HfstTransducer.subtract
def minus(self, another):
pass
## Alias for intersect.
- # @see libhfst.HfstTransducer.intersect
+ # @see hfst.HfstTransducer.intersect
def conjunct(self, another):
pass
- ## Convert the transducer into an equivalent transducer in format \a type.
- #
- # If a weighted transducer is converted into an unweighted one,
- # all weights are lost. In the reverse case, all weights are initialized to the
- # semiring's one.
- #
- # A transducer of type #libhfst.SFST_TYPE, #libhfst.TROPICAL_OPENFST_TYPE,
- # #libhfst.LOG_OPENFST_TYPE or #libhfst.FOMA_TYPE can be converted into an
- # #libhfst.HFST_OL_TYPE or #libhfst.HFST_OLW_TYPE transducer, but an #libhfst.HFST_OL_TYPE
- # or #libhfst.HFST_OLW_TYPE transducer cannot be converted to any other type.
- #
+ ## Convert the transducer into an equivalent transducer in format \a type.
+ #
+ # If a weighted transducer is converted into an unweighted one,
+ # all weights are lost. In the reverse case, all weights are initialized to the
+ # semiring's one.
+ #
+ # A transducer of type #hfst.ImplementationType.SFST_TYPE, #hfst.ImplementationType.TROPICAL_OPENFST_TYPE,
+ # #hfst.ImplementationType.LOG_OPENFST_TYPE or #hfst.ImplementationType.FOMA_TYPE can be converted into an
+ # #hfst.ImplementationType.HFST_OL_TYPE or #hfst.ImplementationType.HFST_OLW_TYPE transducer, but an #hfst.ImplementationType.HFST_OL_TYPE
+ # or #hfst.ImplementationType.HFST_OLW_TYPE transducer cannot be converted to any other type.
+ #
# @note For conversion between HfstBasicTransducer and HfstTransducer,
- # see #libhfst.HfstTransducer.__init__ and #libhfst.HfstBasicTransducer.__init__
+ # see #hfst.HfstTransducer.__init__ and #hfst.HfstBasicTransducer.__init__
def convert(self, type, options=''):
pass
- ## Write the transducer in AT&T format to file \a ofile, \a write_weights defines whether weights are written.
- #
+ ## Write the transducer in AT&T format to file \a ofile, \a write_weights defines whether weights are written.
+ #
# The fields in the resulting AT&T format are separated by tabulator characters.
- #
+ #
# NOTE: If the transition symbols contain space characters,the spaces are printed as '\@_SPACE_\@' because
# whitespace characters are used as field separators in AT&T format. Epsilon symbols are printed as '\@0\@'.
- #
+ #
# If several transducers are written in the same file, they must be separated by a line of two consecutive hyphens "--", so that
- # they will be read correctly by libhfst.read_att.
- #
+ # they will be read correctly by hfst.read_att.
+ #
# An example:
# \verbatim
- # tr1 = libhfst.regex('[foo:bar baz:0 " "]::0.3')
- # tr2 = libhfst.empty_fst()
- # tr3 = libhfst.epsilon_fst(0.5)
- # tr4 = libhfst.regex('[foo]')
- # tr5 = libhfst.empty_fst()
- #
- # f = libhfst.hfst_open('testfile.att', 'w')
+ # tr1 = hfst.regex('[foo:bar baz:0 " "]::0.3')
+ # tr2 = hfst.empty_fst()
+ # tr3 = hfst.epsilon_fst(0.5)
+ # tr4 = hfst.regex('[foo]')
+ # tr5 = hfst.empty_fst()
+ #
+ # f = hfst.hfst_open('testfile.att', 'w')
# for tr in [tr1, tr2, tr3, tr4]:
# tr.write_att(f)
# f.write('--\n')
# tr5.write_att(f)
# f.close()
# \endverbatim
- #
+ #
# This will yield a file 'testfile.att' that looks as follows:
# \verbatim
# 0 1 foo bar 0.299805
@@ -1469,18 +1462,18 @@ class HfstTransducer:
# 1 0.000000
# --
# \endverbatim
- #
- # @throws StreamCannotBeWrittenException
+ #
+ # @throws StreamCannotBeWrittenException
# @throws StreamIsClosedException
- #
- # @see #libhfst.HfstOutputStream.write
- # @see #libhfst.HfstTransducer.__init__
+ #
+ # @see #hfst.HfstOutputStream.write
+ # @see #hfst.HfstTransducer.__init__
def write_att(self, ofile, write_weights=True):
pass
## Write the transducer in AT&T format to file named \a filename. \a write_weights defines whether weights are written.
- #
- # If the file exists, it is overwritten. If the file does not exist, it is created.
+ #
+ # If the file exists, it is overwritten. If the file does not exist, it is created.
def write_att(self, filename, write_weights=True):
pass
@@ -1489,24 +1482,24 @@ class HfstTransducer:
# For the operation t1.priority_union(t2), the result is a union of t1 and t2,
# except that whenever t1 and t2 have the same string on left side,
# the path in t2 overrides the path in t1.
- #
+ #
# Example
# \verbatim
# Transducer 1 (t1):
# a : a
# b : b
- #
+ #
# Transducer 2 (t2):
# b : B
# c : C
- #
+ #
# Result ( t1.priority_union(t2) ):
# a : a
# b : B
# c : C
# \endverbatim
# For more information, read <a href="http://www.fsmbook.com/">fsmbook</a>.
- #
+ #
def priority_union(self, another):
pass
@@ -1517,120 +1510,141 @@ class HfstTransducer:
pass
## Shuffle this transducer with transducer \a another.
- #
+ #
# If transducer A accepts string 'foo' and transducer B string 'bar',
# the transducer that results from shuffling A and B accepts all strings
# [(f|b)(o|a)(o|r)].
- #
+ #
# @pre Both transducers must be automata, i.e. map strings onto themselves.
def shuffle(self, another):
pass
## Freely insert a transition or a transducer into the transducer.
# @param ins The transition or transducer to be inserted.
- #
+ #
# If \a ins is a transition, i.e. a 2-tuple of strings: A transition is added to each state in this transducer.
# The transition leads from that state to itself with input and output symbols defined by \a ins.
# The weight of the transition is zero.
#
- # If \a ins is an #libhfst.HfstTransducer:
- # A copy of \a ins is attached with epsilon transitions
- # to each state of this transducer. After the operation, for each
- # state S in this transducer, there is an epsilon transition
- # that leads from state S to the initial state of \a ins,
+ # If \a ins is an #hfst.HfstTransducer:
+ # A copy of \a ins is attached with epsilon transitions
+ # to each state of this transducer. After the operation, for each
+ # state S in this transducer, there is an epsilon transition
+ # that leads from state S to the initial state of \a ins,
# and for each final state of \a ins, there is an epsilon transition
# that leads from that final state to state S in this transducer.
- # The weights of the final states in \a ins are copied to the
+ # The weights of the final states in \a ins are copied to the
# epsilon transitions leading to state S.
def insert_freely(self, ins):
pass
- ## Set the weights of all final states to \a weight.
- # If the HfstTransducer is of unweighted type (#libhfst.SFST_TYPE or #libhfst.FOMA_TYPE), nothing is done.
+ ## Set the weights of all final states to \a weight.
+ # If the HfstTransducer is of unweighted type (#hfst.ImplementationType.SFST_TYPE or #hfst.ImplementationType.FOMA_TYPE), nothing is done.
def set_final_weights(self, weight):
pass
- ## Push weights towards initial or final state(s) as defined by \a type.
- # @param PushType defining if weights are pushed towards initial state or final states, possible values are libhfst.TO_INITIAL_STATE and libhst.TO_FINALSTATE
- #
- # If the HfstTransducer is of unweighted type (#SFST_TYPE or #FOMA_TYPE), nothing is done.
+ ## Push weights towards initial state.
+ #
+ # If the HfstTransducer is of unweighted type (#hfst.ImplementationType.SFST_TYPE or #hfst.ImplementationType.FOMA_TYPE), nothing is done.
#
# An example:
# \verbatim
- # >>> import libhfst
- # >>> tr = libhfst.regex('[a::1 a:b::0.3 (b::0)]::0.7;')
- # >>> tr.push_weights(libhfst.TO_INITIAL_STATE)
+ # >>> import hfst
+ # >>> tr = hfst.regex('[a::1 a:b::0.3 (b::0)]::0.7;')
+ # >>> tr.push_weights_to_start()
# >>> print(tr)
# 0 1 a a 2.000000
# 1 2 a b 0.000000
# 2 3 b b 0.000000
# 2 0.000000
# 3 0.000000
- #
- # >>> tr.push_weights(libhfst.TO_FINAL_STATE)
+ # \endverbatim
+ #
+ # @see hfst.HfstTransducer.push_weights_to_end
+ #
+ def push_weights_to_start(self):
+ pass
+
+ ## Push weights towards final state(s).
+ #
+ # If the HfstTransducer is of unweighted type (#hfst.ImplementationType.SFST_TYPE or #hfst.ImplementationType.FOMA_TYPE), nothing is done.
+ #
+ # An example:
+ # \verbatim
+ # >>> import hfst
+ # >>> tr = hfst.regex('[a::1 a:b::0.3 (b::0)]::0.7;')
+ # >>> tr.push_weights_to_end()
# >>> print(tr)
# 0 1 a a 0.000000
# 1 2 a b 0.000000
# 2 3 b b 0.000000
# 2 2.000000
# 3 2.000000
- #
+ #
# \endverbatim
#
- # @see #libhfst.TO_INITIAL_STATE #libhfst.TO_FINAL_STATE
- #
- def push_weights(self, type):
+ # @see hfst.HfstTransducer.push_weights_to_start
+ #
+ def push_weights_to_end(self):
pass
## Substitute symbols or transitions in the transducer.
#
# @param s The symbol or transition to be substituted. Can also be a dictionary of substitutions, if S == None.
- # @param S The symbol, transition, a tuple of transitions or a transducer (libhfst.HfstTransducer) that substitutes \a s.
- # @param kvargs Arguments recognized are 'input' and 'output', their values can be False or True, True being the default.
+ # @param S The symbol, transition, a tuple of transitions or a transducer (hfst.HfstTransducer) that substitutes \a s.
+ # @param kwargs Arguments recognized are 'input' and 'output', their values can be False or True, True being the default.
# These arguments are valid only if \a s and \a S are strings, else they are ignored.
# @param input Whether substitution is performed on input side, defaults to True. Valid only if \a s and \a S are strings.
# @param output Whether substitution is performed on output side, defaults to True. Valid only if \a s and \ S are strings.
#
- # For more information, see libhfst.HfstBasicTransducer.substitute. The function works similarly, with the exception
- # of argument \a S, which must be libhfst.HfstTransducer instead of libhfst.HfstBasicTransducer.
+ # For more information, see hfst.HfstBasicTransducer.substitute. The function works similarly, with the exception
+ # of argument \a S, which must be hfst.HfstTransducer instead of hfst.HfstBasicTransducer.
#
- # @see libhfst.HfstBasicTransducer.substitute
- def substitute(self, s, S=None, **kvargs):
+ # @see hfst.HfstBasicTransducer.substitute
+ def substitute(self, s, S=None, **kwargs):
pass
## Lookup string \a input.
- # @param input The input.
- # @param kvargs Possible parameters and their default values are: obey_flags=True, max_number=-1, time_cutoff=0.0, output='tuple'
- # @param obey_flags Whether flag diacritics are obeyed. Currently always True.
+ # @param input The input. A string or a pre-tokenized tuple of symbols (i.e. a tuple of strings).
+ # @param kwargs Possible parameters and their default values are: obey_flags=True, max_number=-1, time_cutoff=0.0, output='tuple'
+ # @param obey_flags Whether flag diacritics are obeyed. Always True for HFST_OL(W)_TYPE transducers.
# @param max_number Maximum number of results returned, defaults to -1, i.e. infinity.
- # @param time_cutoff How long the function can search for results before returning, expressed in seconds. Defaults to 0.0, i.e. infinitely.
+ # @param time_cutoff How long the function can search for results before returning, expressed in seconds. Defaults to 0.0, i.e. infinitely. Always 0.0 for transducers that are not of HFST_OL(W)_TYPE.
# @param output Possible values are 'tuple', 'text' and 'raw', 'tuple' being the default.
#
- # @note This function is implemented only for optimized lookup format (libhfst.HFST_OL_TYPE or libhfst.HFST_OLW_TYPE).
- # Either convert to optimized lookup format or to HfstBasicTransducer if you wish to perform lookup.
- # Conversion to OL might take a while but it lookup is fast.
- # Conversion to HfstBasicTransducer is quick but lookup is slower.
- #
- def lookup(self, input, **kvargs):
+ # @note This function has an efficient implementation only for optimized lookup format
+ # (hfst.ImplementationType.HFST_OL_TYPE or hfst.ImplementationType.HFST_OLW_TYPE). Other formats perform the
+ # lookup via composition. Consider converting the transducer to optimized lookup format
+ # or to a HfstBasicTransducer. Conversion to HFST_OL(W)_TYPE might take a while but the
+ # lookup is fast. Conversion to HfstBasicTransducer is quick but lookup is slower.
+
+ ## Optimize the transducer for lookup.
+ # This effectively converts the transducer into #hfst.ImplementationType.HFST_OL_TYPE.
+ def lookup_optimize(self):
+ pass
+
+ ## Remove lookup optimization.
+ # This effectively converts transducer (back) into default fst type.
+ def remove_optimization(self):
pass
- ## Extract paths that are recognized by the transducer.
+ ## Extract paths that are recognized by the transducer.
#
- # @param kvargs Arguments recognized are filter_flags, max_cycles, max_number, obey_flags, output, random.
+ # @param kwargs Arguments recognized are filter_flags, max_cycles, max_number, obey_flags, output, random.
# @param filter_flags Whether flags diacritics are filtered out from the result (default True).
# @param max_cycles Indicates how many times a cycle will be followed, with negative numbers indicating unlimited (default -1 i.e. unlimited).
# @param max_number The total number of resulting strings is capped at this value, with 0 or negative indicating unlimited (default -1 i.e. unlimited).
# @param obey_flags Whether flag diacritics are validated (default True).
- # @param output Output format. Values recognized: 'text' (as a string, separated by newlines), 'raw' (a dictionary that maps each input string into a list of tuples of an output string and a weight), 'dict' (a dictionary that maps each input string into a tuple of tuples of an output string and a weight, the default).
+ # @param output Output format. Values recognized: 'text', 'raw', 'dict' (the default). 'text' returns a string where paths are separated by newlines and each path is represented as input_string + ":" + output_string + "\t" t weight. 'raw' yields a tuple of all paths where each path is a 2-tuple consisting of a weight and a tuple of all transition symbol pairs, each symbol pair being a 2-tuple of an input and an output symbol. 'dict' gives a dictionary that maps each input string into [...]
# @param random Whether result strings are fetched randomly (default False).
# @return The extracted strings. \a output controls how they are represented.
#
- # @pre The transducer must be acyclic, if both \a max_number and \a max_cycles have unlimited values. Else a libhfst.TransducerIsCyclicException will be thrown.
+ # @pre The transducer must be acyclic, if both \a max_number and \a max_cycles have unlimited values. Else a hfst.exceptions.TransducerIsCyclicException will be thrown.
#
# An example:
- #
+ #
# \verbatim
- # >>> tr = libhfst.regex('a:b+ (a:c+)')
+ # >>> tr = hfst.regex('a:b+ (a:c+)')
# >>> print(tr)
# 0 1 a b 0.000000
# 1 1 a b 0.000000
@@ -1652,7 +1666,7 @@ class HfstTransducer:
# aa:bc 0
# aaa:bcc 0
# aaaa:bccc 0
- #
+ #
# >>> print(tr.extract_paths(max_cycles=1, max_number=4, output='text'))
# a:b 0
# aa:bb 0
@@ -1662,28 +1676,28 @@ class HfstTransducer:
# \endverbatim
#
# @throws TransducerIsCyclicException
- # @see #libhfst.HfstTransducer.n_best
+ # @see #hfst.HfstTransducer.n_best
# @note <a href="Symbols.html">Special symbols</a> are printed as such.
# @todo a link to flag diacritics
- def extract_paths(self, **kvargs):
+ def extract_paths(self, **kwargs):
pass
## Extract shortest paths of the transducer.
# @return A dictionary.
- def extract_shortest_paths():
+ def extract_shortest_paths(self):
pass
## Extract longest paths of the transducer.
# @return A dictionary.
- def extract_longest_paths(**kvargs):
+ def extract_longest_paths(self, **kwargs):
pass
## Get length of longest path of the transducer.
- def longest_path_size(**kvargs):
+ def longest_path_size(self, **kwargs):
pass
# Lookup or apply a single tokenized string \a tok_input and return a maximum of \a limit results.
- #
+ #
# TODO: This is a version of lookup that handles flag diacritics as ordinary
# symbols and does not validate the sequences prior to outputting. Currently, this function calls lookup_fd.
#
@@ -1695,8 +1709,8 @@ class HfstTransducer:
# def lookup(tok_input, limit=-1):
# pass
- ## Lookup or apply a single string \a input and return a maximum of \a limit results.
- #
+ # Lookup or apply a single string \a input and return a maximum of \a limit results.
+ #
# This is an overloaded lookup function that leaves tokenizing to the transducer.
# @return HfstOneLevelPaths pointer
#def lookup(input, limit=-1):
@@ -1722,11 +1736,11 @@ class HfstTransducer:
# Epsilons on the second level are represented by empty strings
# in the results.
#
- # @pre The transducer must be of type #libhfst.HFST_OL_TYPE or #libhfst.HFST_OLW_TYPE. This function is not implemented for other transducer types.
+ # @pre The transducer must be of type #hfst.ImplementationType.HFST_OL_TYPE or #hfst.ImplementationType.HFST_OLW_TYPE. This function is not implemented for other transducer types.
#
# @param tok_input A tuple of consecutive symbols (strings) to look up.
# @param limit (Currently ignored.) Number of strings to look up. -1 tries to look up all and may get stuck if infinitely ambiguous.
- #
+ #
# @see #is_lookup_infinitely_ambiguous
# @return HfstOneLevelPaths pointer
#
@@ -1749,7 +1763,7 @@ class HfstTransducer:
# def lookup_fd(tok, input, limit = -1):
# pass
- # Whether lookup of path \a input will have infinite results.
+ ## Whether lookup of path \a input will have infinite results.
#
# Currently, this function will return whether the transducer
# is infinitely ambiguous on any lookup path found in the transducer,
@@ -1773,7 +1787,7 @@ class HfstTransducer:
# Detokenize \a tokenized_paths.
#
# Concatenate all transition symbols on input and output levels for each path in \a tokenized_paths.
-#
+#
# @param tokenized_paths An HfstOneLevelPaths pointer or an HfstTwoLevelPaths.
# @return A tuple of HfstPaths.
# def detokenize_paths(tokenized_paths):
@@ -1782,17 +1796,17 @@ class HfstTransducer:
# Detokenize and remove all flag diacritics from \a tokenized_paths.
#
# Concatenate all transition symbols except flag diacritics on input and output levels for each path in \a tokenized_paths.
-#
+#
# @param tokenized_paths An HfstOneLevelPaths pointer or an HfstTwoLevelPaths.
# @return A tuple of HfstPaths.
# def detokenize_and_purge_paths(tokenized_paths):
# pass
-## A stream for reading HFST binary transducers.
+## A stream for reading HFST binary transducers.
#
# An example:
# \verbatim
-# istr = libhfst.HfstInputStream('testfile1.hfst')
+# istr = hfst.HfstInputStream('testfile1.hfst')
# transducers = []
# while not (istr.is_eof()):
# transducers.append(istr.read())
@@ -1811,49 +1825,54 @@ class HfstInputStream:
# @throws NotTransducerStreamException
# @throws EndOfStreamException
# @throws TransducerHeaderException
+ #
+ # \verbatim
+ # istr_to_stdin = hfst.HfstInputStream()
+ # istr_to_file = hfst.HfstInputStream(filename='transducer.hfst')
+ # \endverbatim
def __init__(self, filename=None):
pass
## Close the stream.
#
# If the stream points to standard input, nothing is done.
- def close():
+ def close(self):
pass
## Whether the stream is at end.
- def is_eof():
+ def is_eof(self):
pass
## Whether badbit is set.
- def is_bad():
+ def is_bad(self):
pass
## Whether the state of the stream is good for input operations.
- def is_good():
+ def is_good(self):
pass
- ## The type of the first transducer in the stream.
+ ## The type of the first transducer in the stream.
#
# By default, all transducers in a stream have the same type, else
# a TransducerTypeMismatchException is thrown when reading the first
# transducer that has a different type than the previous ones.
- def get_type():
+ def get_type(self):
pass
## Return next transducer.
#
# @throws EndOfStreamException
- def read():
+ def read(self):
pass
-## A stream for writing binary transducers.
+## A stream for writing binary transducers.
#
-# An example:
+# An example:
# \verbatim
# res = ['foo:bar','0','0 - 0','"?":?','a* b+']
-# ostr = libhfst.HfstOutputStream(filename='testfile1.hfst')
+# ostr = hfst.HfstOutputStream(filename='testfile1.hfst')
# for re in res:
-# ostr.write(libhfst.regex(re))
+# ostr.write(hfst.regex(re))
# ostr.flush()
# ostr.close()
# \endverbatim
@@ -1862,28 +1881,47 @@ class HfstInputStream:
class HfstOutputStream:
## Open a stream for writing binary transducers.
- # @param kvargs Arguments recognized are filename, hfst_format, type.
+ # @param kwargs Arguments recognized are filename, hfst_format, type.
# @param filename The name of the file where transducers are written. If the file exists, it is overwritten. If \a filename is not given, transducers are written to standard output.
# @param hfst_format Whether transducers are written in hfst format (default is True) or as such in their backend format.
- # @param type The type of the transducers that will be written to the stream. Default is #libhfst.get_default_fst_type().
- def __init__(self, **kvargs):
+ # @param type The type of the transducers that will be written to the stream. Default is #hfst.get_default_fst_type().
+ #
+ # \verbatim
+ # ostr = hfst.HfstOutputStream() # a stream for writing default type transducers in hfst format to standard output
+ # transducer = hfst.regex('foo:bar::0.5')
+ # ostr.write(transducer)
+ # ostr.flush()
+ # \endverbatim
+ #
+ # \verbatim
+ # ostr = hfst.HfstOutputStream(filename='transducer.sfst', hfst_format=False, type=hfst.ImplementationType.SFST_TYPE) # a stream for writing SFST_TYPE transducers in their back-end format to a file
+ # transducer1 = hfst.regex('foo:bar')
+ # transducer1.convert(hfst.ImplementationType.SFST_TYPE) # if not set as the default type
+ # transducer2 = hfst.regex('bar:baz')
+ # transducer2.convert(hfst.ImplementationType.SFST_TYPE) # if not set as the default type
+ # ostr.write(transducer1)
+ # ostr.write(transducer2)
+ # ostr.flush()
+ # ostr.close()
+ # \endverbatim
+ def __init__(self, **kwargs):
pass
## Flush the stream.
- def flush():
+ def flush(self):
pass
- ## Write the transducer \a transducer in binary format to the stream.
+ ## Write the transducer \a transducer in binary format to the stream.
#
- # All transducers must have the same type as the stream, else a TransducerTypeMismatchException is thrown.
+ # All transducers must have the same type as the stream, else a TransducerTypeMismatchException is thrown.
#
- # @throws TransducerTypeMismatchException
- def write(transducer):
+ # @throws hfst.exceptions.TransducerTypeMismatchException
+ def write(self, transducer):
pass
- ## Close the stream.
+ ## Close the stream.
# If the stream points to standard output, nothing is done.
- def close():
+ def close(self):
pass
## TODO: documentation ???
@@ -1898,482 +1936,422 @@ class MultiCharSymbolTrie:
## TODO: documentation
# @param string const char *
# @return const char *
- def find(string)
+ def find(string)
## A tokenizer for creating transducers from UTF-8 strings.
#
-# Strings are tokenized from left to right using longest match tokenization.
-# For example, if the tokenizer contains a multicharacter symbol
-# 'foo' and a skip symbol 'fo', the string "foo" is tokenized as 'foo:foo'.
-# If the tokenizer contains a multicharacter symbol 'fo' and a skip
-# symbol 'foo', the string "foo" is tokenized as an empty string.
+# With an HfstTokenizer, it is possible to split UTF-8 strings into
+# tuples of symbols which can then in turn be used to create transducers:
#
-# An example:
# \verbatim
+# >>> tok = hfst.HfstTokenizer()
+# >>> tok.add_multichar_symbol('foo')
+# >>> tok.add_skip_symbol('b')
+# >>> tok.tokenize('foobar')
+# (('foo', 'foo'), ('a', 'a'), ('r', 'r'))
+# >>> tok.tokenize_one_level('foobar')
+# ('foo', 'a', 'r')
+# >>> tok.tokenize('foobar','barfoo')
+# (('foo', 'a'), ('a', 'r'), ('r', 'foo'))
# \endverbatim
#
-# @note The tokenizer only tokenizes utf-8 strings.
-# Special symbols are not included in the tokenizer
+# Strings are tokenized from left to right using longest match tokenization.
+#
+# @note The tokenizer only tokenizes utf-8 strings.
+# Special symbols are not included in the tokenizer
# unless added to it. TODO: should they ???
class HfstTokenizer:
- ## Create a tokenizer that recognizes utf-8 symbols.
+ ## Create a tokenizer that recognizes utf-8 symbols.
+ #
+ # \verbatim
+ # tok = hfst.HfstTokenizer()
+ # \endverbatim
+ #
def __init__(self):
pass
- ## Add a symbol to be skipped to this tokenizer.
+ ## Add a symbol to be skipped to this tokenizer.
+ #
+ # \verbatim
+ # >>> tok = hfst.HfstTokenizer()
+ # >>> tok.add_skip_symbol('foo')
+ # >>> tok.tokenize_one_level('foofofoobar')
+ # ('f', 'o', 'b', 'a', 'r')
+ # \endverbatim
+ #
+ # Note that both multicharacter symbols and skip symbols are matched from left to right
+ # using longest match tokenization:
+ #
+ # \verbatim
+ # >>> tok = hfst.HfstTokenizer()
+ # >>> tok.add_multichar_symbol('foo')
+ # >>> tok.add_skip_symbol('fo')
+ # >>> tok.tokenize_one_level('foofo')
+ # ('foo',)
+ # \endverbatim
+ #
+ # \verbatim
+ # >>> tok = hfst.HfstTokenizer()
+ # >>> tok.add_multichar_symbol('fo')
+ # >>> tok.add_skip_symbol('foo')
+ # >>> tok.tokenize_one_level('foofo')
+ # ('fo',)
+ # \endverbatim
#
- # After skipping a symbol, tokenization is always started again.
- # For example if we have a multicharacter symbol 'foo' and a
- # skip symbol 'bar', the string "fobaro" will be tokenized
- # 'f' 'o' 'o', not 'foo'.
def add_skip_symbol(self, symbol):
pass
- ## Add a multicharacter symbol \a symbol to this tokenizer.
+ ## Add a multicharacter symbol \a symbol to this tokenizer.
+ #
+ # Strings are always tokenized from left to right using longest match tokenization.
+ #
+ # \verbatim
+ # >>> tok = hfst.HfstTokenizer()
+ # >>> tok.add_multichar_symbol('fo')
+ # >>> tok.add_multichar_symbol('foo')
+ # >>> tok.add_multichar_symbol('of')
+ # >>> tok.tokenize_one_level('fofoofooof')
+ # ('fo', 'foo', 'foo', 'of')
+ # \endverbatim
#
- # If a multicharacter symbol has a skip symbol inside it, it is
- # not considered a multicharacter symbol. For example if we have
- # a multicharacter symbol 'foo' and a skip symbol 'bar', the string
- # "fobaro" will be tokenized 'f' 'o' 'o', not 'foo'.
def add_multichar_symbol(self, symbol):
pass
- ## Tokenize the string \a input_string.
+ ## Tokenize the string \a input_string.
+ #
+ # \verbatim
+ # >>> tok = hfst.HfstTokenizer()
+ # >>> t = tok.tokenize('foobar')
+ # >>> print(t)
+ # (('f', 'f'), ('o', 'o'), ('o', 'o'), ('b', 'b'), ('a', 'a'), ('r', 'r'))
+ # \endverbatim
+ #
+ # @see #tokenize_one_level
# @return A tuple of string pairs.
def tokenize(self, input_string):
pass
## Tokenize the string \a input_string.
+ #
+ # \verbatim
+ # >>> tok = hfst.HfstTokenizer()
+ # >>> t = tok.tokenize_one_level('foobar')
+ # >>> print(t)
+ # ('f', 'o', 'o', 'b', 'a', 'r')
+ # \endverbatim
+ #
+ # @see #tokenize
# @return A tuple of strings.
def tokenize_one_level(self, input_string):
pass
- ## Tokenize the string pair \a input_string : \a output_string.
+ ## Tokenize the string pair \a input_string : \a output_string.
#
# If one string has more tokens than the other, epsilons will be
# inserted to the end of the tokenized string with less tokens
# so that both tokenized strings have the same number of tokens.
#
+ # \verbatim
+ # >>> tok = hfst.HfstTokenizer()
+ # >>> tok.add_multichar_symbol('foo')
+ # >>> tok.add_skip_symbol('b')
+ # >>> tok.tokenize('foobar','Foobar')
+ # (('foo', 'F'), ('a', 'o'), ('r', 'o'), ('@_EPSILON_SYMBOL_@', 'a'), ('@_EPSILON_SYMBOL_@', 'r'))
+ # \endverbatim
+ #
# @return A tuple of string pairs.
def tokenize(self, input_string, output_string):
pass
## Tokenize \a str and skip all spaces.
+ #
+ # \verbatim
+ # >>> tok = hfst.HfstTokenizer()
+ # >>> tok.tokenize_space_separated('f o o b a r')
+ # (('f', 'f'), ('o', 'o'), ('o', 'o'), ('b', 'b'), ('a', 'a'), ('r', 'r'))
+ # >>> tok.tokenize_space_separated('foo b a r')
+ # (('foo', 'foo'), ('b', 'b'), ('a', 'a'), ('r', 'r'))
+ # >>> tok.tokenize_space_separated('f o o bar')
+ # (('f', 'f'), ('o', 'o'), ('o', 'o'), ('bar', 'bar'))
+ # >>> tok.tokenize_space_separated('foo bar')
+ # (('foo', 'foo'), ('bar', 'bar'))
+ # >>> tok.tokenize_space_separated('foobar')
+ # (('foobar', 'foobar'),)
+ # \endverbatim
+ #
+ # Note that skip symbols and multicharacter symbols defined with #add_skip_symbol and #add_multichar_symbol
+ # have no effect when #tokenize_space_separated is called.
+ #
# @return A tuple of strings.
def tokenize_space_separated(self, str):
pass
## If \a input_string is not valid utf-8, throw an IncorrectUtf8CodingException.
#
- # A string is non-valid if:
+ # A string is non-valid if:
# - It contains one of the unsigned bytes 192, 193, 245, 246 and 247.
- # - If it is not made up of sequences of one initial byte (0xxxxxxx,
- # 110xxxxx, 1110xxxx or 11110xxx) followed by an appropriate number
+ # - If it is not made up of sequences of one initial byte (0xxxxxxx,
+ # 110xxxxx, 1110xxxx or 11110xxx) followed by an appropriate number
# of continuation bytes (10xxxxxx).
# -# Initial bytes 0xxxxxxx represent ASCII chars and may not be
# followed by a continuation byte.
- # -# Initial bytes 110xxxxx are followed by exactly one
+ # -# Initial bytes 110xxxxx are followed by exactly one
# continuation byte.
# -# Initial bytes 1110xxxx are followed by exactly two continuation
# bytes.
- # -# Initial bytes 11110xxx are followed by exactly three
+ # -# Initial bytes 11110xxx are followed by exactly three
# continuation bytes.
# (For reference: http://en.wikipedia.org/wiki/UTF-8)
#
+ # \verbatim
+ # try:
+ # hfst.HfstTokenizer.check_utf8_correctness('föö')
+ # except hfst.exceptions.IncorrectUtf8CodingException as e:
+ # print('FAIL')
+ # \endverbatim
+ #
# This function is a static one.
def check_utf8_correctness(input_string):
pass
-## A compiler holding information contained in lexc style lexicons.
+# A compiler holding information contained in lexc style lexicons.
# A single LexcCompiler can be extended by adding entries to it, but little
# else can be done with it. It is sufficient to implement clone of lexc.
-class LexcCompiler:
+#class LexcCompiler:
+
+ # Create a lexc compiler for unspecified transducer format.
+ #def __init__(self):
+ # pass
+
+ # Create a lexc compiler with \a impl as transducer format.
+ #def __init__(self, impl):
+ # pass
+
+ # Create a lexc compiler with \a impl as transducer format.
+ #def __init__(self, impl, withFlags):
+ # pass
+
+ # Compile lexc description from \a infile into current compiler.
+ #def parse(infile):
+ # pass
+
+ # Compile lexc description from file @a filename into current compiler.
+ #def parse(filename):
+ # pass
+
+ # Set verbosity options. When \a verbose is true, LexcCompiler will output the messages that Xerox lexc compiler does.
+ #def setVerbosity(verbose):
+ # pass
+
+ # todo
+ #def isQuiet():
+ # pass
+ # todo
+ #def setTreatWarningsAsErrors(value):
+ # pass
+ # todo
+ #def areWarningsTreatedAsErrors():
+ # pass
+ # todo
+ #def setAllowMultipleSublexiconDefinitions(value):
+ # pass
+ # todo
+ #def setWithFlags(value):
+ # pass
+ # todo
+ #def setMinimizeFlags(value):
+ # pass
+ # todo
+ #def setRenameFlags(value):
+ # pass
+
+ # Add @a alphabet to multicharacter symbol set.
+ # These symbols may be used for regular expression ? for backends that do
+ # not support open alphabets.
+ #def addAlphabet(alphabet):
+ # pass
- ## Create a lexc compiler for unspecified transducer format.
- def __init__(self):
- pass
+ # todo
+ #def addNoFlag(lexname):
+ # pass
- ## Create a lexc compiler with \a impl as transducer format.
- def __init__(self, impl):
- pass
+ # Set current processing lexicon name to @a lexicon_name.
+ #def setCurrentLexiconName(lexicon_name):
+ # pass
- ## Create a lexc compiler with \a impl as transducer format.
- def __init__(self, impl, withFlags):
- pass
+ # Add entry defined by a @a entry to current lexicon, pointing to @a continuation weighing @a weight to current lexicon.
+ #def addStringEntry(entry, continuation, weight):
+ # pass
- ## Compile lexc description from \a infile into current compiler.
- def parse(infile):
- pass
+ # Add entry defined by @a upper:@a lower, pointing to @a continuation weighing @a weight to current lexicon.
+ #def addStringPairEntry(upper, lower, continuation, weight):
+ # pass
- ## Compile lexc description from file @a filename into current compiler.
- def parse(filename):
- pass
+ # Add entry defined by regular expression @a xre, pointing to @a continuation weighing @a weight to current lexicon.
+ #def addXreEntry(xre, continuation, weight):
+ # pass
- ## Set verbosity options. When \a verbose is true, LexcCompiler will output the messages that Xerox lexc compiler does.
- def setVerbosity(verbose):
- pass
+ # Add macro definition named @a name matching regular expression @a xre to known xerox regular expressions.
+ #def addXreDefinition(name, xre):
+ # pass
- ## todo
- def isQuiet():
- pass
- ## todo
- def setTreatWarningsAsErrors(value):
- pass
- ## todo
- def areWarningsTreatedAsErrors():
+ # Set start lexicon's name to @a lexicon_name.
+ #def setInitialLexiconName(lexicon_name):
+ # pass
+
+ # Create final usable version of current lexicons and entries.
+ # @return HfstTransducer pointer.
+ #def compileLexical():
+ # pass
+
+ # Check that current morphotax is connected and print anomalies.
+ # Works like xerox lexc, for compatibility.
+ #def printConnectedness():
+ # pass
+
+## A regular expression compiler.
+class XreCompiler:
+
+ ## Construct compiler for OpenFst format (the default) transducers.
+ def __init__(self):
pass
- ## todo
- def setAllowMultipleSublexiconDefinitions(value):
+
+ ## Create compiler for \a impl format transducers.
+ def __init__(self, impl):
pass
- ## todo
- def setWithFlags(value):
+
+ ## Add a definition macro. Compiler will replace arcs labeled \a name with a transducer defined by regular expression \a xre in later phases of compilation (if set_expand_definitions(True) has been called).
+ def define_xre(self, name, xre):
pass
- ## todo
- def setMinimizeFlags(value):
+
+ ## Add a definition macro. Compiler will replace arcs labeled \a name with a transducer \a transducer in later phases of compilation (if set_expand_definitions(True) has been called).
+ def define_transducer(self, name, transducer):
pass
+
## todo
- def setRenameFlags(value):
+ def define_list(self, name, symbol_list):
pass
- ## Add @a alphabet to multicharacter symbol set.
- # These symbols may be used for regular expression ? for backends that do
- # not support open alphabets.
- def addAlphabet(alphabet):
+ ## todo
+ def define_function(self, name, arguments, xre):
pass
## todo
- def addNoFlag(lexname):
+ def undefine(self, name):
pass
- ## Set current processing lexicon name to @a lexicon_name.
- def setCurrentLexiconName(lexicon_name):
+ ## Compile a transducer defined by @a xre.
+ # May return a pointer to @e empty transducer on non-fatal error.
+ # A None pointer is returned on fatal error, if abort is not called.
+ # @return An HfstTransducer pointer.
+ def compile(self, xre):
pass
- ## Add entry defined by a @a entry to current lexicon, pointing to @a continuation weighing @a weight to current lexicon.
- def addStringEntry(entry, continuation, weight):
+ ## Whether \a name is a definition.
+ def is_definition(self, name):
pass
- ## Add entry defined by @a upper:@a lower, pointing to @a continuation weighing @a weight to current lexicon.
- def addStringPairEntry(upper, lower, continuation, weight):
+ ## Whether \a name is a function definition.
+ def is_function_definition(self, name):
pass
- ## Add entry defined by regular expression @a xre, pointing to @a continuation weighing @a weight to current lexicon.
- def addXreEntry(xre, continuation, weight):
+ ## Set the verbosity of the compiler.
+ # @arg v True or False
+ def set_verbosity(self, v):
pass
- ## Add macro definition named @a name matching regular expression @a xre to known xerox regular expressions.
- def addXreDefinition(name, xre):
+ ## (Windows-specific) Whether output is printed to console instead of standard output.
+ # @arg v True or False
+ def setOutputToConsole(self, v):
pass
- ## Set start lexicon's name to @a lexicon_name.
- def setInitialLexiconName(lexicon_name):
+ ## (Windows-specific) Whether output is printed to console instead of standard output.
+ def getOutputToConsole(self, ):
pass
- ## Create final usable version of current lexicons and entries.
- # @return HfstTransducer pointer.
- def compileLexical():
+ ## Whether definitions are expanded.
+ # @arg v True or False
+ def set_expand_definitions(self, v):
pass
- ## Check that current morphotax is connected and print anomalies.
- # Works like xerox lexc, for compatibility.
- def printConnectedness():
- pass
+## A class for performing pattern matching.
+class PmatchContainer:
-## A compiler holding information needed to compile XREs.
-class XreCompiler:
-
- ## Construct compiler for unknown format transducers.
+ ## Initialize a PmatchContainer. Is this needed?
def __init__(self):
pass
-
- ## Create compiler for \a impl format transducers
- def __init__(self, impl):
+ ## Create a PmatchContainer based on definitions \a defs.
+ # @arg defs A tuple of transducers in hfst.HFST_OLW_FORMAT defining how pmatch is done.
+ # @see hfst.compile_pmatch_file
+ def __init__(self, defs):
pass
-
- ## Add a definition macro. Compilers will replace arcs labeled \a name, with the transducer defined by \a xre in later phases of compilation.
- def define(name, xre):
+ ## Match input \a input.
+ def match(self, input, time_cutoff = 0):
pass
-
## todo
- def define_list(name, symbol_list):
+ def get_profiling_info(self):
pass
-
## todo
- def define_function(name, arguments, xre):
+ def set_verbose(self, b):
pass
-
## todo
- def undefine(name):
+ def set_extract_tags_mode(self, b):
pass
-
- ## Compile a transducer defined by @a xre.
- # May return a pointer to @e empty transducer on non-fatal error.
- # A None pointer is returned on fatal error, if abort is not called.
- # @return An HfstTransducer pointer.
- def compile(xre):
+ ## todo
+ def set_profile(self, b):
pass
-# For example the transducer
-#
+# For example the transducer
+#
# \verbatim
# [[@U.FEATURE.FOO@ foo] | [@U.FEATURE.BAR@ bar]] | [[foo @U.FEATURE.FOO@] | [bar @U.FEATURE.BAR@]]
# \endverbatim
-#
+#
# will yield the paths <CODE>[foo foo]</CODE> and <CODE>[bar bar]</CODE>.
# <CODE>[foo bar]</CODE> and <CODE>[bar foo]</CODE> are invalidated
# by the flag diacritics so they will not be included in \a results.
## Whether symbol \a symbol is a flag diacritic.
#
-# Flag diacritics are of the form
+# Flag diacritics are of the form
# \verbatim
# @[PNDRCU][.][A-Z]+([.][A-Z]+)?@
# \endverbatim
def is_diacritic(symbol):
pass
-## A transducer that obligatorily performs the mappings defined by \a mappings in the context \a context when the alphabet is \a alphabet.
-#
-# @param context A pair of transducers where the first transducer defines the left context and the second transducer the right context.
-# @param mappings A set of mappings (a tuple of string pairs) that the resulting transducer will perform in the context given in \a context.
-# @param alphabet The set of symbol pairs (a tuple of string pairs) that defines the alphabet (see the example).
-#
-# For example, a transducer yielded by the following arguments (in pseudcode)
-# \verbatim
-# context = pair( [c|d], [e] )
-# mappings = set(a:b)
-# alphabet = set(a, a:b, b, c, d, e, ...)
-# \endverbatim
-# obligatorily maps the symbol a to b if c or d precedes and e follows. (Elsewhere,
-# the mapping of a to b is optional). This expression is identical to ![.* [c|d] [a:. & !a:b] [e] .*]
-# Note that the alphabet must contain the pair a:b here.
-# @see <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>
-def two_level_if(context, mappings, alphabet):
- pass
-
-
-## A transducer that allows the mappings defined by \a mappings only in the context \a context, when the alphabet is \a alphabet.
-#
-# If called with the same arguments as in the example of #two_level_if, the transducer
-# allows the mapping of symbol a to b only if c or d precedes and e follows.
-# The mapping of a to b is optional in this context but cannot occur in any other context.
-#
-# The expression is equivalent to
-# ![ [ ![.* [c|d]] a:b .* ] | [ .* a:b ![[e] .*] ] ]
-#
-# @see #two_level_if
-def two_level_only_if(context, mappings, alphabet):
- pass
-
-## A transducer that always performs the mappings defined by \a mappings in the context \a context and only in that context, when the alphabet is \a alphabet.
-#
-# If called with the same arguments as in the example of #two_level_if, the transducer
-# maps symbol a to b only and only if c or d precedes and e follows.
-# The mapping of a to b is obligatory in this context and cannot occur in any other context.
-#
-# The expression is equivalent to ![.* [c|d] [a:. & !a:b] [e] .*] &
-# ![ [ ![.* [c|d]] a:b .* ] | [ .* a:b ![[e] .*] ] ]
-#
-# @see #two_level_if
-def two_level_if_and_only_if(context, mappings, alphabet):
- pass
-
-## A transducer that performs an upward mapping \a mapping in the context \a context when the alphabet is \a alphabet. \a optional defines whether the mapping is optional.
-#
-# @param context A pair of transducers where the first transducer defines the left context and the second transducer the right context. Both transducers must be automata, i.e. map strings onto themselves.
-# @param mapping The mapping (transducer) that the resulting transducer will perform in the context given in \a context.
-# @param optional Whether the mapping is optional.
-# @param alphabet The set of symbol pairs that defines the alphabet (a tuple of string pairs).
-#
-# Each substring s of the input string which is in the input language of the transducer \a mapping and whose left context is matched
-# by the expression [.* l] (where l is the first element of \a context) and whose right context is matched by [r .*]
-# (where r is the second element in the context) is mapped to the respective surface strings defined by transducer \a mapping. Any other
-# character is mapped to the characters specified in \a alphabet. The left and right contexts must
-# be automata (i.e. transducers which map strings onto themselves).
-#
-# For example, a transducer yielded by the following arguments (in pseudocode)
-# \verbatim
-# context = pair( [c], [c] )
-# mappings = [ a:b a:b ]
-# alphabet = set(a, b, c)
-# \endverbatim
-# would map the string "caacac" to "cbbcac".
-#
-# Note that the alphabet must contain the characters a and b, but not the pair
-# a:b (unless this replacement is to be allowed everywhere in the context).
-#
-# Note that replace operations (unlike the two-level rules) have to be combined by composition
-# rather than intersection.
-#
-# @throws ContextTransducersAreNotAutomataException
-#
-# @see <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>
-def replace_up(context, mapping, optional, alphabet):
- pass
-
-## The same as replace_up, but matching is done on the output side of \a mapping
-#
-# @see replace_up
-# <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
-def replace_down(context, mapping, optional, alphabet):
- pass
-
-## TODO: document
-def replace_down_karttunen(context, mapping, optional, alphabet):
- pass
-
-## The same as replace_up, but left context matching is done on the output side of \a mapping and right context on the input side of \a mapping.
-#
-# @see replace_up
-def replace_left(context, mapping, optional, alphabet):
- pass
-
-## The same as replace_up, but left context matching is done on the input side of \a mapping and right context on the output side of \a mapping.
-#
-# @see replace_up
-# <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
-def replace_right(context, mapping, optional, alphabet):
- pass
+## Back-end implementations.
+class ImplementationType:
+ ## An SFST transducer, unweighted.
+ SFST_TYPE = libhfst.SFST_TYPE
-## The same as replace_up but \a mapping is performed in every context.
-#
-# @see replace_up
-def replace_up(mapping, optional, alphabet):
- pass
+ ## An OpenFst transducer with tropical weights.
+ TROPICAL_OPENFST_TYPE = libhfst.TROPICAL_OPENFST_TYPE
-## The same as replace_down(context, mapping, optional, alphabet) but \a mapping is performed in every context.
-#
-# @see replace_up
-def replace_down(mapping, optional, alphabet):
- pass
+ ## An OpenFst transducer with logarithmic weights (limited support).
+ LOG_OPENFST_TYPE = libhfst.LOG_OPENFST_TYPE
-## Inversion of the replace_up and the result needs to be composed on the upper side of the input language.
-#
-# B <- A is the inversion of A -> B.
-# \a mapping is performed in every context.
-# @see replace_up
-def left_replace_up(mapping, optional, alphabet):
- pass
+ ## A foma transducer, unweighted.
+ FOMA_TYPE = libhfst.FOMA_TYPE
-## Inversion of the replace_up and the result needs to be composed on the upper side of the input language.
-#
-# B <- A is the inversion of A -> B.
-# @see replace_up
-def left_replace_up(context, mapping, optional, alphabet):
- pass
+ ## An HFST optimized lookup transducer, unweighted.
+ HFST_OL_TYPE = libhfst.HFST_OL_TYPE
-## Inversion of the replace_up and the result needs to be composed on the upper side of the input language. However, matching is done on the output side of \a mapping.
-#
-# @see replace_up
-def left_replace_down(context, mapping, optional, alphabet):
- pass
-
-## Inversion of the replace_up and the result needs to be composed on the upper side of the input language. However, matching is done on the output side of \a mapping.
-#
-# @see replace_up
-def left_replace_down_karttunen(context, mapping, optional, alphabet):
- pass
-
-## Inversion of the replace_up and the result needs to be composed on the upper side of the input language. However, left context matching is done on the input side of \a mapping and right context on the output side of \a mapping.
-#
-# @see replace_up */
-def left_replace_left(context, mapping, optional, alphabet):
- pass
-
-## Inversion of the replace_up and the result needs to be composed on the upper side of the input language. However, left context matching is done on the output side of \a mapping and right context on the input side of \a mapping.
-#
-# @see replace_up
-def left_replace_right(context, mapping, optional, alphabet):
- pass
+ ## An HFST optimized lookup transducer with weights.
+ HFST_OLW_TYPE = libhfst.HFST_OLW_TYPE
+ ## HFST2 header present, conversion required.
+ HFST2_TYPE = libhfst.HFST2_TYPE
+ ## Format left open by e.g. default constructor.
+ UNSPECIFIED_TYPE = libhfst.UNSPECIFIED_TYPE
-## A transducer that allows any (substring) mapping defined by \a mapping only if it occurs in any of the contexts in \a contexts. Symbols outside of the matching substrings are mapped to any symbol allowed by \a alphabet.
-#
-# @throws EmptySetOfContextsException
-#
-# @see <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
-# @param contexts A tuple of HfstTransducer pairs.
-# @param mapping An HfstTransducer.
-# @param alphabet A tuple of string pairs.
-def restriction(contexts, mapping, alphabet):
- pass
+ ## Type not recognised. This type might be returned by a function if an error occurs.
+ ERROR_TYPE = libhfst.ERROR_TYPE
-## A transducer that requires that one of the mappings defined by \a mapping must occur in each context in \a contexts. Symbols outside of the matching substrings are mapped to any symbol allowed by \a alphabet.
-#
-# @see <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
-# @param contexts A tuple of HfstTransducer pairs.
-# @param mapping An HfstTransducer.
-# @param alphabet A tuple of string pairs.
-def coercion(contexts, mapping, alphabet):
- pass
-
-## A transducer that is equivalent to the intersection of restriction and coercion and requires that the mappings defined by \a mapping occur always and only in the given contexts in \a contexts. Symbols outside of the matching substrings are mapped to any symbol allowed by \a alphabet.
-#
-# @see #restriction #coercion <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>
-# @param contexts A tuple of HfstTransducer pairs.
-# @param mapping An HfstTransducer.
-# @param alphabet A tuple of string pairs.
-def restriction_and_coercion(contexts, mapping, alphabet):
- pass
-
-
-## A transducer that specifies that a string from the input language of the transducer \a mapping may only be mapped to one of its output strings (according to transducer \a mapping) if it appears in any of the contexts in \a contexts. Symbols outside of the matching substrings are mapped to any symbol allowed by \a alphabet.
-#
-# @see <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
-# @param contexts A tuple of HfstTransducer pairs.
-# @param mapping An HfstTransducer.
-# @param alphabet A tuple of string pairs.
-def surface_restriction(contexts, mapping, alphabet):
- pass
-
-## A transducer that specifies that a string from the input language of the transducer \a mapping always has to the mapped to one of its output strings according to transducer \a mapping if it appears in any of the contexts in \a contexts. Symbols outside of the matching substrings are mapped to any symbol allowed by \a alphabet.
-#
-# @see <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
-# @param contexts A tuple of HfstTransducer pairs.
-# @param mapping An HfstTransducer.
-# @param alphabet A tuple of string pairs.
-def surface_coercion(contexts, mapping, alphabet):
- pass
-
-## A transducer that is equivalent to the intersection of surface_restriction and surface_coercion.
-#
-# @see #surface_restriction #surface_coercion <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
-# @param contexts A tuple of HfstTransducer pairs.
-# @param mapping An HfstTransducer.
-# @param alphabet A tuple of string pairs.
-def surface_restriction_and_coercion(contexts, mapping, alphabet):
- pass
-
-## A transducer that specifies that a string from the output language of the transducer \a mapping may only be mapped to one of its input strings (according to transducer \a mappings) if it appears in any of the contexts in \a contexts.Symbols outside of the matching substrings are mapped to any symbol allowed by \a alphabet.
-#
-# @see <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
-# @param contexts A tuple of HfstTransducer pairs.
-# @param mapping An HfstTransducer.
-# @param alphabet A tuple of string pairs.
-def deep_restriction(contexts, mapping, alphabet):
- pass
-
-## A transducer that specifies that a string from the output language of the transducer \a mapping always has to be mapped to one of its input strings (according to transducer \a mappings) if it appears in any of the contexts in \a contexts. Symbols outside of the matching substrings are mapped to any symbol allowed by \a alphabet.
-#
-# @see <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
-# @param contexts A tuple of HfstTransducer pairs.
-# @param mapping An HfstTransducer.
-# @param alphabet A tuple of string pairs.
-def deep_coercion(contexts, mapping, alphabet):
- pass
-
-## A transducer that is equivalent to the intersection of deep_restriction and deep_coercion.
-#
-# @see #deep_restriction #deep_coercion <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
-# @param contexts A tuple of HfstTransducer pairs.
-# @param mapping An HfstTransducer.
-# @param alphabet A tuple of string pairs.
-def deep_restriction_and_coercion(contexts, mapping, alphabet):
- pass
## @page Symbols.html Symbols in HFST
#
@@ -2387,21 +2365,21 @@ def deep_restriction_and_coercion(contexts, mapping, alphabet):
# The unknown on input side matches any symbol, the unknown on output side produces any symbol.
# If unknown is on both sides of a transition, it matches any symbol and produces any symbol other than the one that was matched on the input side.
# The identity matches any symbol and produces the same symbol. It must always occur on both sides of a transition.
-# There is also a class of special symbols, called flag diacritics. They are of form
+# There is also a class of special symbols, called flag diacritics. They are of form
# \verbatim
# @[PNDRCU][.][A-Z]+([.][A-Z]+)?@
# \endverbatim
#
-# The internal string representation for epsilon is "@_EPSILON_SYMBOL_@" (libhfst.EPSILON), for unknown "@_UNKNOWN_SYMBOL_@" (libhfst.UNKNOWN) and for identity "@_IDENTITY_SYMBOL_@" (libhfst.IDENTITY).
+# The internal string representation for epsilon is "@_EPSILON_SYMBOL_@" (hfst.EPSILON), for unknown "@_UNKNOWN_SYMBOL_@" (hfst.UNKNOWN) and for identity "@_IDENTITY_SYMBOL_@" (hfst.IDENTITY).
# These strings are used when referring to those symbols in individual transitions, e.g.
#
# \verbatim
-# fsm = libhfst.HfstBasicTransducer()
+# fsm = hfst.HfstBasicTransducer()
# fsm.add_state(1)
# fsm.add_state(2)
# fsm.set_final_weight(2, 0.5)
-# fsm.add_transition(0, 1, libhfst.EPSILON, libhfst.UNKNOWN)
-# fsm.add_transition(1, 2, libhfst.IDENTITY, libhfst.IDENTITY)
+# fsm.add_transition(0, 1, hfst.EPSILON, hfst.UNKNOWN)
+# fsm.add_transition(1, 2, hfst.IDENTITY, hfst.IDENTITY)
# \endverbatim
#
# or reading and printing transitions in AT&& format:
@@ -2419,103 +2397,111 @@ def deep_restriction_and_coercion(contexts, mapping, alphabet):
# On either side of a transition, "?" means the unknown. As a single symbol, "?" means identity-to-identity transition.
# On both sides of a transition "?" means the combination of unknown-to-unknown AND identity-to-identity transitions.
# If unknown-to-unknown transition is needed, it can be given as the subtraction [?:? - ?]. Some examples:
-#
+#
# \verbatim
-# libhfst.regex('0:foo') # epsilon to "foo"
-# libhfst.regex('0:foo') # "foo" to epsilon
-# libhfst.regex('?:foo') # any symbol to "foo"
-# libhfst.regex('?:foo') # "foo" to any symbol
-# libhfst.regex('?:?') # any symbol to any symbol
-# libhfst.regex('?') # any symbol to the same symbol
-# libhfst.regex('?:? - ?') # any symbol to any other symbol
+# hfst.regex('0:foo') # epsilon to "foo"
+# hfst.regex('0:foo') # "foo" to epsilon
+# hfst.regex('?:foo') # any symbol to "foo"
+# hfst.regex('?:foo') # "foo" to any symbol
+# hfst.regex('?:?') # any symbol to any symbol
+# hfst.regex('?') # any symbol to the same symbol
+# hfst.regex('?:? - ?') # any symbol to any other symbol
# \endverbatim
#
# Note that unknowns and identities are expanded with the symbols that the expression becomes aware of during its compilation:
#
# \verbatim
-# libhfst.regex('?') # equal to [?]
-# libhfst.regex('? foo') # equal to [[?|foo] foo]
-# libhfst.regex('? foo bar:?') # equal to [[?|foo|bar] foo [bar:?|bar:bar|bar:foo]]
+# hfst.regex('?') # equal to [?]
+# hfst.regex('? foo') # equal to [[?|foo] foo]
+# hfst.regex('? foo bar:?') # equal to [[?|foo|bar] foo [bar:?|bar:bar|bar:foo]]
# \endverbatim
#
# In lookup, the epsilon is printed as empty string and unknowns and identities as those symbols that they are matched with:
# \verbatim
-# >>> tr = libhfst.regex('foo:0 bar:? ?')
+# >>> tr = hfst.regex('foo:0 bar:? ?')
# >>> print(tr.lookup('foobara'))
# (('bara', 0.0), ('fooa', 0.0))
# \endverbatim
#
# In extract_paths, epsilon, unknown and identity are printed as such:
# \verbatim
-# >>> tr = libhfst.regex('foo:0 bar:? ?')
+# >>> tr = hfst.regex('foo:0 bar:? ?')
# >>> print(tr.extract_paths())
# {'foobar at _IDENTITY_SYMBOL_@': [('@_EPSILON_SYMBOL_@@_UNKNOWN_SYMBOL_@@_IDENTITY_SYMBOL_@', 0.0), ('@_EPSILON_SYMBOL_ at bar@_IDENTITY_SYMBOL_@', 0.0), ('@_EPSILON_SYMBOL_ at foo@_IDENTITY_SYMBOL_@', 0.0)],
-# 'foobarfoo': [('@_EPSILON_SYMBOL_@@_UNKNOWN_SYMBOL_ at foo', 0.0), ('@_EPSILON_SYMBOL_ at barfoo', 0.0), ('@_EPSILON_SYMBOL_ at foofoo', 0.0)],
+# 'foobarfoo': [('@_EPSILON_SYMBOL_@@_UNKNOWN_SYMBOL_ at foo', 0.0), ('@_EPSILON_SYMBOL_ at barfoo', 0.0), ('@_EPSILON_SYMBOL_ at foofoo', 0.0)],
# 'foobarbar': [('@_EPSILON_SYMBOL_@@_UNKNOWN_SYMBOL_ at bar', 0.0), ('@_EPSILON_SYMBOL_ at barbar', 0.0), ('@_EPSILON_SYMBOL_ at foobar', 0.0)]}
# \endverbatim
## @page QuickStart.html Quick Start to HFST
-#
+#
# \section using_hfst Using HFST in your own code
-#
-# After <a href="InstallHfst.html">installing</a> HFST on your computer, start python and execute <code>import libhfst</code>.
-#
-# For example, the following simple program
+#
+# After <a href="https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstPython">installing</a> HFST on your computer, start python and execute <code>import hfst</code>.
+#
+# For example, the following simple program
#
# \verbatim
-# import libhfst
-#
-# tr1 = libhfst.regex('foo:bar')
-# tr2 = libhfst.regex('bar:baz')
+# import hfst
+#
+# tr1 = hfst.regex('foo:bar')
+# tr2 = hfst.regex('bar:baz')
# tr1.compose(tr2)
# print(tr1)
# \endverbatim
-#
+#
# should print to standard output the following text when run:
-#
+#
# \verbatim
# 0 1 foo baz 0
# 1 0
# \endverbatim
-#
+#
# <BR>
-#
+#
# \section hfst_structure Structure of the API
-#
-# The HFST API is written in module libhfst that includes the following classes:
-#
-# - HfstTransducer: A class for creating transducers and performing operations on them.
-#
-# - HfstInputStream and HfstOutputStream: Classes for writing and reading binary transducers.
-#
-# - HfstBasicTransducer: A class for creating transducers from scratch and iterating through their states and transitions.
-#
-# - HfstTokenizer: A class used in creating transducers from UTF-8 strings.
-#
-# There are also functions in module libhfst that are not part of any class. For example #libhfst.fst
+#
+# The HFST API is located in a package 'hfst' that includes the following classes:
+#
+# - #hfst.HfstTransducer: A class for creating transducers and performing operations on them.
+#
+# - #hfst.HfstInputStream and #hfst.HfstOutputStream: Classes for writing and reading binary transducers.
+#
+# - #hfst.HfstBasicTransducer: A class for creating transducers from scratch and iterating through their states and transitions.
+#
+# - #hfst.HfstTokenizer: A class used in creating transducers from UTF-8 strings.
+#
+# There are also functions in package 'hfst' that are not part of any class. For example #hfst.fst
+#
+# There are also the following submodules:
+#
+# - #hfst.exceptions: #hfst.exceptions.HfstException and its subclasses that are used to handle exceptional situations and errors
+#
+# - #hfst.xerox_rules: Functions for creating transducers that implement Xerox-type replace rules
+#
+# - #hfst.sfst_rules: Functions for creating transducers that implement various two-level rules
#
# <BR>
-#
+#
# \section hfst_examples Examples of HFST functionalities
-#
+#
# An example of creating a simple transducer from scratch and converting between transducer formats and testing
-# transducer properties and handling exceptions:
-#
+# transducer properties and handling exceptions:
+#
# \verbatim
-# import libhfst
+# import hfst
# # Create as HFST basic transducer [a:b] with transition weight 0.3 and final weight 0.5.
-# t = libhfst.HfstBasicTransducer()
+# t = hfst.HfstBasicTransducer()
# t.add_state(1)
# t.add_transition(0, 1, 'a', 'b', 0.3)
# t.set_final_weight(1, 0.5)
#
# # Convert to tropical OpenFst format (the default) and push weights toward final state.
-# T = libhfst.HfstTransducer(t)
-# T.push_weights(libhfst.TO_FINAL_STATE)
+# T = hfst.HfstTransducer(t)
+# T.push_weights_to_end()
#
# # Convert back to HFST basic transducer.
-# tc = libhfst.HfstBasicTransducer(T)
+# tc = hfst.HfstBasicTransducer(T)
# try:
# # Rounding might affect the precision.
# if (0.79 < tc.get_final_weight(1)) and (tc.get_final_weight(1) < 0.81):
@@ -2525,44 +2511,45 @@ def deep_restriction_and_coercion(contexts, mapping, alphabet):
# print("TEST FAILED")
# exit(1)
# # If the state does not exist or is not final
-# except libhfst.HfstException:
-# print("TEST FAILED: An exception thrown.")
+# except hfst.exceptions.HfstException as e:
+# print("TEST FAILED: An exception was thrown.")
# exit(1)
# \endverbatim
-#
-#
+#
+#
# An example of creating transducers from strings, applying rules to them and printing the string pairs recognized by the resulting transducer.
-#
+#
# \verbatim
-# import libhfst
-# libhfst.set_default_fst_type(libhfst.FOMA_TYPE)
-#
+# import hfst
+# hfst.set_default_fst_type(hfst.ImplementationType.FOMA_TYPE) # we use foma implementation as there are no weights involved
+#
# # Create a simple lexicon transducer [[foo bar foo] | [foo bar baz]].
-# tok = libhfst.HfstTokenizer()
+# tok = hfst.HfstTokenizer()
# tok.add_multichar_symbol('foo')
# tok.add_multichar_symbol('bar')
# tok.add_multichar_symbol('baz')
-#
-# words = libhfst.tokenized_fst(tok.tokenize('foobarfoo'))
-# t = libhfst.tokenized_fst(tok.tokenize('foobarbaz'))
+#
+# words = hfst.tokenized_fst(tok.tokenize('foobarfoo'))
+# t = hfst.tokenized_fst(tok.tokenize('foobarbaz'))
# words.disjunct(t)
-#
+#
# # Create a rule transducer that optionally replaces 'bar' with 'baz' between 'foo' and 'foo'.
-# rule = libhfst.regex('bar (->) baz || foo _ foo')
-#
+# rule = hfst.regex('bar (->) baz || foo _ foo')
+#
# # Apply the rule transducer to the lexicon.
-# words.compose(rule).minimize()
-#
+# words.compose(rule)
+# words.minimize()
+#
# # Extract all string pairs from the result and print them to standard output.
# results = 0
# try:
# # Extract paths and remove tokenization
# results = words.extract_paths(output='dict')
-# except libhfst.TransducerIsCyclicException:
+# except hfst.exceptions.TransducerIsCyclicException as e:
# # This should not happen because transducer is not cyclic.
# print("TEST FAILED")
# exit(1)
-#
+#
# for input,outputs in results.items():
# print('%s:' % input)
# for output in outputs:
@@ -2578,29 +2565,29 @@ def deep_restriction_and_coercion(contexts, mapping, alphabet):
# foobarbaz:
# foobarbaz 0.000000
# \endverbatim
-#
+#
# <BR>
## @page HfstWithSFSTOpenFstAndFoma.html Using HFST with SFST, OpenFst and foma
-#
+#
# A binary HfstTransducer consists of an HFST header (more on HFST headers on the
# <a href="https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstTransducerHeader">
-# wiki pages</a>) and the transducer of the backend implementation.
-# If you want to write backend transducers as such, you can specify it with
+# wiki pages</a>) and the transducer of the backend implementation.
+# If you want to write backend transducers as such, you can specify it with
# the <i>hfst_format</i> keyword argument of HfstOutputStream constructor:
-#
+#
# \verbatim
# HfstOutputStream(hfst_format=True)
# \endverbatim
-#
-# The following piece of code will write a native OpenFst transducer
+#
+# The following piece of code will write a native OpenFst transducer
# with tropical weights to standard output:
-#
+#
# test.py:
# \verbatim
-# import libhfst
-# ab = libhfst.regex('a:b::2.8')
-# out = libhfst.HfstOutputStream(hfst_format=False)
+# import hfst
+# ab = hfst.regex('a:b::2.8')
+# out = hfst.HfstOutputStream(hfst_format=False)
# out.write(ab)
# out.flush()
# out.close()
@@ -2618,7 +2605,7 @@ def deep_restriction_and_coercion(contexts, mapping, alphabet):
# 1
# \endverbatim
#
-# An libhfst.HfstInputStream can also read backend transducers that do not have an HFST header.
+# An hfst.HfstInputStream can also read backend transducers that do not have an HFST header.
# If we have the following files
#
# symbols.txt:
@@ -2636,8 +2623,8 @@ def deep_restriction_and_coercion(contexts, mapping, alphabet):
#
# test.py:
# \verbatim
-# import libhfst
-# istr = libhfst.HfstInputStream()
+# import hfst
+# istr = hfst.HfstInputStream()
# while not istr.is_eof():
# tr = istr.read()
# print('Read transducer:')
@@ -2663,79 +2650,75 @@ def deep_restriction_and_coercion(contexts, mapping, alphabet):
# For more information on HFST transducer formats and conversions, see the
# <a href="https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstTransducerFormats">
# wiki pages</a>.
-#
-#
-#
+#
+#
+#
# <H3>An issue with foma</H3>
-#
+#
# Foma writes its binary transducers in gzipped format using the gz tools.
# However, we experienced problems when trying to write to standard output or
# read from standard input with gz tools (foma tools do not write to or read
-# from standard streams). So we choose to write, and accordingly read,
-# foma transducers unzipped when writing or reading binary HfstTransducers
-# of libhfst.FOMA_TYPE. As a result, when we write an HfstTransducer of FOMA_TYPE
-# in its plain backend format, the user must zip it themselves before it
+# from standard streams). So we choose to write, and accordingly read,
+# foma transducers unzipped when writing or reading binary HfstTransducers
+# of hfst.ImplementationType.FOMA_TYPE. As a result, when we write an HfstTransducer of FOMA_TYPE
+# in its plain backend format, the user must zip it themselves before it
# can be used by foma tools. (update: at least the newest releases of foma
-# are able to read also unzipped transducers.) Similarily, a foma transducer must be unzipped
+# are able to read also unzipped transducers.) Similarily, a foma transducer must be unzipped
# before it can be read by HFST tools.
-#
+#
# Suppose we have written a FOMA_TYPE HfstTransducer and want to use it with
# foma tools. First we write it, in its plain backend format, to file 'ab.foma'
# with the following piece of code:
-#
+#
# \verbatim
-# import libhfst
-# libhfst.set_default_fst_type(libhfst.FOMA_TYPE)
+# import hfst
+# hfst.set_default_fst_type(hfst.ImplementationType.FOMA_TYPE)
# ab = libfst.regex('a:b')
-# out = libhfst.HfstOutputStream(hfst_format=False)
+# out = hfst.HfstOutputStream(hfst_format=False)
# out.write(ab)
# out.flush()
# out.close()
# \endverbatim
-#
+#
# The command
-#
+#
# \verbatim
# gzip ab.foma
# \endverbatim
-#
+#
# will create a file 'ab.foma.gz' that can be used by (older) foma tools.
-#
+#
# The same with command line tools:
-#
+#
# \verbatim
# echo 'a:b' | hfst-strings2fst -f foma > ab.hfst
# hfst-fst2fst --use-backend-format -f foma > ab.foma
# gzip ab.foma
# \endverbatim
-#
+#
# An example of the opposite case follows. Suppose we have a foma transducer
# 'transducer.foma' and want to read it inside an HFST program.
-# The name of the file must be appended a .gz extension so that the program
+# The name of the file must be appended a .gz extension so that the program
# 'gunzip' knows it is a zipped file. The commands
-#
+#
# \verbatim
# mv transducer.foma transducer.foma.gz
# gunzip transducer.foma.gz
# \endverbatim
-#
+#
# overwrite the original file 'transducer.foma' with an unzipped version of the
# same file. Now the file can be used by HFST:
-#
+#
# \verbatim
-# instr = libhfst.HfstInputStream('transducer.foma')
+# instr = hfst.HfstInputStream('transducer.foma')
# tr = instr.read()
# instr.close()
# \endverbatim
-#
+#
# The same with command line tools:
-#
+#
# \verbatim
# mv transducer.foma transducer.foma.gz
# gunzip transducer.foma.gz
# hfst-sometool transducer.foma
# \endverbatim
-
-## @page InstallHfst.html Installing the HFST API library and Swig/Python bindings
-#
-# TODO
diff --git a/python/doc/hfst/exceptions/__init__.py b/python/doc/hfst/exceptions/__init__.py
new file mode 100644
index 0000000..0efc9bb
--- /dev/null
+++ b/python/doc/hfst/exceptions/__init__.py
@@ -0,0 +1,300 @@
+## @package hfst.exceptions
+## exceptions...
+
+## Base class for HfstExceptions. Holds its own name and the file and line number where it was thrown.
+class HfstException:
+ ## A message describing the error in more detail.
+ def what():
+ pass
+
+## Two or more HfstTransducers are not of the same type. Same as HfstTransducerTypeMismatchException ???
+class HfstTransducerTypeMismatchException(HfstException):
+ pass
+
+
+## The library required by the implementation type requested is not linked to HFST.
+class ImplementationTypeNotAvailableException(HfstException):
+ pass
+
+
+## Function has not been implemented (yet).
+class FunctionNotImplementedException(HfstException):
+ pass
+
+
+## Flag diacritics encountered on one but not the other side of a transition.
+class FlagDiacriticsAreNotIdentitiesException(HfstException):
+ pass
+
+
+## The input is not in valid prolog format.
+class NotValidPrologFormatException(HfstException):
+ pass
+
+
+## Stream cannot be read.
+class StreamNotReadableException(HfstException):
+ pass
+
+
+## Stream cannot be written.
+#
+# Thrown by #hfst.HfstOutputStream.write and #hfst.HfstTransducer.write_att
+class StreamCannotBeWrittenException(HfstException):
+ pass
+
+
+## Stream is closed.
+#
+# Thrown by #hfst.HfstTransducer.write_att
+# #hfst.HfstOutputStream.write
+#
+# An example:
+#
+# \verbatim
+# try:
+# tr = hfst.regex('foo')
+# outstr = hfst.HfstOutputStream(filename='testfile')
+# outstr.close()
+# outstr.write(tr)
+# except hfst.exceptions.StreamIsClosedException:
+# print("Could not write transducer: stream to file was closed.")
+# \endverbatim
+class StreamIsClosedException(HfstException):
+ pass
+
+
+## The stream is at end.
+#
+# Thrown by
+# #hfst.HfstTransducer
+# #hfst.HfstInputStream.__init__
+class EndOfStreamException(HfstException):
+ pass
+
+
+## Transducer is cyclic.
+#
+# Thrown by #hfst.HfstTransducer.extract_paths. An example
+# \verbatim
+# transducer = hfst.regex('[a:b]*')
+# try:
+# results = transducer.extract_paths(output='text')
+# print("The transducer has %i paths:" % len(results))
+# print(results)
+# except hfst.exceptions.TransducerIsCyclicException:
+# print("The transducer is cyclic and has an infinite number of paths. Some of them:")
+# results = transducer.extract_paths(output='text', max_cycles=5)
+# print(results)
+# \endverbatim
+class TransducerIsCyclicException(HfstException):
+ pass
+
+
+## The stream does not contain transducers.
+#
+# Thrown by
+# #hfst.HfstTransducer
+# #hfst.HfstInputStream.__init__
+#
+# An example.
+# \verbatim
+# f = open('foofile', 'w')
+# f.write('This is an ordinary text file.\n')
+# f.close()
+# try:
+# instr = hfst.HfstInputStream('foofile')
+# tr = instr.read()
+# print(tr)
+# instr.close()
+# except hfst.exceptions.NotTransducerStreamException:
+# print("Could not print transducer: the file does not contain binary transducers.")
+# \endverbatim
+class NotTransducerStreamException(HfstException):
+ pass
+
+
+## The stream is not in valid AT&T format.
+#
+# An example:
+# \verbatim
+# f = open('testfile1.att', 'w')
+# f.write('0 1 a b\n\
+# 1 2 c\n\
+# 2\n')
+# f.close()
+# f = hfst.hfst_open('testfile1.att', 'r')
+# try:
+# tr = hfst.read_att(f)
+# except hfst.exceptions.NotValidAttFormatException:
+# print('Could not read file: it is not in valid ATT format.')
+# f.close()
+# \endverbatim
+# thrown by
+# #hfst.HfstTransducer.__init__
+class NotValidAttFormatException(HfstException):
+ pass
+
+
+## The input is not in valid LexC format.
+class NotValidLexcFormatException(HfstException):
+ pass
+
+
+## State is not final (and cannot have a final weight).
+#
+# An example :
+#
+# \verbatim
+# tr = hfst.HfstBasicTransducer()
+# tr.add_state(1)
+# # An exception is thrown as state number 1 is not final
+# try:
+# w = tr.get_final_weight(1)
+# except hfst.exceptions.StateIsNotFinalException:
+# print("State is not final.")
+# \endverbatim
+#
+# You should use function #hfst.HfstBasicTransducer.is_final_state if you are not sure whether a
+# state is final.
+#
+# Thrown by #hfst.HfstBasicTransducer get_final_weight.
+class StateIsNotFinalException(HfstException):
+ pass
+
+
+## Transducers given as rule context are not automata.
+# @see hfst.HfstTransducer.is_automaton()
+class ContextTransducersAreNotAutomataException(HfstException):
+ pass
+
+
+## Transducers are not automata.
+#
+# Example:
+# \verbatim
+# tr1 = hfst.regex('foo:bar')
+# tr2 = hfst.regex('bar:baz')
+# try:
+# tr1.cross_product(tr2)
+# except hfst.exceptions.TransducersAreNotAutomataException:
+# print('Transducers must be automata in cross product.')
+# \endverbatim
+# This exception is thrown by
+# #hfst.HfstTransducer.cross_product
+# when either input transducer does not have equivalent input and
+# output symbols in all its transitions.
+class TransducersAreNotAutomataException(HfstException):
+ pass
+
+
+## The state number argument is not valid.
+#
+# An example :
+#
+# \verbatim
+# tr = hfst.HfstBasicTransducer()
+# tr.add_state(1)
+# try:
+# w = tr.get_final_weight(2)
+# except hfst.exceptions.StateIndexOutOfBoundsException:
+# print('State number 2 does not exist')
+# \endverbatim
+class StateIndexOutOfBoundsException(HfstException):
+ pass
+
+
+## Transducer has a malformed HFST header.
+#
+# Thrown by #hfst.HfstTransducer.__init__ #hfst.HfstInputStream
+class TransducerHeaderException(HfstException):
+ pass
+
+
+## An OpenFst transducer does not have an input symbol table.
+#
+# When converting from OpenFst to tropical or log HFST, the OpenFst transducer
+# must have at least an input symbol table. If the output symbol table
+# is missing, it is assumed to be equivalent to the input symbol table.
+#
+# Thrown by hfst.HfstTransducer.__init__
+class MissingOpenFstInputSymbolTableException(HfstException):
+ pass
+
+
+## Two or more transducers do not have the same type.
+#
+# This can happen if (1) the calling and called transducer in a binary
+# operation, (2) two transducers in a pair of transducers,
+# (3) two consecutive transducers coming from an HfstInputStream or
+# (4) two transducers in a function taking two or more transducers as
+# arguments do not have the same type.
+#
+# An example:
+# \verbatim
+# hfst.set_default_fst_type(hfst.types.TROPICAL_OPENFST_TYPE)
+# tr1 = hfst.regex('foo')
+# tr2 = hfst.regex('bar')
+# tr2.convert(hfst.types.FOMA_TYPE)
+# try:
+# tr1.disjunct(tr2)
+# except hfst.exceptions.TransducerTypeMismatchException:
+# print('The implementation types of transducers must be the same.')
+# \endverbatim
+class TransducerTypeMismatchException(HfstException):
+ pass
+
+
+## The set of transducer pairs is empty.
+#
+# Thrown by rule functions.
+class EmptySetOfContextsException(HfstException):
+ pass
+
+
+## The type of a transducer is not specified.
+#
+# This exception is thrown when an implementation type argument
+# is hfst.types.ERROR_TYPE.
+class SpecifiedTypeRequiredException(HfstException):
+ pass
+
+
+## An error happened probably due to a bug in the HFST code.
+class HfstFatalException(HfstException):
+ pass
+
+
+## Transducer has wrong type.
+#
+# This exception suggests that an HfstTransducer has not been properly
+# initialized, probably due to a bug in the HFST library. Alternatively
+# the default constructor of HfstTransducer has been called at some point.
+#
+# @see #hfst.HfstTransducer.__init__
+class TransducerHasWrongTypeException(HfstException):
+ pass
+
+
+## String is not valid utf-8.
+#
+# This exception suggests that an input string is not valid utf8.
+#
+class IncorrectUtf8CodingException(HfstException):
+ pass
+
+
+## An argument string is an empty string.
+# A transition symbol cannot be an empty string.
+class EmptyStringException(HfstException):
+ pass
+
+
+## A bug in the HFST code.
+class SymbolNotFoundException(HfstException):
+ pass
+
+
+## A piece of metadata in an HFST header is not supported.
+class MetadataException(HfstException):
+ pass
diff --git a/python/doc/hfst/sfst_rules/__init__.py b/python/doc/hfst/sfst_rules/__init__.py
new file mode 100644
index 0000000..3092be4
--- /dev/null
+++ b/python/doc/hfst/sfst_rules/__init__.py
@@ -0,0 +1,242 @@
+## @package hfst.sfst_rules
+# rules...
+
+## A transducer that obligatorily performs the mappings defined by \a mappings in the context \a context when the alphabet is \a alphabet.
+#
+# @param context A pair of transducers where the first transducer defines the left context and the second transducer the right context.
+# @param mappings A set of mappings (a tuple of string pairs) that the resulting transducer will perform in the context given in \a context.
+# @param alphabet The set of symbol pairs (a tuple of string pairs) that defines the alphabet (see the example).
+#
+# For example, a transducer yielded by the following arguments (in pseudcode)
+# \verbatim
+# context = pair( [c|d], [e] )
+# mappings = set(a:b)
+# alphabet = set(a, a:b, b, c, d, e, ...)
+# \endverbatim
+# obligatorily maps the symbol a to b if c or d precedes and e follows. (Elsewhere,
+# the mapping of a to b is optional). This expression is identical to ![.* [c|d] [a:. & !a:b] [e] .*]
+# Note that the alphabet must contain the pair a:b here.
+# @see <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>
+def two_level_if(context, mappings, alphabet):
+ pass
+
+
+## A transducer that allows the mappings defined by \a mappings only in the context \a context, when the alphabet is \a alphabet.
+#
+# If called with the same arguments as in the example of #two_level_if, the transducer
+# allows the mapping of symbol a to b only if c or d precedes and e follows.
+# The mapping of a to b is optional in this context but cannot occur in any other context.
+#
+# The expression is equivalent to
+# ![ [ ![.* [c|d]] a:b .* ] | [ .* a:b ![[e] .*] ] ]
+#
+# @see #two_level_if
+def two_level_only_if(context, mappings, alphabet):
+ pass
+
+## A transducer that always performs the mappings defined by \a mappings in the context \a context and only in that context, when the alphabet is \a alphabet.
+#
+# If called with the same arguments as in the example of #two_level_if, the transducer
+# maps symbol a to b only and only if c or d precedes and e follows.
+# The mapping of a to b is obligatory in this context and cannot occur in any other context.
+#
+# The expression is equivalent to ![.* [c|d] [a:. & !a:b] [e] .*] &
+# ![ [ ![.* [c|d]] a:b .* ] | [ .* a:b ![[e] .*] ] ]
+#
+# @see #two_level_if
+def two_level_if_and_only_if(context, mappings, alphabet):
+ pass
+
+## A transducer that performs an upward mapping \a mapping in the context \a context when the alphabet is \a alphabet. \a optional defines whether the mapping is optional.
+#
+# @param context A pair of transducers where the first transducer defines the left context and the second transducer the right context. Both transducers must be automata, i.e. map strings onto themselves.
+# @param mapping The mapping (transducer) that the resulting transducer will perform in the context given in \a context.
+# @param optional Whether the mapping is optional.
+# @param alphabet The set of symbol pairs that defines the alphabet (a tuple of string pairs).
+#
+# Each substring s of the input string which is in the input language of the transducer \a mapping and whose left context is matched
+# by the expression [.* l] (where l is the first element of \a context) and whose right context is matched by [r .*]
+# (where r is the second element in the context) is mapped to the respective surface strings defined by transducer \a mapping. Any other
+# character is mapped to the characters specified in \a alphabet. The left and right contexts must
+# be automata (i.e. transducers which map strings onto themselves).
+#
+# For example, a transducer yielded by the following arguments (in pseudocode)
+# \verbatim
+# context = pair( [c], [c] )
+# mappings = [ a:b a:b ]
+# alphabet = set(a, b, c)
+# \endverbatim
+# would map the string "caacac" to "cbbcac".
+#
+# Note that the alphabet must contain the characters a and b, but not the pair
+# a:b (unless this replacement is to be allowed everywhere in the context).
+#
+# Note that replace operations (unlike the two-level rules) have to be combined by composition
+# rather than intersection.
+#
+# @throws ContextTransducersAreNotAutomataException
+#
+# @see <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>
+def replace_up(context, mapping, optional, alphabet):
+ pass
+
+## The same as replace_up, but matching is done on the output side of \a mapping
+#
+# @see replace_up
+# <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
+def replace_down(context, mapping, optional, alphabet):
+ pass
+
+## TODO: document
+def replace_down_karttunen(context, mapping, optional, alphabet):
+ pass
+
+## The same as replace_up, but left context matching is done on the output side of \a mapping and right context on the input side of \a mapping.
+#
+# @see replace_up
+def replace_left(context, mapping, optional, alphabet):
+ pass
+
+## The same as replace_up, but left context matching is done on the input side of \a mapping and right context on the output side of \a mapping.
+#
+# @see replace_up
+# <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
+def replace_right(context, mapping, optional, alphabet):
+ pass
+
+## The same as replace_up but \a mapping is performed in every context.
+#
+# @see replace_up
+def replace_up(mapping, optional, alphabet):
+ pass
+
+## The same as replace_down(context, mapping, optional, alphabet) but \a mapping is performed in every context.
+#
+# @see replace_up
+def replace_down(mapping, optional, alphabet):
+ pass
+
+## Inversion of the replace_up and the result needs to be composed on the upper side of the input language.
+#
+# B <- A is the inversion of A -> B.
+# \a mapping is performed in every context.
+# @see replace_up
+def left_replace_up(mapping, optional, alphabet):
+ pass
+
+## Inversion of the replace_up and the result needs to be composed on the upper side of the input language.
+#
+# B <- A is the inversion of A -> B.
+# @see replace_up
+def left_replace_up(context, mapping, optional, alphabet):
+ pass
+
+## Inversion of the replace_up and the result needs to be composed on the upper side of the input language. However, matching is done on the output side of \a mapping.
+#
+# @see replace_up
+def left_replace_down(context, mapping, optional, alphabet):
+ pass
+
+## Inversion of the replace_up and the result needs to be composed on the upper side of the input language. However, matching is done on the output side of \a mapping.
+#
+# @see replace_up
+def left_replace_down_karttunen(context, mapping, optional, alphabet):
+ pass
+
+## Inversion of the replace_up and the result needs to be composed on the upper side of the input language. However, left context matching is done on the input side of \a mapping and right context on the output side of \a mapping.
+#
+# @see replace_up */
+def left_replace_left(context, mapping, optional, alphabet):
+ pass
+
+## Inversion of the replace_up and the result needs to be composed on the upper side of the input language. However, left context matching is done on the output side of \a mapping and right context on the input side of \a mapping.
+#
+# @see replace_up
+def left_replace_right(context, mapping, optional, alphabet):
+ pass
+
+
+
+## A transducer that allows any (substring) mapping defined by \a mapping only if it occurs in any of the contexts in \a contexts. Symbols outside of the matching substrings are mapped to any symbol allowed by \a alphabet.
+#
+# @throws EmptySetOfContextsException
+#
+# @see <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
+# @param contexts A tuple of HfstTransducer pairs.
+# @param mapping An HfstTransducer.
+# @param alphabet A tuple of string pairs.
+def restriction(contexts, mapping, alphabet):
+ pass
+
+## A transducer that requires that one of the mappings defined by \a mapping must occur in each context in \a contexts. Symbols outside of the matching substrings are mapped to any symbol allowed by \a alphabet.
+#
+# @see <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
+# @param contexts A tuple of HfstTransducer pairs.
+# @param mapping An HfstTransducer.
+# @param alphabet A tuple of string pairs.
+def coercion(contexts, mapping, alphabet):
+ pass
+
+## A transducer that is equivalent to the intersection of restriction and coercion and requires that the mappings defined by \a mapping occur always and only in the given contexts in \a contexts. Symbols outside of the matching substrings are mapped to any symbol allowed by \a alphabet.
+#
+# @see #restriction #coercion <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>
+# @param contexts A tuple of HfstTransducer pairs.
+# @param mapping An HfstTransducer.
+# @param alphabet A tuple of string pairs.
+def restriction_and_coercion(contexts, mapping, alphabet):
+ pass
+
+
+## A transducer that specifies that a string from the input language of the transducer \a mapping may only be mapped to one of its output strings (according to transducer \a mapping) if it appears in any of the contexts in \a contexts. Symbols outside of the matching substrings are mapped to any symbol allowed by \a alphabet.
+#
+# @see <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
+# @param contexts A tuple of HfstTransducer pairs.
+# @param mapping An HfstTransducer.
+# @param alphabet A tuple of string pairs.
+def surface_restriction(contexts, mapping, alphabet):
+ pass
+
+## A transducer that specifies that a string from the input language of the transducer \a mapping always has to the mapped to one of its output strings according to transducer \a mapping if it appears in any of the contexts in \a contexts. Symbols outside of the matching substrings are mapped to any symbol allowed by \a alphabet.
+#
+# @see <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
+# @param contexts A tuple of HfstTransducer pairs.
+# @param mapping An HfstTransducer.
+# @param alphabet A tuple of string pairs.
+def surface_coercion(contexts, mapping, alphabet):
+ pass
+
+## A transducer that is equivalent to the intersection of surface_restriction and surface_coercion.
+#
+# @see #surface_restriction #surface_coercion <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
+# @param contexts A tuple of HfstTransducer pairs.
+# @param mapping An HfstTransducer.
+# @param alphabet A tuple of string pairs.
+def surface_restriction_and_coercion(contexts, mapping, alphabet):
+ pass
+
+## A transducer that specifies that a string from the output language of the transducer \a mapping may only be mapped to one of its input strings (according to transducer \a mappings) if it appears in any of the contexts in \a contexts.Symbols outside of the matching substrings are mapped to any symbol allowed by \a alphabet.
+#
+# @see <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
+# @param contexts A tuple of HfstTransducer pairs.
+# @param mapping An HfstTransducer.
+# @param alphabet A tuple of string pairs.
+def deep_restriction(contexts, mapping, alphabet):
+ pass
+
+## A transducer that specifies that a string from the output language of the transducer \a mapping always has to be mapped to one of its input strings (according to transducer \a mappings) if it appears in any of the contexts in \a contexts. Symbols outside of the matching substrings are mapped to any symbol allowed by \a alphabet.
+#
+# @see <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
+# @param contexts A tuple of HfstTransducer pairs.
+# @param mapping An HfstTransducer.
+# @param alphabet A tuple of string pairs.
+def deep_coercion(contexts, mapping, alphabet):
+ pass
+
+## A transducer that is equivalent to the intersection of deep_restriction and deep_coercion.
+#
+# @see #deep_restriction #deep_coercion <a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">SFST manual</a>.
+# @param contexts A tuple of HfstTransducer pairs.
+# @param mapping An HfstTransducer.
+# @param alphabet A tuple of string pairs.
+def deep_restriction_and_coercion(contexts, mapping, alphabet):
+ pass
diff --git a/python/doc/hfst/xerox_rules/__init__.py b/python/doc/hfst/xerox_rules/__init__.py
new file mode 100644
index 0000000..d52ec67
--- /dev/null
+++ b/python/doc/hfst/xerox_rules/__init__.py
@@ -0,0 +1,98 @@
+## @package hfst.xerox_rules
+# Xerox-type replace rules.
+
+## Type of Xerox-type replace rules.
+class ReplaceType:
+ ## Match contexts of replace rules on input level.
+ REPL_UP = libhfst.REPL_UP
+ ## Match contexts of replace rules on output level.
+ REPL_DOWN = libhfst.REPL_DOWN
+ ## Match left context of replace rules on output level and right context on input level.
+ REPL_LEFT = libhfst.REPL_LEFT
+ ## Match left context of replace rules on input level and right context on output level.
+ REPL_RIGHT = libhfst.REPL_RIGHT
+
+## A rule that contains mapping and context and replace type (if any).
+# If rule is A -> B || L _ R , than mapping is cross product of transducers A and B,
+# context is pair of transducers L and R, and replType is enum REPL_UP.
+class Rule:
+ ## Rule that performs mappings defined in \a mapping in every context.
+ # @param mapping HfstTransducerPairVector
+ def __init__(self, mapping):
+ pass
+ ## Rule that performs mappings defined in \a mapping in contexts \a context. \a type defines how contexts are matched.
+ # @param mapping HfstTransducerPairVector
+ # @param context HfstTransducerPairVector
+ # @param type ReplType
+ def __init__(self, mapping, context, type):
+ pass
+ ## Copy Rule.
+ # @param rule Rule
+ def __init__(self, rule):
+ pass
+ ## Default constructor needed for SWIG.
+ def __init__(self):
+ pass
+ ## Get mappings defined in Rule.
+ # @return HfstTransducerPairVector
+ def get_mapping(self):
+ pass
+ ## Get contexts defined in Rule.
+ # @return HfstTransducerPairVector
+ def get_context():
+ pass
+ ## Get replace type defined in Rule.
+ # @return ReplaceType
+ def get_replType():
+ pass
+ ## Encode flag diacritics.
+ def encodeFlags():
+ pass
+ # friend std::ostream& operator<<(std::ostream &out, const Rule &r);
+
+## replace up, left, right, down
+# @param rule (Rule, HfstRuleVector)
+# @param optional (Bool)
+def replace(rule, optional):
+ pass
+## replace up, left, right, down
+# @param rule (Rule, HfstRuleVector)
+# @param optional (Bool)
+def replace_left(rule, optional):
+ pass
+## left to right
+# @param rule (Rule, HfstRuleVector)
+def replace_leftmost_longest_match(rule):
+ pass
+## right to left
+# @param rule (Rule, HfstRuleVector)
+def replace_rightmost_longest_match(rule):
+ pass
+## todo
+# @param rule (Rule, HfstRuleVector)
+def replace_leftmost_shortest_match(rule):
+ pass
+## todo
+# @param rule (Rule, HfstRuleVector)
+def replace_rightmost_shortest_match(rule):
+ pass
+## replace up, left, right, down
+# @param rule (Rule, HfstRuleVector)
+# @param optional (Bool)
+def replace_epenthesis(rule, optional):
+ pass
+## Restriction function "=>"
+# @param automaton (HfstTransducer)
+# @param context (HfstTransducerPairVector)
+def restriction(automaton, context):
+ pass
+## todo
+# @param left (HfstTransducer)
+# @param right (HfstTransducer)
+def before(left, right):
+ pass
+## todo
+# @param left (HfstTransducer)
+# @param right (HfstTransducer)
+def after(left, right):
+ pass
diff --git a/python/doc/xml2docstrings.sh b/python/doc/xml2docstrings.sh
new file mode 100644
index 0000000..5dc0909
--- /dev/null
+++ b/python/doc/xml2docstrings.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+
+# Run this script in the xml directory where you have generated xml output from Doxygen.
+# Dependencies: doxy2swig.py script.
+
+rm doc.i &&
+touch doc.i &&
+for f in \
+classhfst_1_1AttReader.xml \
+classhfst_1_1HfstBasicTransducer.xml \
+classhfst_1_1HfstBasicTransition.xml \
+classhfst_1_1HfstInputStream.xml \
+classhfst_1_1HfstOutputStream.xml \
+classhfst_1_1HfstTokenizer.xml \
+classhfst_1_1HfstTransducer.xml \
+classhfst_1_1MultiCharSymbolTrie.xml \
+classhfst_1_1PmatchContainer.xml \
+classhfst_1_1PrologReader.xml \
+classhfst_1_1XreCompiler.xml \
+exceptions_2____init_____8py.xml \
+____init_____8py.xml \
+namespacehfst_1_1exceptions.xml \
+namespacehfst_1_1rules.xml \
+namespacehfst_1_1types.xml \
+namespacehfst.xml \
+rules_2____init_____8py.xml \
+types_2____init_____8py.xml \
+classhfst_1_1exceptions_1_1*.xml ;
+do
+ echo "generating documentation for file "$f"..." &&
+ ./doxy2swig.py $f tmp &&
+ cat tmp >> doc.i;
+done
+
+sed -i 's/\(%feature("docstring").*\)"$/\1\n"""/; s/";$/"""/;' doc.i
diff --git a/python/docstrings.i b/python/docstrings.i
new file mode 100644
index 0000000..2fc74e0
--- /dev/null
+++ b/python/docstrings.i
@@ -0,0 +1,3450 @@
+
+// File: classhfst_1_1AttReader.xml
+
+
+%feature("docstring") hfst::AttReader
+"""
+
+A class for reading input in AT&T text format and converting it into
+transducer(s).
+
+An example that reads AT&T input from file 'testfile.att' where epsilon is
+represented as \"<eps>\" and creates the corresponding transducers and prints
+them. If the input cannot be parsed, a message showing the invalid line in AT&T
+input is printed and reading is stopped.
+
+ with open('testfile.att', 'r') as f:
+ try:
+ r = hfst.AttReader(f, \"<eps>\")
+ for tr in r:
+ print(tr)
+ except hfst.exceptions.NotValidAttFormatException as e:
+ print(e.what())
+"""
+
+%feature("docstring") hfst::AttReader::__init__
+"""
+
+Create an AttReader that reads input from file *f* where the epsilon is
+represented as *epsilonstr*.
+
+Parameters
+----------
+* `f` :
+ A python file.
+* `epsilonstr` :
+ How epsilon is represented in the file. By default, \"@_EPSILON_SYMBOL_@\"
+ and \"@0@\" are both recognized.
+"""
+
+%feature("docstring") hfst::AttReader::__next__
+"""
+
+Return next element (for python version 3).
+
+Needed for 'for ... in' statement.
+
+ for transducer in att_reader:
+ print(transducer)
+
+Exceptions
+----------
+* `StopIteration` :
+"""
+
+%feature("docstring") hfst::AttReader::read
+"""
+
+Read next transducer.
+
+Read next transducer description in AT&T format and return a corresponding
+transducer.
+
+Exceptions
+----------
+* `hfst.exceptions.NotValidAttFormatException` :
+* `hfst.exceptions.EndOfStreamException` :
+"""
+
+%feature("docstring") hfst::AttReader::__iter__
+"""
+
+An iterator to the reader.
+
+Needed for 'for ... in' statement.
+
+ for transducer in att_reader:
+ print(transducer)
+"""
+
+%feature("docstring") hfst::AttReader::next
+"""
+
+Return next element (for python version 2).
+
+Needed for 'for ... in' statement.
+
+ for transducer in att_reader:
+ print(transducer)
+
+Exceptions
+----------
+* `StopIteration` :
+"""
+
+
+// File: classhfst_1_1HfstBasicTransducer.xml
+
+
+%feature("docstring") hfst::HfstBasicTransducer
+"""
+
+A simple transducer class with tropical weights.
+
+An example of creating an HfstBasicTransducer [foo:bar baz:baz] with weight 0.4
+from scratch:
+
+ # Create an empty transducer
+ # The transducer has initially one start state (number zero)
+ # that is not final
+ fsm = hfst.HfstBasicTransducer()
+ # Add two states to the transducer
+ fsm.add_state(1)
+ fsm.add_state(2)
+ # Create a transition [foo:bar] leading to state 1 with weight 0.1
+ tr = hfst.HfstBasicTransition(1, 'foo', 'bar', 0.1)
+ # and add it to state zero
+ fsm.add_transition(0, tr)
+ # Add a transition [baz:baz] with weight 0 from state 1 to state 2
+ fsm.add_transition(1, hfst.HfstBasicTransition(2, 'baz', 'baz', 0.0))
+ # Set state 2 as final with weight 0.3
+ fsm.set_final_weight(2, 0.3)
+
+An example of iterating through the states and transitions of the above
+transducer when printing them in AT&T format to standard output:
+
+ # Go through all states
+ for state, arcs in enumerate(fsm):
+ for arc in arcs:
+ print('%i ' % (state), end='')
+ print(arc)
+ if fsm.is_final_state(state):
+ print('%i %f' % (state, fsm.get_final_weight(state)) )
+
+See also: hfst.HfstBasicTransition
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::states_and_transitions
+"""
+
+The states and transitions of the transducer.
+
+Returns
+-------
+A tuple of tuples of HfstBasicTransitions.
+
+See also: hfst.HfstBasicTransducer.__enumerate__
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::get_final_weight
+"""
+
+Get the final weight of state *state* in this transducer.
+
+Parameters
+----------
+* `state` :
+ The number of the state. If it does not exist, a StateIsNotFinalException is
+ thrown.
+
+Exceptions
+----------
+* `hfst.exceptions.StateIsNotFinalException.` :
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::__init__
+"""
+
+Create a transducer with one initial state that has state number zero and is not
+a final state, i.e.
+
+create an empty transducer.
+
+ tr = hfst.HfstBasicTransducer()
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::__init__
+"""
+
+Create a transducer equivalent to *transducer*.
+
+Parameters
+----------
+* `transducer` :
+ The transducer to be copied, hfst.HfstBasicTransducer or
+ hfst.HfstTransducer.
+
+ tr = hfst.regex('foo') # creates an HfstTransducer
+ TR = hfst.HfstBasicTransducer(tr)
+ TR2 = hfst.HfstBasicTransducer(TR)
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::symbols_used
+"""
+
+Get a list of all symbols used in the transitions of this transducer.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::remove_symbol_from_alphabet
+"""
+
+Remove symbol *symbol* from the alphabet of the graph.
+
+note: Use with care, removing symbols that occur in the transitions of the graph
+ can have unexpected results.
+
+Parameters
+----------
+* `symbol` :
+ The string to be removed.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::get_transition_pairs
+"""
+
+Get a list of all input/output symbol pairs used in the transitions of this
+transducer.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::is_infinitely_ambiguous
+"""
+
+Whether the transducer is infinitely ambiguous.
+
+A transducer is infinitely ambiguous if there exists an input that will yield
+infinitely many results, i.e. there are input epsilon loops that are traversed
+with that input.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::harmonize
+"""
+
+Harmonize this transducer and *another*.
+
+In harmonization the unknown and identity symbols in transitions of both graphs
+are expanded according to the symbols that are previously unknown to the graph.
+
+For example the graphs
+
+ [a:b ?:?]
+ [c:d ? ?:c] are expanded to
+
+ [ a:b [?:? | ?:c | ?:d | c:d | d:c | c:? | d:?] ]
+ [ c:d [? | a | b] [?:c| a:c | b:?] ] when harmonized.
+
+The symbol '?' means hfst.UNKNOWN in either or both sides of a transition
+(transitions of type [?:x], [x:?] and [?:?]). The transition [?] means
+hfst.IDENTITY.
+
+note: This function is always called for all transducer arguments of functions
+ that take two or more graphs as their arguments, unless otherwise said.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::sort_arcs
+"""
+
+Sort the arcs of this transducer according to input and output symbols.
+
+Returns
+-------
+This transducer.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::states
+"""
+
+The states of the transducer.
+
+Returns
+-------
+A tuple of state numbers.
+
+An example: /verbatim for state in fsm.states(): for arc in
+fsm.transitions(state): print('i ' % (state), end='') print(arc) if
+fsm.is_final_state(state): print('i f' % (state, fsm.get_final_weight(state)) )
+/endverbatim
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::get_max_state
+"""
+
+Get the biggest state number in use.
+
+Returns
+-------
+The biggest state number in use.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::add_state
+"""
+
+Add a new state to this transducer and return its number.
+
+Returns
+-------
+The next (smallest) free state number.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::add_state
+"""
+
+Add a state *s* to this graph.
+
+Parameters
+----------
+* `state` :
+ The number of the state to be added.
+
+Returns
+-------
+*state*
+
+If the state already exists, it is not added again. All states with state number
+smaller than *s* are also added to the transducer if they did not exist before.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::write_att
+"""
+
+Write this transducer in AT&T format to file *f*, *write_weights* defines
+whether weights are written.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::remove_symbols_from_alphabet
+"""
+
+Remove symbols *symbols* from the alphabet of the graph.
+
+note: Use with care, removing symbols that occur in the transitions of the graph
+ can have unexpected results.
+
+Parameters
+----------
+* `symbols` :
+ A tuple of strings to be removed.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::__enumerate__
+"""
+
+Return an enumeration of the states and transitions of the transducer.
+
+ for state, arcs in enumerate(fsm):
+ for arc in arcs:
+ print('%i ' % (state), end='')
+ print(arc)
+ if fsm.is_final_state(state):
+ print('%i %f' % (state, fsm.get_final_weight(state)) )
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::longest_path_size
+"""
+
+The length of the longest path in transducer.
+
+Length of a path means number of arcs on that path.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::get_alphabet
+"""
+
+The symbols in the alphabet of the transducer.
+
+The symbols do not necessarily occur in any transitions of the transducer.
+Epsilon, unknown and identity symbols are always included in the alphabet.
+
+Returns
+-------
+A tuple of strings.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::substitute
+"""
+
+Substitute symbols or transitions in the transducer.
+
+Parameters
+----------
+* `s` :
+ The symbol or transition to be substituted. Can also be a dictionary of
+ substitutions, if S == None.
+* `S` :
+ The symbol, transition, a tuple of transitions or a transducer
+ (hfst.HfstBasicTransducer) that substitutes *s*.
+* `kwargs` :
+ Arguments recognized are 'input' and 'output', their values can be False or
+ True, True being the default. These arguments are valid only if *s* and *S*
+ are strings, else they are ignored.
+* `input` :
+ Whether substitution is performed on input side, defaults to True. Valid
+ only if *s* and *S* are strings.
+* `output` :
+ Whether substitution is performed on output side, defaults to True. Valid
+ only if *s* and *S* are strings.
+
+Possible combinations of arguments and their types are:
+
+(1) substitute(str, str, input=bool, output=bool): substitute symbol with symbol
+on input, output or both sides of each transition in the transducer. (2)
+substitute(strpair, strpair): substitute transition with transition (3)
+substitute(strpair, strpairtuple): substitute transition with several
+transitions (4) substitute(strpair, transducer): substitute transition with a
+transducer (5) substitute(dict): perform several symbol-to-symbol substitutions
+(6) substitute(dict): perform several transition-to-transition substitutions
+
+Examples:
+
+(1) tr.substitute('a', 'A', input=True, output=False): substitute lowercase a:s
+with uppercase ones (2) tr.substitute(('a','b'),('A','B')): substitute
+transitions that map lowercase a into lowercase b with transitions that map
+uppercase a into uppercase b (3) tr.substitute(('a','b'),
+(('A','B'),('a','B'),('A','b'))): change either or both sides of a transition
+[a:b] to uppercase (4) tr.substitute(('a','b'), hfst.regex('[a:b]+')) change
+[a:b] transition into one or more consecutive [a:b] transitions (5)
+tr.substitute({'a':'A', 'b':'B', 'c':'C'}) change lowercase a, b and c into
+their uppercase variants (6) tr.substitute( {('a','a'):('A','A'),
+('b','b'):('B','B'), ('c','c'):('C','C')} ): change lowercase a, b and c into
+their uppercase variants
+
+In case (4), epsilon transitions are used to attach copies of transducer *S*
+between the SOURCE and TARGET state of each transition that is substituted. The
+transition itself is deleted, but its weight is copied to the epsilon transition
+leading from SOURCE to the initial state of *S*. Each final state of *S* is made
+non-final and an epsilon transition leading to TARGET is attached to it. The
+final weight is copied to the epsilon transition.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::remove_transition
+"""
+
+Remove transition *transition* from state *s*.
+
+Parameters
+----------
+* `s` :
+ The state which *transition* belongs to.
+* `transition` :
+ The transition to be removed.
+* `remove_symbols_from_alphabet` :
+ (?)
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::add_transition
+"""
+
+Add a transition *transition* to state *state*, *add_symbols_to_alphabet*
+defines whether the transition symbols are added to the alphabet.
+
+Parameters
+----------
+* `state` :
+ The number of the state where the transition is added. If it does not exist,
+ it is created.
+* `transition` :
+ A hfst.HfstBasicTransition that is added to *state*.
+* `add_symbols_to_alphabet` :
+ Whether the transition symbols are added to the alphabet of the transducer.
+ (In special cases this is not wanted.)
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::add_transition
+"""
+
+Add a transition from state *source* to state *target* with input symbol
+*input*, output symbol *output* and weight *weight*.
+
+Parameters
+----------
+* `source` :
+ The number of the state where the transition is added. If it does not exist,
+ it is created.
+* `target` :
+ The number of the state where the transition leads. If it does not exist, it
+ is created. (?)
+* `input` :
+ The input symbol of the transition.
+* `output` :
+ The output symbol of the transition.
+* `weight` :
+ The weight of the transition.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::read_prolog
+"""
+
+Read a transducer from prolog file *f*.
+
+*linecount* is incremented as lines are read (is it in python?).
+
+Returns
+-------
+A transducer constructed by reading from file *file*. This function is a static
+one.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::lookup_fd
+"""
+
+Lookup tokenized input *input* in the transducer minding flag diacritics.
+
+Parameters
+----------
+* `str` :
+ A list/tuple of strings to look up.
+* `kwargs` :
+ infinite_cutoff=-1, max_weight=None
+* `infinite_cutoff` :
+ Defaults to -1, i.e. infinite.
+* `max_weight` :
+ Defaults to None, i.e. infinity.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::prune_alphabet
+"""
+
+Remove all symbols that do not occur in transitions of the transducer from its
+alphabet.
+
+Epsilon, unknown and identity symbols are always included in the alphabet.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::write_prolog
+"""
+
+Write the transducer in prolog format to file *f*.
+
+Name the transducer *name*.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::disjunct
+"""
+
+Disjunct this transducer with a one-path transducer defined by consecutive
+string pairs in *spv* that has weight *weight*.
+
+pre: This graph must be a trie where all weights are in final states, i.e. all
+ transitions have a zero weight.
+
+There is no way to test whether a graph is a trie, so the use of this function
+is probably limited to fast construction of a lexicon. Here is an example:
+
+ lexicon = hfst.HfstBasicTransducer()
+ tok = hfst.HfstTokenizer()
+ lexicon.disjunct(tok.tokenize('dog'), 0.3)
+ lexicon.disjunct(tok.tokenize('cat'), 0.5)
+ lexicon.disjunct(tok.tokenize('elephant'), 1.6)
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::read_att
+"""
+
+Read a transducer in AT&T format from file *f*.
+
+*epsilon_symbol* defines the symbol used for epsilon, *linecount* is incremented
+as lines are read.
+
+Returns
+-------
+A transducer constructed by reading from file *file*. This function is a static
+one.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::add_symbol_to_alphabet
+"""
+
+Explicitly add *symbol* to the alphabet of the graph.
+
+note: Usually the user does not have to take care of the alphabet of a graph.
+ This function can be useful in some special cases. @ param symbol The string
+ to be added.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::__str__
+"""
+
+Return a string representation of the transducer.
+
+ print(fsm)
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::set_final_weight
+"""
+
+Set the final weight of state *state* in this transducer to *weight*.
+
+If the state does not exist, it is created.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::add_symbols_to_alphabet
+"""
+
+Explicitly add *symbols* to the alphabet of the graph.
+
+note: Usually the user does not have to take care of the alphabet of a graph.
+ This function can be useful in some special cases.
+
+Parameters
+----------
+* `symbols` :
+ A tuple of strings to be added.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::transitions
+"""
+
+Get the transitions of state *state* in this transducer.
+
+If the state does not exist, a *StateIndexOutOfBoundsException* is thrown.
+
+Returns
+-------
+A tuple of HfstBasicTransitions.
+
+ for state in fsm.states():
+ for arc in fsm.transitions(state):
+ print('%i ' % (state), end='')
+ print(arc)
+ if fsm.is_final_state(state):
+ print('%i %f' % (state, fsm.get_final_weight(state)) )
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::insert_freely
+"""
+
+Insert freely any number of *symbol_pair* in the transducer with weight
+*weight*.
+
+Parameters
+----------
+* `symbol_pair` :
+ A string pair to be inserted.
+* `weight` :
+ The weight of the inserted symbol pair.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::insert_freely
+"""
+
+Insert freely any number of *transducer* in this transducer.
+
+param transducer An HfstBasicTransducer to be inserted.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::write_xfst
+"""
+
+Write the transducer in xfst format to file *f*.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::is_final_state
+"""
+
+Whether state *state* is final.
+
+Parameters
+----------
+* `state` :
+ The state whose finality is returned.
+"""
+
+%feature("docstring") hfst::HfstBasicTransducer::is_lookup_infinitely_ambiguous
+"""
+
+Whether the transducer is infinitely ambiguous with input *str*.
+
+Parameters
+----------
+* `str` :
+ The input.
+
+A transducer is infinitely ambiguous with a given input if the input yields
+infinitely many results, i.e. there are input epsilon loops that are traversed
+with the input.
+"""
+
+
+// File: classhfst_1_1HfstBasicTransition.xml
+
+
+%feature("docstring") hfst::HfstBasicTransition
+"""
+
+A transition class that consists of a target state, input and output symbols and
+a a tropical weight.
+
+See also: hfst.HfstBasicTransducer
+"""
+
+%feature("docstring") hfst::HfstBasicTransition::get_input_symbol
+"""
+
+Get the input symbol of the transition.
+"""
+
+%feature("docstring") hfst::HfstBasicTransition::get_weight
+"""
+
+Get the weight of the transition.
+"""
+
+%feature("docstring") hfst::HfstBasicTransition::set_weight
+"""
+
+Set the weight of the transition.
+
+Parameters
+----------
+* `weight` :
+ Weight of the transition.
+"""
+
+%feature("docstring") hfst::HfstBasicTransition::get_output_symbol
+"""
+
+Get the output symbol of the transition.
+"""
+
+%feature("docstring") hfst::HfstBasicTransition::get_target_state
+"""
+
+Get number of the target state of the transition.
+"""
+
+%feature("docstring") hfst::HfstBasicTransition::__init__
+"""
+
+Create an HfstBasicTransition leading to target state *state* with input symbol
+*input*, output symbol *output* and weight *weight*.
+
+Parameters
+----------
+* `state` :
+ Number of the target state.
+* `input` :
+ The input string.
+* `output` :
+ The output string.
+* `weight` :
+ The weight.
+
+Exceptions
+----------
+* `EmptyStringException` :
+ transition = hfst.HfstBasicTransition(1, 'foo', 'bar', 0.5)
+"""
+
+
+// File: classhfst_1_1HfstInputStream.xml
+
+
+%feature("docstring") hfst::HfstInputStream
+"""
+
+A stream for reading HFST binary transducers.
+
+An example:
+
+ istr = hfst.HfstInputStream('testfile1.hfst')
+ transducers = []
+ while not (istr.is_eof()):
+ transducers.append(istr.read())
+ istr.close()
+ print(\"Read %i transducers in total.\" % len(transducers))
+
+For documentation on the HFST binary transducer format, see here.
+"""
+
+%feature("docstring") hfst::HfstInputStream::is_bad
+"""
+
+Whether badbit is set.
+"""
+
+%feature("docstring") hfst::HfstInputStream::is_eof
+"""
+
+Whether the stream is at end.
+"""
+
+%feature("docstring") hfst::HfstInputStream::__init__
+"""
+
+Create a stream for reading binary transducers.
+
+Parameters
+----------
+* `filename` :
+ The name of the transducer file. If not given, standard input is used.
+
+Exceptions
+----------
+* `StreamNotReadableException` :
+* `NotTransducerStreamException` :
+* `EndOfStreamException` :
+* `TransducerHeaderException` :
+ istr_to_stdin = hfst.HfstInputStream()
+ istr_to_file = hfst.HfstInputStream(filename='transducer.hfst')
+"""
+
+%feature("docstring") hfst::HfstInputStream::read
+"""
+
+Return next transducer.
+
+Exceptions
+----------
+* `EndOfStreamException` :
+"""
+
+%feature("docstring") hfst::HfstInputStream::close
+"""
+
+Close the stream.
+
+If the stream points to standard input, nothing is done.
+"""
+
+%feature("docstring") hfst::HfstInputStream::get_type
+"""
+
+The type of the first transducer in the stream.
+
+By default, all transducers in a stream have the same type, else a
+TransducerTypeMismatchException is thrown when reading the first transducer that
+has a different type than the previous ones.
+"""
+
+%feature("docstring") hfst::HfstInputStream::is_good
+"""
+
+Whether the state of the stream is good for input operations.
+"""
+
+
+// File: classhfst_1_1HfstOutputStream.xml
+
+
+%feature("docstring") hfst::HfstOutputStream
+"""
+
+A stream for writing binary transducers.
+
+An example:
+
+ res = ['foo:bar','0','0 - 0','\"?\":?','a* b+']
+ ostr = hfst.HfstOutputStream(filename='testfile1.hfst')
+ for re in res:
+ ostr.write(hfst.regex(re))
+ ostr.flush()
+ ostr.close()
+
+For more information on HFST transducer structure, see this page.
+"""
+
+%feature("docstring") hfst::HfstOutputStream::close
+"""
+
+Close the stream.
+
+If the stream points to standard output, nothing is done.
+"""
+
+%feature("docstring") hfst::HfstOutputStream::__init__
+"""
+
+Open a stream for writing binary transducers.
+
+Parameters
+----------
+* `kwargs` :
+ Arguments recognized are filename, hfst_format, type.
+* `filename` :
+ The name of the file where transducers are written. If the file exists, it
+ is overwritten. If *filename* is not given, transducers are written to
+ standard output.
+* `hfst_format` :
+ Whether transducers are written in hfst format (default is True) or as such
+ in their backend format.
+* `type` :
+ The type of the transducers that will be written to the stream. Default is
+ hfst.get_default_fst_type().
+
+ ostr = hfst.HfstOutputStream() # a stream for writing default type
+transducers in hfst format to standard output
+ transducer = hfst.regex('foo:bar::0.5')
+ ostr.write(transducer)
+ ostr.flush()
+
+ ostr = hfst.HfstOutputStream(filename='transducer.sfst', hfst_format=False,
+type=hfst.ImplementationType.SFST_TYPE) # a stream for writing SFST_TYPE transducers in
+their back-end format to a file
+ transducer1 = hfst.regex('foo:bar')
+ transducer1.convert(hfst.ImplementationType.SFST_TYPE) # if not set as the default type
+ transducer2 = hfst.regex('bar:baz')
+ transducer2.convert(hfst.ImplementationType.SFST_TYPE) # if not set as the default type
+ ostr.write(transducer1)
+ ostr.write(transducer2)
+ ostr.flush()
+ ostr.close()
+"""
+
+%feature("docstring") hfst::HfstOutputStream::flush
+"""
+
+Flush the stream.
+"""
+
+%feature("docstring") hfst::HfstOutputStream::write
+"""
+
+Write the transducer *transducer* in binary format to the stream.
+
+All transducers must have the same type as the stream, else a
+TransducerTypeMismatchException is thrown.
+
+Exceptions
+----------
+* `hfst.exceptions.TransducerTypeMismatchException` :
+"""
+
+
+// File: classhfst_1_1HfstTransducer.xml
+
+
+%feature("docstring") hfst::HfstTransducer
+"""
+
+A synchronous finite-state transducer.
+
+"""
+
+%feature("docstring") hfst::HfstTransducer::get_type
+"""
+
+The implementation type of the transducer.
+
+Returns
+-------
+hfst.ImplementationType
+"""
+
+%feature("docstring") hfst::HfstTransducer::get_name
+"""
+
+Get the name of the transducer.
+
+See also: set_name
+"""
+
+%feature("docstring") hfst::HfstTransducer::repeat_plus
+"""
+
+A concatenation of N transducers where N is any number from one to infinity.
+"""
+
+%feature("docstring") hfst::HfstTransducer::output_project
+"""
+
+Extract the output language of the transducer.
+
+All transition symbol pairs *isymbol:osymbol* are changed to *osymbol:osymbol*.
+"""
+
+%feature("docstring") hfst::HfstTransducer::number_of_states
+"""
+
+The number of states in the transducer.
+"""
+
+%feature("docstring") hfst::HfstTransducer::minus
+"""
+
+Alias for subtract.
+
+See also: hfst.HfstTransducer.subtract
+"""
+
+%feature("docstring") hfst::HfstTransducer::is_infinitely_ambiguous
+"""
+
+Whether the transducer is infinitely ambiguous.
+
+A transducer is infinitely ambiguous if there exists an input that will yield
+infinitely many results, i.e. there are input epsilon loops that are traversed
+with that input.
+"""
+
+%feature("docstring") hfst::HfstTransducer::eliminate_flags
+"""
+
+Eliminate flag diacritics listed in *symbols* from the transducer.
+
+Parameters
+----------
+* `symbols` :
+ The flags to be eliminated. TODO: explain more.
+
+An equivalent transducer with no flags listed in *symbols*.
+"""
+
+%feature("docstring") hfst::HfstTransducer::longest_path_size
+"""
+
+Get length of longest path of the transducer.
+"""
+
+%feature("docstring") hfst::HfstTransducer::get_properties
+"""
+
+Get all properties from the transducer.
+
+Returns
+-------
+A dictionary whose keys are properties and whose values are the values of those
+properties.
+"""
+
+%feature("docstring") hfst::HfstTransducer::optionalize
+"""
+
+Disjunct the transducer with an epsilon transducer.
+"""
+
+%feature("docstring") hfst::HfstTransducer::intersect
+"""
+
+Intersect this transducer with *another*.
+"""
+
+%feature("docstring") hfst::HfstTransducer::repeat_n_plus
+"""
+
+A concatenation of N transducers where N is any number from *n* to infinity,
+inclusive.
+"""
+
+%feature("docstring") hfst::HfstTransducer::compare
+"""
+
+Whether this transducer and *another* are equivalent.
+
+Parameters
+----------
+* `another` :
+ The compared transducer.
+
+pre: *self* and *another* must have the same implementation type.
+
+Two transducers are equivalent iff they accept the same input/output string
+pairs with the same weights and the same alignments.
+
+note: For weighted transducers, the function often returns false negatives due
+ to weight precision issues.
+"""
+
+%feature("docstring") hfst::HfstTransducer::remove_from_alphabet
+"""
+
+Remove *symbol* from the alphabet of the transducer.
+
+Parameters
+----------
+* `symbol` :
+ The symbol (string) to be removed.
+
+pre: *symbol* does not occur in any transition of the transducer.
+
+note: Use with care, removing a symbol that occurs in a transition of the
+ transducer can have unexpected results.
+"""
+
+%feature("docstring") hfst::HfstTransducer::substitute
+"""
+
+Substitute symbols or transitions in the transducer.
+
+Parameters
+----------
+* `s` :
+ The symbol or transition to be substituted. Can also be a dictionary of
+ substitutions, if S == None.
+* `S` :
+ The symbol, transition, a tuple of transitions or a transducer
+ (hfst.HfstTransducer) that substitutes *s*.
+* `kwargs` :
+ Arguments recognized are 'input' and 'output', their values can be False or
+ True, True being the default. These arguments are valid only if *s* and *S*
+ are strings, else they are ignored.
+* `input` :
+ Whether substitution is performed on input side, defaults to True. Valid
+ only if *s* and *S* are strings.
+* `output` :
+ Whether substitution is performed on output side, defaults to True. Valid
+ only if *s* and \\ S are strings.
+
+For more information, see hfst.HfstBasicTransducer.substitute. The function
+works similarly, with the exception of argument *S*, which must be
+hfst.HfstTransducer instead of hfst.HfstBasicTransducer.
+
+See also: hfst.HfstBasicTransducer.substitute
+"""
+
+%feature("docstring") hfst::HfstTransducer::compose
+"""
+
+Compose this transducer with *another*.
+
+Parameters
+----------
+* `another` :
+ The second argument in the composition. Not modified.
+"""
+
+%feature("docstring") hfst::HfstTransducer::write_prolog
+"""
+
+Write the transducer in prolog format with name *name* to file *f*,
+*write_weights* defined whether weights are written.
+
+Parameters
+----------
+* `f` :
+ A python file where the transducer is written.
+* `name` :
+ The name of the transducer that must be given in a prolog file.
+* `write_weights` :
+ Whether weights are written.
+"""
+
+%feature("docstring") hfst::HfstTransducer::repeat_n_to_k
+"""
+
+A concatenation of N transducers where N is any number from *n* to *k*,
+inclusive.
+"""
+
+%feature("docstring") hfst::HfstTransducer::conjunct
+"""
+
+Alias for intersect.
+
+See also: hfst.HfstTransducer.intersect
+"""
+
+%feature("docstring") hfst::HfstTransducer::repeat_n_minus
+"""
+
+A concatenation of N transducers where N is any number from zero to *n*,
+inclusive.
+"""
+
+%feature("docstring") hfst::HfstTransducer::disjunct
+"""
+
+Disjunct this transducer with *another*.
+"""
+
+%feature("docstring") hfst::HfstTransducer::lookup_optimize
+"""
+
+Optimize the transducer for lookup.
+
+This effectively converts the transducer into hfst.ImplementationType.HFST_OL_TYPE.
+"""
+
+%feature("docstring") hfst::HfstTransducer::extract_longest_paths
+"""
+
+Extract longest paths of the transducer.
+
+Returns
+-------
+A dictionary.
+"""
+
+%feature("docstring") hfst::HfstTransducer::n_best
+"""
+
+Extract *n* best paths of the transducer.
+
+In the case of a weighted transducer (hfst.ImplementationType.TROPICAL_OPENFST_TYPE or
+hfst.ImplementationType.LOG_OPENFST_TYPE), best paths are defined as paths with the lowest
+weight. In the case of an unweighted transducer (hfst.ImplementationType.SFST_TYPE or
+hfst.ImplementationType.FOMA_TYPE), the function returns random paths.
+
+This function is not implemented for hfst.ImplementationType.FOMA_TYPE or
+hfst.ImplementationType.SFST_TYPE. If this function is called by an HfstTransducer of type
+hfst.ImplementationType.FOMA_TYPE or hfst.ImplementationType.SFST_TYPE, it is converted to
+hfst.ImplementationType.TROPICAL_OPENFST_TYPE, paths are extracted and it is converted back
+to hfst.ImplementationType.FOMA_TYPE or hfst.ImplementationType.SFST_TYPE. If HFST is not linked to
+OpenFst library, an hfst.exceptions.ImplementationTypeNotAvailableException is
+thrown.
+"""
+
+%feature("docstring") hfst::HfstTransducer::is_implementation_type_available
+"""
+
+Whether HFST is linked to the transducer library needed by implementation type
+*type*.
+"""
+
+%feature("docstring") hfst::HfstTransducer::invert
+"""
+
+Swap the input and output symbols of each transition in the transducer.
+"""
+
+%feature("docstring") hfst::HfstTransducer::extract_paths
+"""
+
+Extract paths that are recognized by the transducer.
+
+Parameters
+----------
+* `kwargs` :
+ Arguments recognized are filter_flags, max_cycles, max_number, obey_flags,
+ output, random.
+* `filter_flags` :
+ Whether flags diacritics are filtered out from the result (default True).
+* `max_cycles` :
+ Indicates how many times a cycle will be followed, with negative numbers
+ indicating unlimited (default -1 i.e. unlimited).
+* `max_number` :
+ The total number of resulting strings is capped at this value, with 0 or
+ negative indicating unlimited (default -1 i.e. unlimited).
+* `obey_flags` :
+ Whether flag diacritics are validated (default True).
+* `output` :
+ Output format. Values recognized: 'text', 'raw', 'dict' (the default). 'text'
+ returns a string where paths are separated by newlines and each path is represented
+ as input_string + ':' + output_string + '\t' + weight. 'raw' yields a tuple of
+ all paths where each path is a 2-tuple consisting of a weight and a tuple of
+ all transition symbol pairs, each symbol pair being a 2-tuple of an input and
+ an output symbol. 'dict' gives a dictionary that maps each input string into a
+ list of possible outputs, each output being a 2-tuple of an output string and
+ a weight.
+* `random` :
+ Whether result strings are fetched randomly (default False).
+
+Returns
+-------
+The extracted strings. *output* controls how they are represented.
+
+pre: The transducer must be acyclic, if both *max_number* and *max_cycles* have
+ unlimited values. Else a hfst.exceptions.TransducerIsCyclicException will be
+ thrown.
+
+An example:
+
+ >>> tr = hfst.regex('a:b+ (a:c+)')
+ >>> print(tr)
+ 0 1 a b 0.000000
+ 1 1 a b 0.000000
+ 1 2 a c 0.000000
+ 1 0.000000
+ 2 2 a c 0.000000
+ 2 0.000000
+
+ >>> print(tr.extract_paths(max_cycles=1, output='text'))
+ a:b 0
+ aa:bb 0
+ aaa:bbc 0
+ aaaa:bbcc 0
+ aa:bc 0
+ aaa:bcc 0
+
+ >>> print(tr.extract_paths(max_number=4, output='text'))
+ a:b 0
+ aa:bc 0
+ aaa:bcc 0
+ aaaa:bccc 0
+
+ >>> print(tr.extract_paths(max_cycles=1, max_number=4, output='text'))
+ a:b 0
+ aa:bb 0
+ aa:bc 0
+ aaa:bcc 0
+
+Exceptions
+----------
+* `TransducerIsCyclicException` :
+
+See also: hfst.HfstTransducer.n_best
+
+note: Special symbols are printed as such.
+Todo
+a link to flag diacritics
+"""
+
+%feature("docstring") hfst::HfstTransducer::__str__
+"""
+
+An AT&T representation of the transducer.
+
+Defined for print command. An example:
+
+ >>> print(hfst.regex('[foo:bar::2]+'))
+ 0 1 foo bar 2.000000
+ 1 1 foo bar 2.000000
+ 1 0.000000 Todo
+Works only for small transducers.
+"""
+
+%feature("docstring") hfst::HfstTransducer::is_cyclic
+"""
+
+Whether the transducer is cyclic.
+"""
+
+%feature("docstring") hfst::HfstTransducer::has_flag_diacritics
+"""
+
+Whether the transducer has flag diacritics in its transitions.
+"""
+
+%feature("docstring") hfst::HfstTransducer::eliminate_flag
+"""
+
+Eliminate flag diacritic *symbol* from the transducer.
+
+Parameters
+----------
+* `symbol` :
+ The flag to be eliminated. TODO: explain more.
+
+An equivalent transducer with no flags *symbol*.
+"""
+
+%feature("docstring") hfst::HfstTransducer::is_lookup_infinitely_ambiguous
+"""
+
+Whether lookup of path *input* will have infinite results.
+
+Currently, this function will return whether the transducer is infinitely
+ambiguous on any lookup path found in the transducer, i.e. the argument *input*
+is ignored.
+
+Todo
+Do not ignore the argument *input*
+"""
+
+%feature("docstring") hfst::HfstTransducer::set_name
+"""
+
+Rename the transducer *name*.
+
+Parameters
+----------
+* `name` :
+ The name of the transducer.
+
+See also: get_name
+"""
+
+%feature("docstring") hfst::HfstTransducer::priority_union
+"""
+
+Make priority union of this transducer with *another*.
+
+For the operation t1.priority_union(t2), the result is a union of t1 and t2,
+except that whenever t1 and t2 have the same string on left side, the path in t2
+overrides the path in t1.
+
+Example
+
+ Transducer 1 (t1):
+ a : a
+ b : b
+
+ Transducer 2 (t2):
+ b : B
+ c : C
+
+ Result ( t1.priority_union(t2) ):
+ a : a
+ b : B
+ c : C For more information, read fsmbook.
+"""
+
+%feature("docstring") hfst::HfstTransducer::insert_freely
+"""
+
+Freely insert a transition or a transducer into the transducer.
+
+Parameters
+----------
+* `ins` :
+ The transition or transducer to be inserted.
+
+If *ins* is a transition, i.e. a 2-tuple of strings: A transition is added to
+each state in this transducer. The transition leads from that state to itself
+with input and output symbols defined by *ins*. The weight of the transition is
+zero.
+
+If *ins* is an hfst.HfstTransducer: A copy of *ins* is attached with epsilon
+transitions to each state of this transducer. After the operation, for each
+state S in this transducer, there is an epsilon transition that leads from state
+S to the initial state of *ins*, and for each final state of *ins*, there is an
+epsilon transition that leads from that final state to state S in this
+transducer. The weights of the final states in *ins* are copied to the epsilon
+transitions leading to state S.
+"""
+
+%feature("docstring") hfst::HfstTransducer::input_project
+"""
+
+Extract the input language of the transducer.
+
+All transition symbol pairs *isymbol:osymbol* are changed to *isymbol:isymbol*.
+"""
+
+%feature("docstring") hfst::HfstTransducer::reverse
+"""
+
+Reverse the transducer.
+
+A reverted transducer accepts the string 'n(0) n(1) ... n(N)' iff the original
+transducer accepts the string 'n(N) n(N-1) ... n(0)'
+"""
+
+%feature("docstring") hfst::HfstTransducer::compose_intersect
+"""
+
+Compose this transducer with the intersection of transducers in *v*.
+
+If *invert* is true, then compose the intersection of the transducers in *v*
+with this transducer.
+
+The algorithm used by this function is faster than intersecting all transducers
+one by one and then composing this transducer with the intersection.
+
+pre: The transducers in *v* are deterministic and epsilon-free.
+
+Parameters
+----------
+* `v` :
+ A tuple of transducers.
+* `invert` :
+ Whether the intersection of the transducers in *v* is composed with this
+ transducer.
+"""
+
+%feature("docstring") hfst::HfstTransducer::copy
+"""
+
+Return a deep copy of the transducer.
+
+ tr = hfst.regex('[foo:bar::0.3]*')
+ TR = tr.copy()
+ assert(tr.compare(TR))
+"""
+
+%feature("docstring") hfst::HfstTransducer::shuffle
+"""
+
+Shuffle this transducer with transducer *another*.
+
+If transducer A accepts string 'foo' and transducer B string 'bar', the
+transducer that results from shuffling A and B accepts all strings
+[(f|b)(o|a)(o|r)].
+
+pre: Both transducers must be automata, i.e. map strings onto themselves.
+"""
+
+%feature("docstring") hfst::HfstTransducer::prune
+"""
+
+Make transducer coaccessible.
+
+A transducer is coaccessible iff there is a path from every state to a final
+state.
+"""
+
+%feature("docstring") hfst::HfstTransducer::extract_shortest_paths
+"""
+
+Extract shortest paths of the transducer.
+
+Returns
+-------
+A dictionary.
+"""
+
+%feature("docstring") hfst::HfstTransducer::set_final_weights
+"""
+
+Set the weights of all final states to *weight*.
+
+If the HfstTransducer is of unweighted type (hfst.ImplementationType.SFST_TYPE or
+hfst.ImplementationType.FOMA_TYPE), nothing is done.
+"""
+
+%feature("docstring") hfst::HfstTransducer::push_weights_to_end
+"""
+
+Push weights towards final state(s).
+
+If the HfstTransducer is of unweighted type (hfst.ImplementationType.SFST_TYPE or
+hfst.ImplementationType.FOMA_TYPE), nothing is done.
+
+An example:
+
+ >>> import hfst
+ >>> tr = hfst.regex('[a::1 a:b::0.3 (b::0)]::0.7;')
+ >>> tr.push_weights_to_end()
+ >>> print(tr)
+ 0 1 a a 0.000000
+ 1 2 a b 0.000000
+ 2 3 b b 0.000000
+ 2 2.000000
+ 3 2.000000
+
+See also: hfst.HfstTransducer.push_weights_to_start
+"""
+
+%feature("docstring") hfst::HfstTransducer::push_weights_to_start
+"""
+
+Push weights towards initial state.
+
+If the HfstTransducer is of unweighted type (hfst.ImplementationType.SFST_TYPE or
+hfst.ImplementationType.FOMA_TYPE), nothing is done.
+
+An example:
+
+ >>> import hfst
+ >>> tr = hfst.regex('[a::1 a:b::0.3 (b::0)]::0.7;')
+ >>> tr.push_weights_to_start()
+ >>> print(tr)
+ 0 1 a a 2.000000
+ 1 2 a b 0.000000
+ 2 3 b b 0.000000
+ 2 0.000000
+ 3 0.000000
+
+See also: hfst.HfstTransducer.push_weights_to_end
+"""
+
+%feature("docstring") hfst::HfstTransducer::subtract
+"""
+
+Subtract transducer *another* from this transducer.
+"""
+
+%feature("docstring") hfst::HfstTransducer::repeat_star
+"""
+
+A concatenation of N transducers where N is any number from zero to infinity.
+"""
+
+%feature("docstring") hfst::HfstTransducer::write_att
+"""
+
+Write the transducer in AT&T format to file *f*, *write_weights* defined whether
+weights are written.
+
+Parameters
+----------
+* `f` :
+ A python file where transducer is written.
+* `write_weights` :
+ Whether weights are written.
+"""
+
+%feature("docstring") hfst::HfstTransducer::write_att
+"""
+
+Write the transducer in AT&T format to file *ofile*, *write_weights* defines
+whether weights are written.
+
+The fields in the resulting AT&T format are separated by tabulator characters.
+
+NOTE: If the transition symbols contain space characters,the spaces are printed
+as '@_SPACE_@' because whitespace characters are used as field separators in
+AT&T format. Epsilon symbols are printed as '@0@'.
+
+If several transducers are written in the same file, they must be separated by a
+line of two consecutive hyphens \"--\", so that they will be read correctly by
+hfst.read_att.
+
+An example:
+
+ tr1 = hfst.regex('[foo:bar baz:0 \" \"]::0.3')
+ tr2 = hfst.empty_fst()
+ tr3 = hfst.epsilon_fst(0.5)
+ tr4 = hfst.regex('[foo]')
+ tr5 = hfst.empty_fst()
+
+ f = hfst.hfst_open('testfile.att', 'w')
+ for tr in [tr1, tr2, tr3, tr4]:
+ tr.write_att(f)
+ f.write('--\\n')
+ tr5.write_att(f)
+ f.close()
+
+This will yield a file 'testfile.att' that looks as follows:
+
+ 0 1 foo bar 0.299805
+ 1 2 baz @0@ 0.000000
+ 2 3 @_SPACE_@ @_SPACE_@ 0.000000
+ 3 0.000000
+ --
+ --
+ 0 0.500000
+ --
+ 0 1 foo foo 0.000000
+ 1 0.000000
+ --
+
+Exceptions
+----------
+* `StreamCannotBeWrittenException` :
+* `StreamIsClosedException` :
+
+See also: hfst.HfstOutputStream.write
+
+See also: hfst.HfstTransducer.__init__
+"""
+
+%feature("docstring") hfst::HfstTransducer::write_att
+"""
+
+Write the transducer in AT&T format to file named *filename*.
+
+*write_weights* defines whether weights are written.
+
+If the file exists, it is overwritten. If the file does not exist, it is
+created.
+"""
+
+%feature("docstring") hfst::HfstTransducer::lookup
+"""
+
+Lookup string *input*.
+
+Parameters
+----------
+* `input` :
+ The input.
+* `kwargs` :
+ Possible parameters and their default values are: obey_flags=True,
+ max_number=-1, time_cutoff=0.0, output='tuple'
+* `obey_flags` :
+ Whether flag diacritics are obeyed. Currently always True.
+* `max_number` :
+ Maximum number of results returned, defaults to -1, i.e. infinity.
+* `time_cutoff` :
+ How long the function can search for results before returning, expressed in
+ seconds. Defaults to 0.0, i.e. infinitely.
+* `output` :
+ Possible values are 'tuple', 'text' and 'raw', 'tuple' being the default.
+
+note: This function is implemented only for optimized lookup format
+ (hfst.ImplementationType.HFST_OL_TYPE or hfst.ImplementationType.HFST_OLW_TYPE). Either convert to
+ optimized lookup format or to HfstBasicTransducer if you wish to perform
+ lookup. Conversion to OL might take a while but it lookup is fast.
+ Conversion to HfstBasicTransducer is quick but lookup is slower.
+"""
+
+%feature("docstring") hfst::HfstTransducer::__init__
+"""
+
+Create an empty transducer.
+
+ tr = hfst.HfstTransducer()
+ assert(tr.compare(hfst.empty_fst()))
+"""
+
+%feature("docstring") hfst::HfstTransducer::__init__
+"""
+
+Create a deep copy of HfstTransducer *another* or a transducer equivalent to
+HfstBasicTransducer *another*.
+
+Parameters
+----------
+* `another` :
+ An HfstTransducer or HfstBasicTransducer.
+
+An example:
+
+ tr1 = hfst.regex('foo bar foo')
+ tr2 = hfst.HfstTransducer(tr1)
+ tr2.substitute('foo','FOO')
+ tr1.concatenate(tr2)
+"""
+
+%feature("docstring") hfst::HfstTransducer::__init__
+"""
+
+Create an HFST transducer equivalent to HfstBasicTransducer *t*.
+
+The type of the created transducer is defined by *type*.
+
+Parameters
+----------
+* `t` :
+ An HfstBasicTransducer.
+* `type` :
+ The type of the resulting transducer. If you want to use the default type,
+ you can just call hfst.HfstTransducer(fsm)
+"""
+
+%feature("docstring") hfst::HfstTransducer::insert_to_alphabet
+"""
+
+Explicitly insert *symbol* to the alphabet of the transducer.
+
+Parameters
+----------
+* `symbol` :
+ The symbol (string) to be inserted.
+
+note: Usually this function is not needed since new symbols are added to the
+ alphabet by default.
+"""
+
+%feature("docstring") hfst::HfstTransducer::concatenate
+"""
+
+Concatenate this transducer with *another*.
+"""
+
+%feature("docstring") hfst::HfstTransducer::determinize
+"""
+
+Determinize the transducer.
+
+Determinizing a transducer yields an equivalent transducer that has no state
+with two or more transitions whose input:output symbol pairs are the same.
+"""
+
+%feature("docstring") hfst::HfstTransducer::is_automaton
+"""
+
+Whether each transition in the transducer has equivalent input and output
+symbols.
+
+note: Transition with hfst.UNKNOWN on both sides IS NOT a transition with
+ equivalent input and output symbols.
+
+note: Transition with hfst.IDENTITY on both sides IS a transition with
+ equivalent input and output symbols.
+"""
+
+%feature("docstring") hfst::HfstTransducer::minimize
+"""
+
+Minimize the transducer.
+
+Minimizing a transducer yields an equivalent transducer with the smallest number
+of states.
+
+Bug
+OpenFst's minimization algorithm seems to add epsilon transitions to weighted
+transducers?
+"""
+
+%feature("docstring") hfst::HfstTransducer::set_property
+"""
+
+Set arbitrary string property *property* to *value*.
+
+Parameters
+----------
+* `property` :
+ A string naming the property.
+* `value` :
+ A string expressing the value of *property*.
+
+set_property('name', 'name of the transducer') equals set_name('name of the
+transducer').
+
+note: While this function is capable of creating endless amounts of arbitrary
+ metadata, it is suggested that property names are drawn from central
+ repository, or prefixed with \"x-\". A property that does not follow this
+ convention may affect the behavior of transducer in future releases.
+"""
+
+%feature("docstring") hfst::HfstTransducer::convert
+"""
+
+Convert the transducer into an equivalent transducer in format *type*.
+
+If a weighted transducer is converted into an unweighted one, all weights are
+lost. In the reverse case, all weights are initialized to the semiring's one.
+
+A transducer of type hfst.ImplementationType.SFST_TYPE, hfst.ImplementationType.TROPICAL_OPENFST_TYPE,
+hfst.ImplementationType.LOG_OPENFST_TYPE or hfst.ImplementationType.FOMA_TYPE can be converted into an
+hfst.ImplementationType.HFST_OL_TYPE or hfst.ImplementationType.HFST_OLW_TYPE transducer, but an
+hfst.ImplementationType.HFST_OL_TYPE or hfst.ImplementationType.HFST_OLW_TYPE transducer cannot be
+converted to any other type.
+
+note: For conversion between HfstBasicTransducer and HfstTransducer, see
+ hfst.HfstTransducer.__init__ and hfst.HfstBasicTransducer.__init__
+"""
+
+%feature("docstring") hfst::HfstTransducer::remove_epsilons
+"""
+
+Remove all *epsilon:epsilon* transitions from the transducer so that the
+resulting transducer is equivalent to the original one.
+"""
+
+%feature("docstring") hfst::HfstTransducer::repeat_n
+"""
+
+A concatenation of *n* transducers.
+"""
+
+%feature("docstring") hfst::HfstTransducer::number_of_arcs
+"""
+
+The number of transitions in the transducer.
+"""
+
+%feature("docstring") hfst::HfstTransducer::cross_product
+"""
+
+Make cross product of this transducer with *another*.
+
+It pairs every string of this with every string of *another*. If strings are not
+the same length, epsilon padding will be added in the end of the shorter string.
+
+pre: Both transducers must be automata, i.e. map strings onto themselves.
+"""
+
+%feature("docstring") hfst::HfstTransducer::write
+"""
+
+Write the transducer in binary format to *ostr*.
+
+Parameters
+----------
+* `ostr` :
+ A hfst.HfstOutputStream where the transducer is written.
+"""
+
+%feature("docstring") hfst::HfstTransducer::lenient_composition
+"""
+
+Perform a lenient composition on this transducer and *another*.
+
+TODO: explain more.
+"""
+
+%feature("docstring") hfst::HfstTransducer::remove_optimization
+"""
+
+Remove lookup optimization.
+
+This effectively converts transducer (back) into default fst type.
+"""
+
+%feature("docstring") hfst::HfstTransducer::get_property
+"""
+
+Get arbitrary string propert *property*.
+
+Parameters
+----------
+* `property` :
+ The name of the property whose value is returned. get_property('name') works
+ like get_name().
+"""
+
+%feature("docstring") hfst::HfstTransducer::get_alphabet
+"""
+
+Get the alphabet of the transducer.
+
+The alphabet is defined as the set of symbols known to the transducer.
+
+Returns
+-------
+A tuple of strings.
+"""
+
+
+// File: classhfst_1_1MultiCharSymbolTrie.xml
+
+
+%feature("docstring") hfst::MultiCharSymbolTrie
+"""
+
+TODO: documentation ???
+"""
+
+%feature("docstring") hfst::MultiCharSymbolTrie::__init__
+"""
+
+TODO: documentation.
+"""
+
+%feature("docstring") hfst::MultiCharSymbolTrie::add
+"""
+
+TODO: documentation.
+
+Parameters
+----------
+* `string` :
+ const char *
+"""
+
+
+// File: classhfst_1_1PmatchContainer.xml
+
+
+%feature("docstring") hfst::PmatchContainer
+"""
+
+A class for performing pattern matching.
+"""
+
+%feature("docstring") hfst::PmatchContainer::__init__
+"""
+
+Initialize a PmatchContainer.
+
+Is this needed?
+"""
+
+%feature("docstring") hfst::PmatchContainer::__init__
+"""
+
+Create a PmatchContainer based on definitions *defs*.
+
+* defs A tuple of transducers in hfst.HFST_OLW_FORMAT defining how pmatch is
+ done.
+
+ See also: hfst.compile_pmatch_file
+"""
+
+%feature("docstring") hfst::PmatchContainer::get_profiling_info
+"""
+
+todo
+"""
+
+%feature("docstring") hfst::PmatchContainer::set_profile
+"""
+
+todo
+"""
+
+%feature("docstring") hfst::PmatchContainer::set_extract_tags_mode
+"""
+
+todo
+"""
+
+%feature("docstring") hfst::PmatchContainer::match
+"""
+
+Match input *input*.
+"""
+
+%feature("docstring") hfst::PmatchContainer::set_verbose
+"""
+
+todo
+"""
+
+
+// File: classhfst_1_1PrologReader.xml
+
+
+%feature("docstring") hfst::PrologReader
+"""
+
+A class for reading input in prolog text format and converting it into
+transducer(s).
+
+An example that reads prolog input from file 'testfile.prolog' and creates the
+corresponding transducers and prints them. If the input cannot be parsed, a
+message showing the invalid line in prolog input is printed and reading is
+stopped.
+
+ with open('testfile.prolog', 'r') as f:
+ try:
+ r = hfst.PrologReader(f)
+ for tr in r:
+ print(tr)
+ except hfst.exceptions.NotValidPrologFormatException as e:
+ print(e.what())
+"""
+
+%feature("docstring") hfst::PrologReader::next
+"""
+
+Return next element (for python version 2).
+
+Needed for 'for ... in' statement.
+
+ for transducer in prolog_reader:
+ print(transducer)
+
+Exceptions
+----------
+* `StopIteration` :
+"""
+
+%feature("docstring") hfst::PrologReader::__iter__
+"""
+
+An iterator to the reader.
+
+Needed for 'for ... in' statement.
+
+ for transducer in prolog_reader:
+ print(transducer)
+"""
+
+%feature("docstring") hfst::PrologReader::__next__
+"""
+
+Return next element (for python version 3).
+
+Needed for 'for ... in' statement.
+
+ for transducer in prolog_reader:
+ print(transducer)
+
+Exceptions
+----------
+* `StopIteration` :
+"""
+
+%feature("docstring") hfst::PrologReader::__init__
+"""
+
+Create a PrologReader that reads input from file *f*.
+
+Parameters
+----------
+* `f` :
+ A python file.
+"""
+
+%feature("docstring") hfst::PrologReader::read
+"""
+
+Read next transducer.
+
+Read next transducer description in prolog format and return a corresponding
+transducer.
+
+Exceptions
+----------
+* `hfst.exceptions.NotValidPrologFormatException` :
+* `hfst.exceptions.EndOfStreamException` :
+"""
+
+
+// File: classhfst_1_1XreCompiler.xml
+
+
+%feature("docstring") hfst::XreCompiler
+"""
+
+A regular expression compiler.
+"""
+
+%feature("docstring") hfst::XreCompiler::__init__
+"""
+
+Construct compiler for OpenFst format (the default) transducers.
+"""
+
+%feature("docstring") hfst::XreCompiler::__init__
+"""
+
+Create compiler for *impl* format transducers.
+"""
+
+%feature("docstring") hfst::XreCompiler::define_list
+"""
+
+todo
+"""
+
+%feature("docstring") hfst::XreCompiler::is_function_definition
+"""
+
+Whether *name* is a function definition.
+"""
+
+%feature("docstring") hfst::XreCompiler::is_definition
+"""
+
+Whether *name* is a definition.
+"""
+
+%feature("docstring") hfst::XreCompiler::set_verbosity
+"""
+
+Set the verbosity of the compiler.
+
+* v True or False
+"""
+
+%feature("docstring") hfst::XreCompiler::undefine
+"""
+
+todo
+"""
+
+%feature("docstring") hfst::XreCompiler::setOutputToConsole
+"""
+
+(Windows-specific) Whether output is printed to console instead of standard
+output.
+
+* v True or False
+"""
+
+%feature("docstring") hfst::XreCompiler::compile
+"""
+
+Compile a transducer defined by *xre*.
+
+May return a pointer to *empty* transducer on non-fatal error. A None pointer is
+returned on fatal error, if abort is not called.
+
+Returns
+-------
+An HfstTransducer pointer.
+"""
+
+%feature("docstring") hfst::XreCompiler::set_expand_definitions
+"""
+
+Whether definitions are expanded.
+
+* v True or False
+"""
+
+%feature("docstring") hfst::XreCompiler::define_xre
+"""
+
+Add a definition macro.
+
+Compilers will replace arcs labeled *name*, with a transducer defined by regular
+expression *xre* in later phases of compilation (if set_expand_definitions(True)
+has been called).
+"""
+
+%feature("docstring") hfst::XreCompiler::define_transducer
+"""
+
+Add a definition macro.
+
+Compilers will replace arcs labeled *name*, with a transducer *transducer* in
+later phases of compilation (if set_expand_definitions(True) has been called).
+"""
+
+%feature("docstring") hfst::XreCompiler::define_function
+"""
+
+todo
+"""
+
+%feature("docstring") hfst::XreCompiler::getOutputToConsole
+"""
+
+(Windows-specific) Whether output is printed to console instead of standard
+output.
+"""
+
+
+// File: exceptions_2____init_____8py.xml
+
+
+// File: ____init_____8py.xml
+
+
+// File: namespacehfst_1_1exceptions.xml
+
+
+// File: namespacehfst_1_1rules.xml
+
+%feature("docstring") hfst::xeroxRules::Rule
+"""
+todo
+"""
+
+%feature("docstring") hfst::xeroxRules::replace
+"""
+todo
+"""
+
+// note: this function is offered as 'replace_left' in package hfst.xerox_rules
+%feature("docstring") hfst::xeroxRules::xerox_replace_left
+"""
+todo
+"""
+
+%feature("docstring") hfst::xeroxRules::replace_leftmost_longest_match
+"""
+todo
+"""
+
+%feature("docstring") hfst::xeroxRules::replace_rightmost_longest_match
+"""
+todo
+"""
+
+%feature("docstring") hfst::xeroxRules::replace_leftmost_shortest_match
+"""
+todo
+"""
+
+%feature("docstring") hfst::xeroxRules::replace_rightmost_shortest_match
+"""
+todo
+"""
+
+%feature("docstring") hfst::xeroxRules::replace_epenthesis
+"""
+todo
+"""
+
+// note: this function is offered as 'restriction' in package hfst.xerox_rules
+%feature("docstring") hfst::xeroxRules::xerox_restriction
+"""
+todo
+"""
+
+%feature("docstring") hfst::xeroxRules::before
+"""
+todo
+"""
+
+%feature("docstring") hfst::xeroxRules::after
+"""
+todo
+"""
+
+
+%feature("docstring") hfst::rules::two_level_if
+"""
+
+A transducer that obligatorily performs the mappings defined by *mappings* in
+the context *context* when the alphabet is *alphabet*.
+
+Parameters
+----------
+* `context` :
+ A pair of transducers where the first transducer defines the left context
+ and the second transducer the right context.
+* `mappings` :
+ A set of mappings (a tuple of string pairs) that the resulting transducer
+ will perform in the context given in *context*.
+* `alphabet` :
+ The set of symbol pairs (a tuple of string pairs) that defines the alphabet
+ (see the example).
+
+For example, a transducer yielded by the following arguments (in pseudcode)
+
+ context = pair( [c|d], [e] )
+ mappings = set(a:b)
+ alphabet = set(a, a:b, b, c, d, e, ...) obligatorily maps the symbol a to b
+if c or d precedes and e follows. (Elsewhere, the mapping of a to b is
+optional). This expression is identical to ![.* [c|d] [a:. & !a:b] [e] .*] Note
+that the alphabet must contain the pair a:b here.
+
+See also: SFST manual
+"""
+
+%feature("docstring") hfst::rules::replace_down
+"""
+
+The same as replace_up, but matching is done on the output side of *mapping*.
+
+See also: replace_up SFST manual.
+"""
+
+%feature("docstring") hfst::rules::replace_down
+"""
+
+The same as replace_down(context, mapping, optional, alphabet) but *mapping* is
+performed in every context.
+
+See also: replace_up
+"""
+
+%feature("docstring") hfst::rules::deep_restriction_and_coercion
+"""
+
+A transducer that is equivalent to the intersection of deep_restriction and
+deep_coercion.
+
+See also: deep_restriction deep_coercion SFST manual.
+
+Parameters
+----------
+* `contexts` :
+ A tuple of HfstTransducer pairs.
+* `mapping` :
+ An HfstTransducer.
+* `alphabet` :
+ A tuple of string pairs.
+"""
+
+%feature("docstring") hfst::rules::surface_restriction_and_coercion
+"""
+
+A transducer that is equivalent to the intersection of surface_restriction and
+surface_coercion.
+
+See also: surface_restriction surface_coercion SFST manual.
+
+Parameters
+----------
+* `contexts` :
+ A tuple of HfstTransducer pairs.
+* `mapping` :
+ An HfstTransducer.
+* `alphabet` :
+ A tuple of string pairs.
+"""
+
+%feature("docstring") hfst::rules::left_replace_down_karttunen
+"""
+
+Inversion of the replace_up and the result needs to be composed on the upper
+side of the input language.
+
+However, matching is done on the output side of *mapping*.
+
+See also: replace_up
+"""
+
+%feature("docstring") hfst::rules::coercion
+"""
+
+A transducer that requires that one of the mappings defined by *mapping* must
+occur in each context in *contexts*.
+
+Symbols outside of the matching substrings are mapped to any symbol allowed by
+*alphabet*.
+
+See also: SFST manual.
+
+Parameters
+----------
+* `contexts` :
+ A tuple of HfstTransducer pairs.
+* `mapping` :
+ An HfstTransducer.
+* `alphabet` :
+ A tuple of string pairs.
+"""
+
+%feature("docstring") hfst::rules::left_replace_down
+"""
+
+Inversion of the replace_up and the result needs to be composed on the upper
+side of the input language.
+
+However, matching is done on the output side of *mapping*.
+
+See also: replace_up
+"""
+
+%feature("docstring") hfst::rules::surface_coercion
+"""
+
+A transducer that specifies that a string from the input language of the
+transducer *mapping* always has to the mapped to one of its output strings
+according to transducer *mapping* if it appears in any of the contexts in
+*contexts*.
+
+Symbols outside of the matching substrings are mapped to any symbol allowed by
+*alphabet*.
+
+See also: SFST manual.
+
+Parameters
+----------
+* `contexts` :
+ A tuple of HfstTransducer pairs.
+* `mapping` :
+ An HfstTransducer.
+* `alphabet` :
+ A tuple of string pairs.
+"""
+
+%feature("docstring") hfst::rules::deep_coercion
+"""
+
+A transducer that specifies that a string from the output language of the
+transducer *mapping* always has to be mapped to one of its input strings
+(according to transducer *mappings*) if it appears in any of the contexts in
+*contexts*.
+
+Symbols outside of the matching substrings are mapped to any symbol allowed by
+*alphabet*.
+
+See also: SFST manual.
+
+Parameters
+----------
+* `contexts` :
+ A tuple of HfstTransducer pairs.
+* `mapping` :
+ An HfstTransducer.
+* `alphabet` :
+ A tuple of string pairs.
+"""
+
+%feature("docstring") hfst::rules::replace_down_karttunen
+"""
+
+TODO: document.
+"""
+
+%feature("docstring") hfst::rules::replace_left
+"""
+
+The same as replace_up, but left context matching is done on the output side of
+*mapping* and right context on the input side of *mapping*.
+
+See also: replace_up
+"""
+
+%feature("docstring") hfst::rules::two_level_if_and_only_if
+"""
+
+A transducer that always performs the mappings defined by *mappings* in the
+context *context* and only in that context, when the alphabet is *alphabet*.
+
+If called with the same arguments as in the example of two_level_if, the
+transducer maps symbol a to b only and only if c or d precedes and e follows.
+The mapping of a to b is obligatory in this context and cannot occur in any
+other context.
+
+The expression is equivalent to ![.* [c|d] [a:. & !a:b] [e] .*] & ![ [ ![.*
+[c|d]] a:b .* ] | [ .* a:b ![[e] .*] ] ]
+
+See also: two_level_if
+"""
+
+%feature("docstring") hfst::rules::surface_restriction
+"""
+
+A transducer that specifies that a string from the input language of the
+transducer *mapping* may only be mapped to one of its output strings (according
+to transducer *mapping*) if it appears in any of the contexts in *contexts*.
+
+Symbols outside of the matching substrings are mapped to any symbol allowed by
+*alphabet*.
+
+See also: SFST manual.
+
+Parameters
+----------
+* `contexts` :
+ A tuple of HfstTransducer pairs.
+* `mapping` :
+ An HfstTransducer.
+* `alphabet` :
+ A tuple of string pairs.
+"""
+
+%feature("docstring") hfst::rules::left_replace_left
+"""
+
+Inversion of the replace_up and the result needs to be composed on the upper
+side of the input language.
+
+However, left context matching is done on the input side of *mapping* and right
+context on the output side of *mapping*.
+
+See also: replace_up */
+"""
+
+%feature("docstring") hfst::rules::deep_restriction
+"""
+
+A transducer that specifies that a string from the output language of the
+transducer *mapping* may only be mapped to one of its input strings (according
+to transducer *mappings*) if it appears in any of the contexts in
+*contexts.Symbols* outside of the matching substrings are mapped to any symbol
+allowed by *alphabet*.
+
+See also: SFST manual.
+
+Parameters
+----------
+* `contexts` :
+ A tuple of HfstTransducer pairs.
+* `mapping` :
+ An HfstTransducer.
+* `alphabet` :
+ A tuple of string pairs.
+"""
+
+%feature("docstring") hfst::rules::left_replace_up
+"""
+
+Inversion of the replace_up and the result needs to be composed on the upper
+side of the input language.
+
+B <- A is the inversion of A -> B. *mapping* is performed in every context.
+
+See also: replace_up
+"""
+
+%feature("docstring") hfst::rules::left_replace_up
+"""
+
+Inversion of the replace_up and the result needs to be composed on the upper
+side of the input language.
+
+B <- A is the inversion of A -> B.
+
+See also: replace_up
+"""
+
+%feature("docstring") hfst::rules::left_replace_right
+"""
+
+Inversion of the replace_up and the result needs to be composed on the upper
+side of the input language.
+
+However, left context matching is done on the output side of *mapping* and right
+context on the input side of *mapping*.
+
+See also: replace_up
+"""
+
+%feature("docstring") hfst::rules::replace_up
+"""
+
+A transducer that performs an upward mapping *mapping* in the context *context*
+when the alphabet is *alphabet*.
+
+*optional* defines whether the mapping is optional.
+
+Parameters
+----------
+* `context` :
+ A pair of transducers where the first transducer defines the left context
+ and the second transducer the right context. Both transducers must be
+ automata, i.e. map strings onto themselves.
+* `mapping` :
+ The mapping (transducer) that the resulting transducer will perform in the
+ context given in *context*.
+* `optional` :
+ Whether the mapping is optional.
+* `alphabet` :
+ The set of symbol pairs that defines the alphabet (a tuple of string pairs).
+
+Each substring s of the input string which is in the input language of the
+transducer *mapping* and whose left context is matched by the expression [.* l]
+(where l is the first element of *context*) and whose right context is matched
+by [r .*] (where r is the second element in the context) is mapped to the
+respective surface strings defined by transducer *mapping*. Any other character
+is mapped to the characters specified in *alphabet*. The left and right contexts
+must be automata (i.e. transducers which map strings onto themselves).
+
+For example, a transducer yielded by the following arguments (in pseudocode)
+
+ context = pair( [c], [c] )
+ mappings = [ a:b a:b ]
+ alphabet = set(a, b, c) would map the string \"caacac\" to \"cbbcac\".
+
+Note that the alphabet must contain the characters a and b, but not the pair a:b
+(unless this replacement is to be allowed everywhere in the context).
+
+Note that replace operations (unlike the two-level rules) have to be combined by
+composition rather than intersection.
+
+Exceptions
+----------
+* `ContextTransducersAreNotAutomataException` :
+
+See also: SFST manual
+"""
+
+%feature("docstring") hfst::rules::replace_up
+"""
+
+The same as replace_up but *mapping* is performed in every context.
+
+See also: replace_up
+"""
+
+%feature("docstring") hfst::rules::two_level_only_if
+"""
+
+A transducer that allows the mappings defined by *mappings* only in the context
+*context*, when the alphabet is *alphabet*.
+
+If called with the same arguments as in the example of two_level_if, the
+transducer allows the mapping of symbol a to b only if c or d precedes and e
+follows. The mapping of a to b is optional in this context but cannot occur in
+any other context.
+
+The expression is equivalent to ![ [ ![.* [c|d]] a:b .* ] | [ .* a:b ![[e] .*] ]
+]
+
+See also: two_level_if
+"""
+
+%feature("docstring") hfst::rules::restriction_and_coercion
+"""
+
+A transducer that is equivalent to the intersection of restriction and coercion
+and requires that the mappings defined by *mapping* occur always and only in the
+given contexts in *contexts*.
+
+Symbols outside of the matching substrings are mapped to any symbol allowed by
+*alphabet*.
+
+See also: restriction coercion SFST manual
+
+Parameters
+----------
+* `contexts` :
+ A tuple of HfstTransducer pairs.
+* `mapping` :
+ An HfstTransducer.
+* `alphabet` :
+ A tuple of string pairs.
+"""
+
+%feature("docstring") hfst::rules::restriction
+"""
+
+A transducer that allows any (substring) mapping defined by *mapping* only if it
+occurs in any of the contexts in *contexts*.
+
+Symbols outside of the matching substrings are mapped to any symbol allowed by
+*alphabet*.
+
+Exceptions
+----------
+* `EmptySetOfContextsException` :
+
+See also: SFST manual.
+
+Parameters
+----------
+* `contexts` :
+ A tuple of HfstTransducer pairs.
+* `mapping` :
+ An HfstTransducer.
+* `alphabet` :
+ A tuple of string pairs.
+"""
+
+%feature("docstring") hfst::rules::replace_right
+"""
+
+The same as replace_up, but left context matching is done on the input side of
+*mapping* and right context on the output side of *mapping*.
+
+See also: replace_up SFST manual.
+"""
+
+
+// File: namespacehfst_1_1types.xml
+
+
+// File: namespacehfst.xml
+
+%feature("docstring") set_default_fst_type
+"""
+
+Set the default implementation type.
+
+Parameters
+----------
+* `impl` :
+ An hfst.ImplementationType.
+
+Set the implementation type (SFST_TYPE, TROPICAL_OPENFST_TYPE, FOMA_TYPE) that
+is used by default by all operations that create transducers. The default value
+is TROPICAL_OPENFST_TYPE
+"""
+
+%feature("docstring") read_att_transducer
+"""
+
+Read next transducer from AT&T file pointed by *f*.
+
+*epsilonstr* defines the symbol used for epsilon in the file.
+
+Parameters
+----------
+* `f` :
+ A python file
+* `epsilonstr` :
+ How epsilon is represented in the file. By default, \"@_EPSILON_SYMBOL_@\"
+ and \"@0@\" are both recognized.
+
+If the file contains several transducers, they must be separated by \"--\"
+lines. In AT&T format, the transition lines are of the form:
+
+ [0-9]+[\\w]+[0-9]+[\\w]+[^\\w]+[\\w]+[^\\w]([\\w]+(-)[0-9]+(\\.[0-9]+))
+
+and final state lines:
+
+ [0-9]+[\\w]+([\\w]+(-)[0-9]+(\\.[0-9]+))
+
+If several transducers are listed in the same file, they are separated by lines
+of two consecutive hyphens \"--\". If the weight
+
+ ([\\w]+(-)[0-9]+(\\.[0-9]+)) is missing, the transition or final state is
+given a zero weight.
+
+NOTE: If transition symbols contains spaces, they must be escaped as '@_SPACE_@'
+because spaces are used as field separators. Both '@0@' and '@_EPSILON_SYMBOL_@'
+are always interpreted as epsilons.
+
+An example:
+
+ 0 1 foo bar 0.3
+ 1 0.5
+ --
+ 0 0.0
+ --
+ --
+ 0 0.0
+ 0 0 a <eps> 0.2
+
+The example lists four transducers in AT&T format: one transducer accepting the
+string pair <'foo','bar'>, one epsilon transducer, one empty transducer and one
+transducer that accepts any number of 'a's and produces an empty string in all
+cases. The transducers can be read with the following commands (from a file
+named 'testfile.att'):
+
+ transducers = []
+ ifile = open('testfile.att', 'r')
+ try:
+ while (True):
+ t = hfst.read_att_transducer(ifile, '<eps>')
+ transducers.append(t)
+ print(\"read one transducer\")
+ except hfst.exceptions.NotValidAttFormatException as e:
+ print(\"Error reading transducer: not valid AT&T format.\")
+ except hfst.exceptions.EndOfStreamException as e:
+ pass
+ ifile.close()
+ print(\"Read %i transducers in total\" % len(transducers))
+
+Epsilon will be represented as hfst.EPSILON in the resulting transducer. The
+argument *epsilon_symbol* only denotes how epsilons are represented in *ifile*.
+
+Bug
+Empty transducers are in theory represented as empty strings in AT&T format.
+However, this sometimes results in them getting interpreted as end-of-file. To
+avoid this, use an empty line instead, i.e. a single newline character.
+
+Exceptions
+----------
+* `NotValidAttFormatException` :
+* `StreamNotReadableException` :
+* `StreamIsClosedException` :
+* `EndOfStreamException` :
+
+See also: #write_att
+"""
+
+%feature("docstring") read_att_input
+"""
+
+Read AT&T input from the user and return a transducer.
+
+Returns
+-------
+An HfstTransducer whose type is hfst.get_default_fst_type(). Read one AT&T line
+at a time from standard input and finally return an equivalent transducer. An
+empty line signals the end of input.
+"""
+
+%feature("docstring") compile_lexc_file
+"""
+
+Compile lexc file *filename* into a transducer.
+
+Parameters
+----------
+* `filename` :
+ The name of the lexc file.
+* `kwargs` :
+ Arguments recognized are: verbosity, with_flags, output.
+* `verbosity` :
+ The verbosity of the compiler, defaults to 0 (silent). Possible values are:
+ 0, 1, 2.
+* `with_flags` :
+ Whether lexc flags are used when compiling, defaults to False.
+* `output` :
+ Where output is printed. Possible values are sys.stdout, sys.stderr, a
+ StringIO, sys.stderr being the default?
+"""
+
+%feature("docstring") fst_type_to_string
+"""
+
+Get a string representation of transducer implementation type *type*.
+
+Parameters
+----------
+* `type` :
+ An hfst.ImplementationType.
+"""
+
+%feature("docstring") epsilon_fst
+"""
+
+Get an epsilon transducer.
+
+Parameters
+----------
+* `weight` :
+ The weight of the final state. Epsilon transducer has one state that is
+ final (with final weight *weight*), i.e. it recognizes the empty string.
+"""
+
+%feature("docstring") empty_fst
+"""
+
+Get an empty transducer.
+
+Empty transducer has one state that is not final, i.e. it does not recognize any
+string.
+"""
+
+%feature("docstring") compile_pmatch_expression
+"""
+
+Compile a pmatch expression into a tuple of transducers.
+
+Parameters
+----------
+* `expr` :
+ A string defining how pmatch is done.
+
+See also: hfst.compile_pmatch_file
+"""
+
+%feature("docstring") start_xfst
+"""
+
+Start interactive xfst compiler.
+
+Parameters
+----------
+* `kwargs` :
+ Arguments recognized are: type, quit_on_fail.
+* `quit_on_fail` :
+ Whether the compiler exits on any error, defaults to False.
+* `type` :
+ Implementation type of the compiler, defaults to
+ hfst.get_default_fst_type().
+"""
+
+%feature("docstring") compile_pmatch_file
+"""
+
+Compile pmatch expressions as defined in *filename* and return a tuple of
+transducers.
+
+An example:
+
+If we have a file named streets.txt that contains:
+
+define CapWord UppercaseAlpha Alpha* ; define StreetWordFr [{avenue} |
+{boulevard} | {rue}] ; define DeFr [ [{de} | {du} | {des} | {de la}] Whitespace
+] | [{d'} | {l'}] ; define StreetFr StreetWordFr (Whitespace DeFr) CapWord+ ;
+regex StreetFr EndTag(FrenchStreetName) ;
+
+we can run:
+
+defs = hfst.compile_pmatch_file('streets.txt') const =
+hfst.PmatchContainer(defs) assert cont.match(\"Je marche seul dans l'avenue des
+Ternes.\") == \"Je marche seul dans l'<FrenchStreetName>avenue des
+Ternes</FrenchStreetName>.\"
+"""
+
+%feature("docstring") fst
+"""
+
+Get a transducer that recognizes one or more paths.
+
+Parameters
+----------
+* `arg` :
+ See example below
+
+Possible inputs:
+
+ One unweighted identity path:
+ 'foo' -> [f o o]
+ Weighted path: a tuple of string and number, e.g.
+ ('foo',1.4)
+ ('bar',-3)
+ ('baz',0)
+ Several paths: a list or a tuple of paths and/or weighted paths, e.g.
+ ['foo', 'bar']
+ ('foo', ('bar',5.0))
+ ('foo', ('bar',5.0), 'baz', 'Foo', ('Bar',2.4))
+ [('foo',-1), ('bar',0), ('baz',3.5)]
+ A dictionary mapping strings to any of the above cases:
+ {'foo':'foo', 'bar':('foo',1.4), 'baz':(('foo',-1),'BAZ')}
+"""
+
+%feature("docstring") get_default_fst_type
+"""
+
+Get default transducer implementation type.
+
+If the default type is not set, it defaults to hfst.ImplementationType.TROPICAL_OPENFST_TYPE
+"""
+
+%feature("docstring") regex
+"""
+
+Get a transducer as defined by regular expression *regexp*.
+
+Parameters
+----------
+* `regexp` :
+ The regular expression defined with Xerox transducer notation.
+* `kwargs` :
+ Argumnets recognized are: error.
+* `error` :
+ Where warnings and errors are printed. Possible values are sys.stdout,
+ sys.stderr (the default), a StringIO or None, indicating a quiet mode.
+"""
+
+%feature("docstring") is_diacritic
+"""
+
+Whether symbol *symbol* is a flag diacritic.
+
+Flag diacritics are of the form
+
+ @[PNDRCU][.][A-Z]+([.][A-Z]+)?@
+"""
+
+%feature("docstring") compile_xfst_file
+"""
+
+Compile (is 'run' a better term?) xfst file *filename*.
+
+Parameters
+----------
+* `filename` :
+ The name of the xfst file.
+* `kwargs` :
+ Arguments recognized are: verbosity, quit_on_fail, output, type.
+* `verbosity` :
+ The verbosity of the compiler, defaults to 0 (silent). Possible values are:
+ 0, 1, 2.
+* `quit_on_fail` :
+ Whether the script is exited on any error, defaults to True.
+* `output` :
+ Where output is printed. Possible values are sys.stdout, sys.stderr, a
+ StringIO, sys.stderr being the default?
+* `type` :
+ Implementation type of the compiler, defaults to
+ hfst.get_default_fst_type().
+"""
+
+%feature("docstring") tokenized_fst
+"""
+
+Get a transducer that recognizes the concatenation of symbols or symbol pairs in
+*arg*.
+
+Parameters
+----------
+* `arg` :
+ The symbols or symbol pairs that form the path to be recognized.
+
+Example
+
+ import libhfst
+ tok = hfst.HfstTokenizer()
+ tok.add_multichar_symbol('foo')
+ tok.add_multichar_symbol('bar')
+ tr = hfst.tokenized_fst(tok.tokenize('foobar', 'foobaz')) will create the
+transducer [foo:foo bar:b 0:a 0:z]
+"""
+
+%feature("docstring") read_att_string
+"""
+
+Read a multiline string *att* and return a transducer.
+
+Parameters
+----------
+* `att` :
+ A string in AT&& format that defines the transducer.
+
+Returns
+-------
+An HfstTransducer whose type is hfst.get_default_fst_type(). Read *att* and
+create a transducer as defined in it.
+"""
+
+%feature("docstring") read_prolog_transducer
+"""
+"""
+
+
+// File: rules_2____init_____8py.xml
+
+
+// File: types_2____init_____8py.xml
+
+
+// File: classhfst_1_1exceptions_1_1ContextTransducersAreNotAutomataException.xml
+
+
+%feature("docstring") hfst::exceptions::ContextTransducersAreNotAutomataException
+"""
+
+Transducers given as rule context are not automata.
+
+See also: hfst.HfstTransducer.is_automaton()
+"""
+
+
+// File: classhfst_1_1exceptions_1_1EmptySetOfContextsException.xml
+
+
+%feature("docstring") hfst::exceptions::EmptySetOfContextsException
+"""
+
+The set of transducer pairs is empty.
+
+Thrown by rule functions.
+"""
+
+
+// File: classhfst_1_1exceptions_1_1EmptyStringException.xml
+
+
+%feature("docstring") hfst::exceptions::EmptyStringException
+"""
+
+An argument string is an empty string.
+
+A transition symbol cannot be an empty string.
+"""
+
+
+// File: classhfst_1_1exceptions_1_1EndOfStreamException.xml
+
+
+%feature("docstring") hfst::exceptions::EndOfStreamException
+"""
+
+The stream is at end.
+
+Thrown by hfst.HfstTransducer hfst.HfstInputStream.__init__
+"""
+
+
+// File: classhfst_1_1exceptions_1_1FlagDiacriticsAreNotIdentitiesException.xml
+
+
+%feature("docstring") hfst::exceptions::FlagDiacriticsAreNotIdentitiesException
+"""
+
+Flag diacritics encountered on one but not the other side of a transition.
+"""
+
+
+// File: classhfst_1_1exceptions_1_1FunctionNotImplementedException.xml
+
+
+%feature("docstring") hfst::exceptions::FunctionNotImplementedException
+"""
+
+Function has not been implemented (yet).
+"""
+
+
+// File: classhfst_1_1exceptions_1_1HfstException.xml
+
+
+%feature("docstring") hfst::exceptions::HfstException
+"""
+
+Base class for HfstExceptions.
+
+Holds its own name and the file and line number where it was thrown.
+"""
+
+%feature("docstring") hfst::exceptions::HfstException::what
+"""
+
+A message describing the error in more detail.
+"""
+
+
+// File: classhfst_1_1exceptions_1_1HfstFatalException.xml
+
+
+%feature("docstring") hfst::exceptions::HfstFatalException
+"""
+
+An error happened probably due to a bug in the HFST code.
+"""
+
+
+// File: classhfst_1_1exceptions_1_1HfstTransducerTypeMismatchException.xml
+
+
+%feature("docstring") hfst::exceptions::HfstTransducerTypeMismatchException
+"""
+
+Two or more HfstTransducers are not of the same type.
+
+Same as HfstTransducerTypeMismatchException ???
+"""
+
+
+// File: classhfst_1_1exceptions_1_1ImplementationTypeNotAvailableException.xml
+
+
+%feature("docstring") hfst::exceptions::ImplementationTypeNotAvailableException
+"""
+
+The library required by the implementation type requested is not linked to HFST.
+"""
+
+
+// File: classhfst_1_1exceptions_1_1IncorrectUtf8CodingException.xml
+
+
+%feature("docstring") hfst::exceptions::IncorrectUtf8CodingException
+"""
+
+String is not valid utf-8.
+
+This exception suggests that an input string is not valid utf8.
+"""
+
+
+// File: classhfst_1_1exceptions_1_1MetadataException.xml
+
+
+%feature("docstring") hfst::exceptions::MetadataException
+"""
+
+A piece of metadata in an HFST header is not supported.
+"""
+
+
+// File: classhfst_1_1exceptions_1_1MissingOpenFstInputSymbolTableException.xml
+
+
+%feature("docstring") hfst::exceptions::MissingOpenFstInputSymbolTableException
+"""
+
+An OpenFst transducer does not have an input symbol table.
+
+When converting from OpenFst to tropical or log HFST, the OpenFst transducer
+must have at least an input symbol table. If the output symbol table is missing,
+it is assumed to be equivalent to the input symbol table.
+
+Thrown by hfst.HfstTransducer.__init__
+"""
+
+
+// File: classhfst_1_1exceptions_1_1NotTransducerStreamException.xml
+
+
+%feature("docstring") hfst::exceptions::NotTransducerStreamException
+"""
+
+The stream does not contain transducers.
+
+Thrown by hfst.HfstTransducer hfst.HfstInputStream.__init__
+
+An example.
+
+ f = open('foofile', 'w')
+ f.write('This is an ordinary text file.\\n')
+ f.close()
+ try:
+ instr = hfst.HfstInputStream('foofile')
+ tr = instr.read()
+ print(tr)
+ instr.close()
+ except hfst.exceptions.NotTransducerStreamException:
+ print(\"Could not print transducer: the file does not contain binary
+transducers.\")
+"""
+
+
+// File: classhfst_1_1exceptions_1_1NotValidAttFormatException.xml
+
+
+%feature("docstring") hfst::exceptions::NotValidAttFormatException
+"""
+
+The stream is not in valid AT&T format.
+
+An example:
+
+ f = open('testfile1.att', 'w')
+ f.write('0 1 a b\\n\\
+ 1 2 c\\n\\
+ 2\\n')
+ f.close()
+ f = hfst.hfst_open('testfile1.att', 'r')
+ try:
+ tr = hfst.read_att(f)
+ except hfst.exceptions.NotValidAttFormatException:
+ print('Could not read file: it is not in valid ATT format.')
+ f.close() thrown by hfst.HfstTransducer.__init__
+"""
+
+
+// File: classhfst_1_1exceptions_1_1NotValidLexcFormatException.xml
+
+
+%feature("docstring") hfst::exceptions::NotValidLexcFormatException
+"""
+
+The input is not in valid LexC format.
+"""
+
+
+// File: classhfst_1_1exceptions_1_1NotValidPrologFormatException.xml
+
+
+%feature("docstring") hfst::exceptions::NotValidPrologFormatException
+"""
+
+The input is not in valid prolog format.
+"""
+
+
+// File: classhfst_1_1exceptions_1_1SpecifiedTypeRequiredException.xml
+
+
+%feature("docstring") hfst::exceptions::SpecifiedTypeRequiredException
+"""
+
+The type of a transducer is not specified.
+
+This exception is thrown when an implementation type argument is
+hfst.ImplementationType.ERROR_TYPE.
+"""
+
+
+// File: classhfst_1_1exceptions_1_1StateIndexOutOfBoundsException.xml
+
+
+%feature("docstring") hfst::exceptions::StateIndexOutOfBoundsException
+"""
+
+The state number argument is not valid.
+
+An example :
+
+ tr = hfst.HfstBasicTransducer()
+ tr.add_state(1)
+ try:
+ w = tr.get_final_weight(2)
+ except hfst.exceptions.StateIndexOutOfBoundsException:
+ print('State number 2 does not exist')
+"""
+
+
+// File: classhfst_1_1exceptions_1_1StateIsNotFinalException.xml
+
+
+%feature("docstring") hfst::exceptions::StateIsNotFinalException
+"""
+
+State is not final (and cannot have a final weight).
+
+An example :
+
+ tr = hfst.HfstBasicTransducer()
+ tr.add_state(1)
+ # An exception is thrown as state number 1 is not final
+ try:
+ w = tr.get_final_weight(1)
+ except hfst.exceptions.StateIsNotFinalException:
+ print(\"State is not final.\")
+
+You should use function hfst.HfstBasicTransducer.is_final_state if you are not
+sure whether a state is final.
+
+Thrown by hfst.HfstBasicTransducer get_final_weight.
+"""
+
+
+// File: classhfst_1_1exceptions_1_1StreamCannotBeWrittenException.xml
+
+
+%feature("docstring") hfst::exceptions::StreamCannotBeWrittenException
+"""
+
+Stream cannot be written.
+
+Thrown by hfst.HfstOutputStream.write and hfst.HfstTransducer.write_att
+"""
+
+
+// File: classhfst_1_1exceptions_1_1StreamIsClosedException.xml
+
+
+%feature("docstring") hfst::exceptions::StreamIsClosedException
+"""
+
+Stream is closed.
+
+Thrown by hfst.HfstTransducer.write_att hfst.HfstOutputStream.write
+
+An example:
+
+ try:
+ tr = hfst.regex('foo')
+ outstr = hfst.HfstOutputStream(filename='testfile')
+ outstr.close()
+ outstr.write(tr)
+ except hfst.exceptions.StreamIsClosedException:
+ print(\"Could not write transducer: stream to file was closed.\")
+"""
+
+
+// File: classhfst_1_1exceptions_1_1StreamNotReadableException.xml
+
+
+%feature("docstring") hfst::exceptions::StreamNotReadableException
+"""
+
+Stream cannot be read.
+"""
+
+
+// File: classhfst_1_1exceptions_1_1SymbolNotFoundException.xml
+
+
+%feature("docstring") hfst::exceptions::SymbolNotFoundException
+"""
+
+A bug in the HFST code.
+"""
+
+
+// File: classhfst_1_1exceptions_1_1TransducerHasWrongTypeException.xml
+
+
+%feature("docstring") hfst::exceptions::TransducerHasWrongTypeException
+"""
+
+Transducer has wrong type.
+
+This exception suggests that an HfstTransducer has not been properly
+initialized, probably due to a bug in the HFST library. Alternatively the
+default constructor of HfstTransducer has been called at some point.
+
+See also: hfst.HfstTransducer.__init__
+"""
+
+
+// File: classhfst_1_1exceptions_1_1TransducerHeaderException.xml
+
+
+%feature("docstring") hfst::exceptions::TransducerHeaderException
+"""
+
+Transducer has a malformed HFST header.
+
+Thrown by hfst.HfstTransducer.__init__ hfst.HfstInputStream
+"""
+
+
+// File: classhfst_1_1exceptions_1_1TransducerIsCyclicException.xml
+
+
+%feature("docstring") hfst::exceptions::TransducerIsCyclicException
+"""
+
+Transducer is cyclic.
+
+Thrown by hfst.HfstTransducer.extract_paths. An example
+
+ transducer = hfst.regex('[a:b]*')
+ try:
+ results = transducer.extract_paths(output='text')
+ print(\"The transducer has %i paths:\" % len(results))
+ print(results)
+ except hfst.exceptions.TransducerIsCyclicException:
+ print(\"The transducer is cyclic and has an infinite number of paths.
+Some of them:\")
+ results = transducer.extract_paths(output='text', max_cycles=5)
+ print(results)
+"""
+
+
+// File: classhfst_1_1exceptions_1_1TransducerTypeMismatchException.xml
+
+
+%feature("docstring") hfst::exceptions::TransducerTypeMismatchException
+"""
+
+Two or more transducers do not have the same type.
+
+This can happen if (1) the calling and called transducer in a binary operation,
+(2) two transducers in a pair of transducers, (3) two consecutive transducers
+coming from an HfstInputStream or (4) two transducers in a function taking two
+or more transducers as arguments do not have the same type.
+
+An example:
+
+ hfst.set_default_fst_type(hfst.ImplementationType.TROPICAL_OPENFST_TYPE)
+ tr1 = hfst.regex('foo')
+ tr2 = hfst.regex('bar')
+ tr2.convert(hfst.ImplementationType.FOMA_TYPE)
+ try:
+ tr1.disjunct(tr2)
+ except hfst.exceptions.TransducerTypeMismatchException:
+ print('The implementation types of transducers must be the same.')
+"""
+
+
+// File: classhfst_1_1exceptions_1_1TransducersAreNotAutomataException.xml
+
+
+%feature("docstring") hfst::exceptions::TransducersAreNotAutomataException
+"""
+
+Transducers are not automata.
+
+Example:
+
+ tr1 = hfst.regex('foo:bar')
+ tr2 = hfst.regex('bar:baz')
+ try:
+ tr1.cross_product(tr2)
+ except hfst.exceptions.TransducersAreNotAutomataException:
+ print('Transducers must be automata in cross product.') This exception
+is thrown by hfst.HfstTransducer.cross_product when either input transducer does
+not have equivalent input and output symbols in all its transitions.
+"""
+
diff --git a/python/hfst/__init__.py b/python/hfst/__init__.py
new file mode 100644
index 0000000..1f40ccc
--- /dev/null
+++ b/python/hfst/__init__.py
@@ -0,0 +1,1261 @@
+"""
+
+Python bindings for HFST finite-state transducer library written in C++.
+
+FUNCTIONS:
+
+ compile_lexc_file
+ compile_pmatch_expression
+ compile_pmatch_file
+ compile_xfst_file
+ concatenate
+ disjunct
+ empty_fst
+ epsilon_fst
+ fsa
+ fst
+ fst_type_to_string
+ get_default_fst_type
+ get_output_to_console
+ intersect
+ is_diacritic
+ read_att_input
+ read_att_string
+ read_att_transducer
+ read_prolog_transducer
+ regex
+ set_default_fst_type
+ set_output_to_console
+ start_xfst
+ tokenized_fst
+
+CLASSES:
+
+ AttReader
+ HfstBasicTransducer
+ HfstBasicTransition
+ HfstInputStream
+ HfstOutputStream
+ HfstTokenizer
+ HfstTransducer
+ ImplementationType
+ LexcCompiler
+ PmatchContainer
+ PrologReader
+ XfstCompiler
+ XreCompiler
+
+"""
+
+__version__ = "3.12.1.0"
+
+import hfst.exceptions
+import hfst.sfst_rules
+import hfst.xerox_rules
+from libhfst import is_diacritic, compile_pmatch_expression, HfstTransducer, HfstOutputStream, HfstInputStream, \
+HfstTokenizer, HfstBasicTransducer, HfstBasicTransition, XreCompiler, LexcCompiler, \
+XfstCompiler, set_default_fst_type, get_default_fst_type, fst_type_to_string, PmatchContainer
+import libhfst
+
+from sys import version
+if int(version[0]) > 2:
+ def unicode(s, c):
+ return s
+
+EPSILON='@_EPSILON_SYMBOL_@'
+UNKNOWN='@_UNKNOWN_SYMBOL_@'
+IDENTITY='@_IDENTITY_SYMBOL_@'
+
+# Windows...
+OUTPUT_TO_CONSOLE=False
+
+def set_output_to_console(val):
+ """
+ (Windows-specific:) set whether output is printed to console instead of standard output.
+ """
+ global OUTPUT_TO_CONSOLE
+ OUTPUT_TO_CONSOLE=val
+
+def get_output_to_console():
+ """
+ (Windows-specific:) get whether output is printed to console instead of standard output.
+ """
+ return OUTPUT_TO_CONSOLE
+
+def start_xfst(**kvargs):
+ """
+ Start interactive xfst compiler.
+
+ Parameters
+ ----------
+ * `kvargs` :
+ Arguments recognized are: type, quit_on_fail.
+ * `quit_on_fail` :
+ Whether the compiler exits on any error, defaults to False.
+ * `type` :
+ Implementation type of the compiler, defaults to
+ hfst.get_default_fst_type().
+ """
+ import sys
+ idle = 'idlelib' in sys.modules
+ if idle:
+ print('It seems that you are running python in in IDLE. Note that all output from xfst will be buffered.')
+ print('This means that all warnings, e.g. about time-consuming operations, will be printed only after the operation is carried out.')
+ print('Consider running python from shell, for example command prompt, if you wish to see output with no delays.')
+
+ type = get_default_fst_type()
+ quit_on_fail = 'OFF'
+ to_console=get_output_to_console()
+ for k,v in kvargs.items():
+ if k == 'type':
+ type = v
+ elif k == 'output_to_console':
+ to_console=v
+ elif k == 'quit_on_fail':
+ if v == True:
+ quit_on_fail='ON'
+ else:
+ print('Warning: ignoring unknown argument %s.' % (k))
+
+ comp = XfstCompiler(type)
+ comp.setReadInteractiveTextFromStdin(True)
+ comp.setReadline(False) # do not mix python and c++ readline
+
+ if to_console and idle:
+ print('Cannot output to console when running libhfst from IDLE.')
+ to_console=False
+ comp.setOutputToConsole(to_console)
+ comp.set('quit-on-fail', quit_on_fail)
+
+ rl_length_1 = 0
+ rl_found = False
+ try:
+ import readline
+ rl_found = True
+ rl_length_1 = readline.get_current_history_length()
+ except ImportError:
+ pass
+
+ import sys
+ expression=""
+ while True:
+ expression += input(comp.get_prompt()).rstrip().lstrip()
+ if len(expression) == 0:
+ continue
+ if expression[-1] == '\\':
+ expression = expression[:-2] + '\n'
+ continue
+ retval = -1
+ if idle:
+ retval = libhfst.hfst_compile_xfst_to_string_one(comp, expression)
+ stdout.write(libhfst.get_hfst_xfst_string_one())
+ else:
+ # interactive command
+ if (expression == "apply down" or expression == "apply up") and rl_found:
+ rl_length_2 = readline.get_current_history_length()
+ while True:
+ try:
+ line = input().rstrip().lstrip()
+ except EOFError:
+ break
+ if expression == "apply down":
+ comp.apply_down(line)
+ elif expression == "apply up":
+ comp.apply_up(line)
+ for foo in range(readline.get_current_history_length() - rl_length_2):
+ readline.remove_history_item(rl_length_2)
+ retval = 0
+ elif expression == "inspect" or expression == "inspect net":
+ print('inspect net not supported')
+ retval = 0
+ else:
+ retval = comp.parse_line(expression + "\n")
+ if retval != 0:
+ print("expression '%s' could not be parsed" % expression)
+ if comp.get("quit-on-fail") == "ON":
+ return
+ if comp.quit_requested():
+ break
+ expression = ""
+
+ if rl_found:
+ for foo in range(readline.get_current_history_length() - rl_length_1):
+ readline.remove_history_item(rl_length_1)
+
+from sys import stdout
+
+def regex(re, **kvargs):
+ """
+ Get a transducer as defined by regular expression *re*.
+
+ Parameters
+ ----------
+ * `re` :
+ The regular expression defined with Xerox transducer notation.
+ * `kvargs` :
+ Arguments recognized are: 'error' and 'definitions'.
+ * `error` :
+ Where warnings and errors are printed. Possible values are sys.stdout,
+ sys.stderr (the default), a StringIO or None, indicating a quiet mode.
+ * `definitions` :
+ A dictionary mapping variable names into transducers.
+
+
+ Regular expression operators:
+
+ ~ complement
+ \ term complement
+ & intersection
+ - minus
+
+ $. contains once
+ $? contains optionally
+ $ contains once or more
+ ( ) optionality
+
+ + Kleene plus
+ * Kleene star
+
+ ./. ignore internally (not yet implemented)
+ / ignoring
+
+ | union
+
+ <> shuffle
+ < before
+ > after
+
+ .o. composition
+ .O. lenient composition
+ .m>. merge right
+ .<m. merge left
+ .x. cross product
+ .P. input priority union
+ .p. output priority union
+ .-u. input minus
+ .-l. output minus
+ `[ ] substitute
+
+ ^n,k catenate from n to k times, inclusive
+ ^>n catenate more than n times
+ ^>n catenate less than n times
+ ^n catenate n times
+
+ .r reverse
+ .i invert
+ .u input side
+ .l output side
+
+ \\\\\\ left quotient
+
+ Two-level rules:
+
+ \<= left restriction
+ <=> left and right arrow
+ <= left arrow
+ => right arrow
+
+ Replace rules:
+
+ -> replace right
+ (->) optionally replace right
+ <- replace left
+ (<-) optionally replace left
+ <-> replace left and right
+ (<->) optionally replace left and right
+ @-> left-to-right longest match
+ @> left-to-right shortest match
+ ->@ right-to-left longest match
+ >@ right-to-left shortest match
+
+ Rule contexts, markers and separators:
+
+ || match contexts on input sides
+ // match left context on output side and right context on input side
+ \\ match left context on input side and right context on output side
+ \/ match contexts on output sides
+ _ center marker
+ ... markup marker
+ ,, rule separator in parallel rules
+ , context separator
+ [. .] match epsilons only once
+
+ Read from file:
+
+ @bin" " read binary transducer
+ @txt" " read transducer in att text format
+ @stxt" " read spaced text
+ @pl" " read transducer in prolog text format
+ @re" " read regular expression
+
+ Symbols:
+
+ .#. word boundary symbol in replacements, restrictions
+ 0 the epsilon
+ ? any token
+ % escape character
+ { } concatenate symbols
+ " " quote symbol
+
+ : pair separator
+ :: weight
+
+ ; end of expression
+ ! starts a comment until end of line
+ # starts a comment until end of line
+ """
+ type_ = get_default_fst_type()
+ to_console=get_output_to_console()
+ import sys
+ err=None
+ defs=None
+
+ for k,v in kvargs.items():
+ if k == 'output_to_console':
+ to_console=v
+ if k == 'error':
+ err=v
+ if k == 'definitions':
+ defs=v;
+ else:
+ print('Warning: ignoring unknown argument %s.' % (k))
+
+ comp = XreCompiler(type_)
+ comp.setOutputToConsole(to_console)
+ if not defs == None:
+ for k,v in defs.items():
+ vtype = str(type(v))
+ if "HfstTransducer" in vtype:
+ comp.define_transducer(k,v)
+ print('defining transducer')
+ else:
+ pass
+
+ if err == None:
+ return libhfst.hfst_regex(comp, re, "")
+ elif err == sys.stdout:
+ return libhfst.hfst_regex(comp, re, "cout")
+ elif err == sys.stderr:
+ return libhfst.hfst_regex(comp, re, "cerr")
+ else:
+ retval = libhfst.hfst_regex(comp, re, "")
+ err.write(unicode(libhfst.get_hfst_regex_error_message(), 'utf-8'))
+ return retval
+
+def _replace_symbols(symbol, epsilonstr=EPSILON):
+ if symbol == epsilonstr:
+ return EPSILON
+ if symbol == "@0@":
+ return EPSILON
+ symbol = symbol.replace("@_SPACE_@", " ")
+ symbol = symbol.replace("@_TAB_@", "\t")
+ symbol = symbol.replace("@_COLON_@", ":")
+ return symbol
+
+def _parse_att_line(line, fsm, epsilonstr=EPSILON):
+ # get rid of extra whitespace
+ line = line.replace('\t',' ')
+ line = " ".join(line.split())
+ fields = line.split(' ')
+ try:
+ if len(fields) == 1:
+ if fields[0] == '': # empty transducer...
+ return True
+ fsm.add_state(int(fields[0]))
+ fsm.set_final_weight(int(fields[0]), 0)
+ elif len(fields) == 2:
+ fsm.add_state(int(fields[0]))
+ fsm.set_final_weight(int(fields[0]), float(fields[1]))
+ elif len(fields) == 4:
+ fsm.add_transition(int(fields[0]), int(fields[1]), _replace_symbols(fields[2]), _replace_symbols(fields[3]), 0)
+ elif len(fields) == 5:
+ fsm.add_transition(int(fields[0]), int(fields[1]), _replace_symbols(fields[2]), _replace_symbols(fields[3]), float(fields[4]))
+ else:
+ return False
+ except ValueError as e:
+ return False
+ return True
+
+def read_att_string(att):
+ """
+ Create a transducer as defined in AT&T format in *att*.
+ """
+ linecount = 0
+ fsm = HfstBasicTransducer()
+ lines = att.split('\n')
+ for line in lines:
+ linecount = linecount + 1
+ if not _parse_att_line(line, fsm):
+ raise hfst.exceptions.NotValidAttFormatException(line, "", linecount)
+ return HfstTransducer(fsm, get_default_fst_type())
+
+def read_att_input():
+ """
+ Create a transducer as defined in AT&T format in user input.
+ An empty line signals the end of input.
+ """
+ linecount = 0
+ fsm = HfstBasicTransducer()
+ while True:
+ line = input().rstrip()
+ if line == "":
+ break
+ linecount = linecount + 1
+ if not _parse_att_line(line, fsm):
+ raise hfst.exceptions.NotValidAttFormatException(line, "", linecount)
+ return HfstTransducer(fsm, get_default_fst_type())
+
+def read_att_transducer(f, epsilonstr=EPSILON, linecount=[0]):
+ """
+ Create a transducer as defined in AT&T format in file *f*. *epsilonstr*
+ defines how epsilons are represented. *linecount* keeps track of the current
+ line in the file.
+ """
+ linecount_ = 0
+ fsm = HfstBasicTransducer()
+ while True:
+ line = f.readline()
+ if line == "":
+ if linecount_ == 0:
+ raise hfst.exceptions.EndOfStreamException("","",0)
+ else:
+ linecount_ = linecount_ + 1
+ break
+ linecount_ = linecount_ + 1
+ if line[0] == '-':
+ break
+ if not _parse_att_line(line, fsm, epsilonstr):
+ raise hfst.exceptions.NotValidAttFormatException(line, "", linecount[0] + linecount_)
+ linecount[0] = linecount[0] + linecount_
+ return HfstTransducer(fsm, get_default_fst_type())
+
+class AttReader:
+ """
+ A class for reading input in AT&T text format and converting it into
+ transducer(s).
+
+ An example that reads AT&T input from file 'testfile.att' where epsilon is
+ represented as \"<eps>\" and creates the corresponding transducers and prints
+ them. If the input cannot be parsed, a message showing the invalid line in AT&T
+ input is printed and reading is stopped.
+
+ with open('testfile.att', 'r') as f:
+ try:
+ r = hfst.AttReader(f, \"<eps>\")
+ for tr in r:
+ print(tr)
+ except hfst.exceptions.NotValidAttFormatException as e:
+ print(e.what())
+ """
+ def __init__(self, f, epsilonstr=EPSILON):
+ """
+ Create an AttReader that reads input from file *f* where the epsilon is
+ represented as *epsilonstr*.
+
+ Parameters
+ ----------
+ * `f` :
+ A python file.
+ * `epsilonstr` :
+ How epsilon is represented in the file. By default, \"@_EPSILON_SYMBOL_@\"
+ and \"@0@\" are both recognized.
+ """
+ self.file = f
+ self.epsilonstr = epsilonstr
+ self.linecount = [0]
+
+ def read(self):
+ """
+ Read next transducer.
+
+ Read next transducer description in AT&T format and return a corresponding
+ transducer.
+
+ Exceptions
+ ----------
+ * `hfst.exceptions.NotValidAttFormatException` :
+ * `hfst.exceptions.EndOfStreamException` :
+ """
+ return read_att_transducer(self.file, self.epsilonstr, self.linecount)
+
+ def __iter__(self):
+ """
+ An iterator to the reader.
+
+ Needed for 'for ... in' statement.
+
+ for transducer in att_reader:
+ print(transducer)
+ """
+ return self
+
+ def next(self):
+ """
+ Return next element (for python version 3).
+
+ Needed for 'for ... in' statement.
+
+ for transducer in att_reader:
+ print(transducer)
+
+ Exceptions
+ ----------
+ * `StopIteration` :
+ """
+ try:
+ return self.read()
+ except hfst.exceptions.EndOfStreamException as e:
+ raise StopIteration
+
+ def __next__(self):
+ """
+ Return next element (for python version 2).
+
+ Needed for 'for ... in' statement.
+
+ for transducer in att_reader:
+ print(transducer)
+
+ Exceptions
+ ----------
+ * `StopIteration` :
+ """
+ return self.next()
+
+def read_prolog_transducer(f, linecount=[0]):
+ """
+ Create a transducer as defined in prolog format in file *f*. *linecount*
+ keeps track of the current line in the file.
+ """
+ linecount_ = 0
+ fsm = HfstBasicTransducer()
+
+ line = ""
+ while(True):
+ line = f.readline()
+ linecount_ = linecount_ + 1
+ if line == "":
+ raise hfst.exceptions.EndOfStreamException("","",linecount[0] + linecount_)
+ line = line.rstrip()
+ if line == "":
+ pass # allow extra prolog separator(s)
+ if line[0] == '#':
+ pass # comment line
+ else:
+ break
+
+ if not libhfst.parse_prolog_network_line(line, fsm):
+ raise hfst.exceptions.NotValidPrologFormatException(line,"",linecount[0] + linecount_)
+
+ while(True):
+ line = f.readline()
+ if (line == ""):
+ retval = HfstTransducer(fsm, get_default_fst_type())
+ retval.set_name(fsm.name)
+ linecount[0] = linecount[0] + linecount_
+ return retval
+ line = line.rstrip()
+ linecount_ = linecount_ + 1
+ if line == "": # prolog separator
+ retval = HfstTransducer(fsm, get_default_fst_type())
+ retval.set_name(fsm.name)
+ linecount[0] = linecount[0] + linecount_
+ return retval
+ if libhfst.parse_prolog_arc_line(line, fsm):
+ pass
+ elif libhfst.parse_prolog_final_line(line, fsm):
+ pass
+ elif libhfst.parse_prolog_symbol_line(line, fsm):
+ pass
+ else:
+ raise hfst.exceptions.NotValidPrologFormatException(line,"",linecount[0] + linecount_)
+
+class PrologReader:
+ """
+ A class for reading input in prolog text format and converting it into
+ transducer(s).
+
+ An example that reads prolog input from file 'testfile.prolog' and creates the
+ corresponding transducers and prints them. If the input cannot be parsed, a
+ message showing the invalid line in prolog input is printed and reading is
+ stopped.
+
+ with open('testfile.prolog', 'r') as f:
+ try:
+ r = hfst.PrologReader(f)
+ for tr in r:
+ print(tr)
+ except hfst.exceptions.NotValidPrologFormatException as e:
+ print(e.what())
+ """
+ def __init__(self, f):
+ """
+ Create a PrologReader that reads input from file *f*.
+
+ Parameters
+ ----------
+ * `f` :
+ A python file.
+ """
+ self.file = f
+ self.linecount = [0]
+
+ def read(self):
+ """
+
+ Read next transducer.
+
+ Read next transducer description in prolog format and return a corresponding
+ transducer.
+
+ Exceptions
+ ----------
+ * `hfst.exceptions.NotValidPrologFormatException` :
+ * `hfst.exceptions.EndOfStreamException` :
+ """
+ return read_prolog_transducer(self.file, self.linecount)
+
+ def __iter__(self):
+ """
+ An iterator to the reader.
+
+ Needed for 'for ... in' statement.
+
+ for transducer in prolog_reader:
+ print(transducer)
+ """
+ return self
+
+ def next(self):
+ """
+ Return next element (for python version 2).
+
+ Needed for 'for ... in' statement.
+
+ for transducer in prolog_reader:
+ print(transducer)
+
+ Exceptions
+ ----------
+ * `StopIteration` :
+ """
+ try:
+ return self.read()
+ except hfst.exceptions.EndOfStreamException as e:
+ raise StopIteration
+
+ def __next__(self):
+ """
+ Return next element (for python version 2).
+
+ Needed for 'for ... in' statement.
+
+ for transducer in prolog_reader:
+ print(transducer)
+
+ Exceptions
+ ----------
+ * `StopIteration` :
+ """
+ return self.next()
+
+def compile_xfst_file(filename, **kvargs):
+ """
+ Compile (run) xfst file *filename*.
+
+ Parameters
+ ----------
+ * `filename` :
+ The name of the xfst file.
+ * `kvargs` :
+ Arguments recognized are: verbosity, quit_on_fail, output, type.
+ * `verbosity` :
+ The verbosity of the compiler, defaults to 0 (silent). Possible values are:
+ 0, 1, 2.
+ * `quit_on_fail` :
+ Whether the script is exited on any error, defaults to True.
+ * `output` :
+ Where output is printed. Possible values are sys.stdout, sys.stderr, a
+ StringIO, sys.stderr being the default?
+ * `type` :
+ Implementation type of the compiler, defaults to
+ hfst.get_default_fst_type().
+
+ Returns
+ -------
+ On success 0, else an integer greater than 0.
+ """
+ if int(version[0]) > 2:
+ pass
+ else:
+ raise RuntimeError('hfst.compile_xfst_file not supported for python version 2')
+ verbosity=0
+ quit_on_fail='ON'
+ type = get_default_fst_type()
+ output=None
+ error=None
+ to_console=get_output_to_console()
+
+ for k,v in kvargs.items():
+ if k == 'verbosity':
+ verbosity=v
+ elif k == 'quit_on_fail':
+ if v == False:
+ quit_on_fail='OFF'
+ elif k == 'output':
+ output=v
+ elif k == 'error':
+ error=v
+ elif k == 'output_to_console':
+ to_console=v
+ else:
+ print('Warning: ignoring unknown argument %s.' % (k))
+
+ if verbosity > 1:
+ print('Compiling with %s implementation...' % fst_type_to_string(type))
+ xfstcomp = XfstCompiler(type)
+ xfstcomp.setOutputToConsole(to_console)
+ xfstcomp.setVerbosity(verbosity > 0)
+ xfstcomp.set('quit-on-fail', quit_on_fail)
+ if verbosity > 1:
+ print('Opening xfst file %s...' % filename)
+ f = open(filename, 'r', encoding='utf-8')
+ data = f.read()
+ f.close()
+ if verbosity > 1:
+ print('File closed...')
+
+ retval=-1
+ import sys
+ from io import StringIO
+
+ # check special case
+ if isinstance(output, StringIO) and isinstance(error, StringIO) and output == error:
+ retval = libhfst.hfst_compile_xfst_to_string_one(xfstcomp, data)
+ output.write(unicode(libhfst.get_hfst_xfst_string_one(), 'utf-8'))
+ else:
+ arg1 = ""
+ arg2 = ""
+ if output == None or output == sys.stdout:
+ arg1 = "cout"
+ if output == sys.stderr:
+ arg1 == "cerr"
+ if error == None or error == sys.stderr:
+ arg2 = "cerr"
+ if error == sys.stdout:
+ arg2 == "cout"
+
+ retval = hfst_compile_xfst(xfstcomp, data, arg1, arg2)
+
+ if isinstance(output, StringIO):
+ output.write(unicode(libhfst.get_hfst_xfst_string_one(), 'utf-8'))
+ if isinstance(error, StringIO):
+ error.write(unicode(libhfst.get_hfst_xfst_string_two(), 'utf-8'))
+
+ if verbosity > 1:
+ print('Parsed file with return value %i (0 indicating succesful parsing).' % retval)
+ return retval
+
+def compile_pmatch_file(filename):
+ """
+ Compile pmatch expressions as defined in *filename* and return a tuple of
+ transducers.
+
+ An example:
+
+ If we have a file named streets.txt that contains:
+
+ define CapWord UppercaseAlpha Alpha* ; define StreetWordFr [{avenue} |
+ {boulevard} | {rue}] ; define DeFr [ [{de} | {du} | {des} | {de la}] Whitespace
+ ] | [{d'} | {l'}] ; define StreetFr StreetWordFr (Whitespace DeFr) CapWord+ ;
+ regex StreetFr EndTag(FrenchStreetName) ;
+
+ we can run:
+
+ defs = hfst.compile_pmatch_file('streets.txt')
+ const = hfst.PmatchContainer(defs)
+ assert cont.match("Je marche seul dans l'avenue desTernes.") ==
+ "Je marche seul dans l'<FrenchStreetName>avenue des Ternes</FrenchStreetName>."
+ """
+ with open(filename, 'r') as myfile:
+ data=myfile.read()
+ myfile.close()
+ defs = compile_pmatch_expression(data)
+ return defs
+
+def compile_sfst_file(filename, **kvargs):
+ """
+ Compile sfst file *filename* into a transducer.
+
+ Parameters
+ ----------
+ * `filename` :
+ The name of the sfst file.
+ * `kvargs` :
+ Arguments recognized are: verbose, output.
+ * `verbose` :
+ Whether sfst file is processed in verbose mode, defaults to False.
+ * `output` :
+ Where output is printed. Possible values are sys.stdout, sys.stderr, a
+ StringI0, sys.stderr being the default. TODO
+
+ Returns
+ -------
+ On success the resulting transducer, else None.
+ """
+ verbosity=False
+ type = get_default_fst_type()
+ output=None
+ to_console=get_output_to_console()
+
+ for k,v in kvargs.items():
+ if k == 'verbose':
+ verbosity=v
+ elif k == 'output':
+ output=v
+ elif k == 'output_to_console':
+ to_console=v
+ else:
+ print('Warning: ignoring unknown argument %s.' % (k))
+
+ retval=None
+ import sys
+ if output == None:
+ retval = libhfst.hfst_compile_sfst(filename, "", verbosity, to_console)
+ elif output == sys.stdout:
+ retval = libhfst.hfst_compile_sfst(filename, "cout", verbosity, to_console)
+ elif output == sys.stderr:
+ retval = libhfst.hfst_compile_sfst(filename, "cerr", verbosity, to_console)
+ else:
+ retval = libhfst.hfst_compile_sfst(filename, "", verbosity, to_console)
+ output.write(unicode(libhfst.get_hfst_sfst_output(), 'utf-8'))
+
+ return retval
+
+def compile_lexc_file(filename, **kvargs):
+ """
+ Compile lexc file *filename* into a transducer.
+
+ Parameters
+ ----------
+ * `filename` :
+ The name of the lexc file.
+ * `kvargs` :
+ Arguments recognized are: verbosity, with_flags, output.
+ * `verbosity` :
+ The verbosity of the compiler, defaults to 0 (silent). Possible values are:
+ 0, 1, 2.
+ * `with_flags` :
+ Whether lexc flags are used when compiling, defaults to False.
+ * `output` :
+ Where output is printed. Possible values are sys.stdout, sys.stderr, a
+ StringIO, sys.stderr being the default.
+
+ Returns
+ -------
+ On success the resulting transducer, else None.
+ """
+ verbosity=0
+ withflags=False
+ alignstrings=False
+ type = get_default_fst_type()
+ output=None
+ to_console=get_output_to_console()
+
+ for k,v in kvargs.items():
+ if k == 'verbosity':
+ verbosity=v
+ elif k == 'with_flags':
+ if v == True:
+ withflags = v
+ elif k == 'align_strings':
+ alignstrings = v
+ elif k == 'output':
+ output=v
+ elif k == 'output_to_console':
+ to_console=v
+ else:
+ print('Warning: ignoring unknown argument %s.' % (k))
+
+ lexccomp = LexcCompiler(type, withflags, alignstrings)
+ lexccomp.setVerbosity(verbosity)
+ lexccomp.setOutputToConsole(to_console)
+
+ retval=-1
+ import sys
+ if output == None:
+ retval = libhfst.hfst_compile_lexc(lexccomp, filename, "")
+ elif output == sys.stdout:
+ retval = libhfst.hfst_compile_lexc(lexccomp, filename, "cout")
+ elif output == sys.stderr:
+ retval = libhfst.hfst_compile_lexc(lexccomp, filename, "cerr")
+ else:
+ retval = libhfst.hfst_compile_lexc(lexccomp, filename, "")
+ output.write(unicode(libhfst.get_hfst_lexc_output(), 'utf-8'))
+
+ return retval
+
+def _is_weighted_word(arg):
+ if isinstance(arg, tuple) and len(arg) == 2 and isinstance(arg[0], str) and isinstance(arg[1], (int, float)):
+ return True
+ return False
+
+def _check_word(arg):
+ if len(arg) == 0:
+ raise RuntimeError('Empty word.')
+ return arg
+
+def fsa(arg):
+ """
+ Get a transducer (automaton in this case) that recognizes one or more paths.
+
+ Parameters
+ ----------
+ * `arg` :
+ See example below
+
+ Possible inputs:
+
+ One unweighted identity path:
+ 'foo' -> [f o o]
+
+ Weighted path: a tuple of string and number, e.g.
+ ('foo',1.4)
+ ('bar',-3)
+ ('baz',0)
+
+ Several paths: a list or a tuple of paths and/or weighted paths, e.g.
+ ['foo', 'bar']
+ ('foo', ('bar',5.0))
+ ('foo', ('bar',5.0), 'baz', 'Foo', ('Bar',2.4))
+ [('foo',-1), ('bar',0), ('baz',3.5)]
+
+ """
+ deftok = HfstTokenizer()
+ retval = HfstBasicTransducer()
+ if isinstance(arg, str):
+ retval.disjunct(deftok.tokenize(_check_word(arg)), 0)
+ elif _is_weighted_word(arg):
+ retval.disjunct(deftok.tokenize(_check_word(arg[0])), arg[1])
+ elif isinstance(arg, tuple) or isinstance(arg, list):
+ for word in arg:
+ if _is_weighted_word(word):
+ retval.disjunct(deftok.tokenize(_check_word(word[0])), word[1])
+ elif isinstance(word, str):
+ retval.disjunct(deftok.tokenize(_check_word(word)), 0)
+ else:
+ raise RuntimeError('Tuple/list element not a string or tuple of string and weight.')
+ else:
+ raise RuntimeError('Not a string or tuple/list of strings.')
+ return HfstTransducer(retval, get_default_fst_type())
+
+def fst(arg):
+ """
+ Get a transducer that recognizes one or more paths.
+
+ Parameters
+ ----------
+ * `arg` :
+ See example below
+
+ Possible inputs:
+
+ One unweighted identity path:
+ 'foo' -> [f o o]
+
+ Weighted path: a tuple of string and number, e.g.
+ ('foo',1.4)
+ ('bar',-3)
+ ('baz',0)
+
+ Several paths: a list or a tuple of paths and/or weighted paths, e.g.
+ ['foo', 'bar']
+ ('foo', ('bar',5.0))
+ ('foo', ('bar',5.0), 'baz', 'Foo', ('Bar',2.4))
+ [('foo',-1), ('bar',0), ('baz',3.5)]
+
+ A dictionary mapping strings to any of the above cases:
+ {'foo':'foo', 'bar':('foo',1.4), 'baz':(('foo',-1),'BAZ')}
+ """
+ if isinstance(arg, dict):
+ retval = regex('[0-0]') # empty transducer
+ for input, output in arg.items():
+ if not isinstance(input, str):
+ raise RuntimeError('Key not a string.')
+ left = fsa(input)
+ right = 0
+ if isinstance(output, str):
+ right = fsa(output)
+ elif isinstance(output, list) or isinstance(output, tuple):
+ right = fsa(output)
+ else:
+ raise RuntimeError('Value not a string or tuple/list of strings.')
+ left.cross_product(right)
+ retval.disjunct(left)
+ return retval
+ return fsa(arg)
+
+def fst_to_fsa(fst, separator=''):
+ """
+ (Experimental)
+
+ Encode a transducer into an automaton, i.e. create a transducer where each
+ transition <in:out> of *fst* is replaced with a transition <inSout:inSout>
+ where 'S' is *separator*, except if the transition symbol on both sides is
+ hfst.EPSILON, hfst.IDENTITY or hfst.UNKNOWN.
+
+ All states and weights of transitions and end states are copied otherwise
+ as such. The alphabet is copied, and new symbols which are created when
+ encoding the transitions, are inserted to it.
+
+ Parameters
+ ----------
+ * `fst` :
+ The transducer.
+ * `separator` :
+ The separator symbol inserted between input and output symbols.
+
+ Examples:
+
+ import hfst
+ foo2bar = hfst.fst({'foo':'bar'})
+
+ creates a transducer [f:b o:a o:r]. Calling
+
+ foobar = hfst.fst_to_fsa(foo2bar)
+
+ will create the transducer [fb:fb oa:oa or:or] and
+
+ foobar = hfst.fst_to_fsa(foo2bar, '^')
+
+ the transducer [f^b:f^b o^a:o^a o^r:o^r].
+
+ """
+ encoded_symbols = libhfst.StringSet()
+ retval = hfst.HfstBasicTransducer(fst)
+ for state in retval.states():
+ arcs = retval.transitions(state)
+ for arc in arcs:
+ input = arc.get_input_symbol()
+ output = arc.get_output_symbol()
+ if (input == output) and ((input == hfst.EPSILON) or (input == hfst.UNKNOWN) or (input == hfst.IDENTITY)):
+ continue
+ symbol = input + separator + output
+ arc.set_input_symbol(symbol)
+ arc.set_output_symbol(symbol)
+ encoded_symbols.insert(symbol)
+ retval.add_symbols_to_alphabet(encoded_symbols)
+ if 'HfstTransducer' in str(type(fst)):
+ return hfst.HfstTransducer(retval)
+ else:
+ return retval
+
+def fsa_to_fst(fsa, separator=''):
+ """
+ (Experimental)
+
+ Decode an encoded automaton back into a transducer, i.e. create a
+ transducer where each transition <inSout:inSout> of *fsa*, where 'S' is
+ the first *separator* found in the compound symbol 'inSout', is replaced
+ with a transition <in:out>.
+
+ If no *separator* is found in the symbol, transition is copied as such. All
+ states and weights of transitions and end states are copied as such. The
+ alphabet is copied, omitting encoded symbols which were decoded according
+ to *separator*. Any new input and output symbols extracted from encoded
+ symbols are added to the alphabet.
+
+ If *separator* is the empty string, 'in' must either be single-character
+ symbol or a special symbol of form '@...@'.
+
+ Parameters
+ ----------
+ * `fsa` :
+ The encoded transducer. Must be an automaton, i.e. for each
+ transition, the input and output symbols must be the same. Else, a
+ RuntimeError is thrown.
+ * `separator` :
+ The symbol separating input and output symbol parts in *fsa*. If it is
+ the empty string, each encoded transition symbol is must be of form
+ 'x...' (single-character input symbol 'x') or '@... at ...' (special
+ symbol as input symbol). Else, a RuntimeError is thrown.
+
+ Examples:
+
+ import hfst
+ foo2bar = hfst.fst({'foo':'bar'}) # creates transducer [f:b o:a o:r]
+ foobar = hfst.fst_to_fsa(foo2bar, '^')
+
+ creates the transducer [f^b:f^b o^a:o^a o^r:o^r]. Then calling
+
+ foo2bar = hfst.fsa_to_fst(foobar, '^')
+
+ will create again the original transducer [f:b o:a o:r].
+ """
+ retval = hfst.HfstBasicTransducer(fsa)
+ encoded_symbols = libhfst.StringSet()
+ for state in retval.states():
+ arcs = retval.transitions(state)
+ for arc in arcs:
+ input = arc.get_input_symbol()
+ output = arc.get_output_symbol()
+ symbols = []
+ if not (input == output):
+ raise RuntimeError('Transition input and output symbols differ.')
+ if input == "":
+ raise RuntimeError('Transition symbol cannot be the empty string.')
+ # separator given:
+ if len(separator) > 0:
+ symbols = input.split(separator, 1)
+ # no separator given:
+ else:
+ index = input.find('@')
+ if not index == 0:
+ symbols.append(input[0])
+ if not input[1] == '':
+ symbols.append(input[1:])
+ else:
+ index = input.find('@', 1)
+ if index == -1:
+ raise RuntimeError('Transition symbol cannot have only one "@" sign.')
+ symbols.append(input[0:index+1])
+ if not input[index+1] == '':
+ symbols.append(input[index+1:])
+ arc.set_input_symbol(symbols[0])
+ arc.set_output_symbol(symbols[-1])
+ # encoded symbol to be removed from alphabet of result
+ if len(symbols) > 1:
+ encoded_symbols.insert(input)
+ retval.remove_symbols_from_alphabet(encoded_symbols)
+ if 'HfstTransducer' in str(type(fsa)):
+ return hfst.HfstTransducer(retval)
+ else:
+ return retval
+
+def tokenized_fst(arg, weight=0):
+ """
+ Get a transducer that recognizes the concatenation of symbols or symbol pairs in
+ *arg*.
+
+ Parameters
+ ----------
+ * `arg` :
+ The symbols or symbol pairs that form the path to be recognized.
+
+ Example
+
+ import hfst
+ tok = hfst.HfstTokenizer()
+ tok.add_multichar_symbol('foo')
+ tok.add_multichar_symbol('bar')
+ tr = hfst.tokenized_fst(tok.tokenize('foobar', 'foobaz'))
+
+ will create the transducer [foo:foo bar:b 0:a 0:z].
+ """
+ retval = HfstBasicTransducer()
+ state = 0
+ if isinstance(arg, list) or isinstance(arg, tuple):
+ for token in arg:
+ if isinstance(token, str):
+ new_state = retval.add_state()
+ retval.add_transition(state, new_state, token, token, 0)
+ state = new_state
+ elif isinstance(token, list) or isinstance(token, tuple):
+ if len(token) == 2:
+ new_state = retval.add_state()
+ retval.add_transition(state, new_state, token[0], token[1], 0)
+ state = new_state
+ elif len(token) == 1:
+ new_state = retval.add_state()
+ retval.add_transition(state, new_state, token, token, 0)
+ state = new_state
+ else:
+ raise RuntimeError('Symbol or symbol pair must be given.')
+ retval.set_final_weight(state, weight)
+ return HfstTransducer(retval, get_default_fst_type())
+ else:
+ raise RuntimeError('Argument must be a list or a tuple')
+
+def empty_fst():
+ """
+ Get an empty transducer.
+
+ Empty transducer has one state that is not final, i.e. it does not recognize any
+ string.
+ """
+ return regex('[0-0]')
+
+def epsilon_fst(weight=0):
+ """
+ Get an epsilon transducer.
+
+ Parameters
+ ----------
+ * `weight` :
+ The weight of the final state. Epsilon transducer has one state that is
+ final (with final weight *weight*), i.e. it recognizes the empty string.
+ """
+ return regex('[0]::' + str(weight))
+
+def concatenate(transducers):
+ """
+ Return a concatenation of *transducers*.
+ """
+ retval = epsilon_fst()
+ for tr in transducers:
+ retval.concatenate(tr)
+ retval.minimize()
+ return retval
+
+def disjunct(transducers):
+ """
+ Return a disjunction of *transducers*.
+ """
+ retval = empty_fst()
+ for tr in transducers:
+ retval.disjunct(tr)
+ retval.minimize()
+ return retval
+
+def intersect(transducers):
+ """
+ Return an intersection of *transducers*.
+ """
+ retval = None
+ for tr in transducers:
+ if retval == None:
+ retval = HfstTransducer(tr)
+ else:
+ retval.intersect(tr)
+ retval.minimize()
+ return retval
+
+class ImplementationType:
+ """
+ Back-end implementation.
+
+ Attributes:
+
+ SFST_TYPE: SFST type, unweighted
+ TROPICAL_OPENFST_TYPE: OpenFst type with tropical weights
+ LOG_OPENFST_TYPE: OpenFst type with logarithmic weights (limited support)
+ FOMA_TYPE: FOMA type, unweighted
+ XFSM_TYPE: XFST type, unweighted (limited support)
+ HFST_OL_TYPE: HFST optimized-lookup type, unweighted
+ HFST_OLW_TYPE: HFST optimized-lookup type, weighted
+ HFST2_TYPE: HFST version 2 legacy type
+ UNSPECIFIED_TYPE: type not specified
+ ERROR_TYPE: (something went wrong)
+
+ """
+ SFST_TYPE = libhfst.SFST_TYPE
+ TROPICAL_OPENFST_TYPE = libhfst.TROPICAL_OPENFST_TYPE
+ LOG_OPENFST_TYPE = libhfst.LOG_OPENFST_TYPE
+ FOMA_TYPE = libhfst.FOMA_TYPE
+ XFSM_TYPE = libhfst.XFSM_TYPE
+ HFST_OL_TYPE = libhfst.HFST_OL_TYPE
+ HFST_OLW_TYPE = libhfst.HFST_OLW_TYPE
+ HFST2_TYPE = libhfst.HFST2_TYPE
+ UNSPECIFIED_TYPE = libhfst.UNSPECIFIED_TYPE
+ ERROR_TYPE = libhfst.ERROR_TYPE
+
diff --git a/python/hfst/exceptions/__init__.py b/python/hfst/exceptions/__init__.py
new file mode 100644
index 0000000..34c682e
--- /dev/null
+++ b/python/hfst/exceptions/__init__.py
@@ -0,0 +1,66 @@
+"""
+class HfstException and its subclasses
+
+CLASSES:
+
+ ContextTransducersAreNotAutomataException
+ EmptySetOfContextsException
+ EmptyStringException
+ EndOfStreamException
+ FlagDiacriticsAreNotIdentitiesException
+ FunctionNotImplementedException
+ HfstException
+ HfstFatalException
+ HfstTransducerTypeMismatchException
+ ImplementationTypeNotAvailableException
+ IncorrectUtf8CodingException
+ MetadataException
+ MissingOpenFstInputSymbolTableException
+ NotTransducerStreamException
+ NotValidAttFormatException
+ NotValidLexcFormatException
+ NotValidPrologFormatException
+ SpecifiedTypeRequiredException
+ StateIndexOutOfBoundsException
+ StateIsNotFinalException
+ StreamCannotBeWrittenException
+ StreamIsClosedException
+ StreamNotReadableException
+ SymbolNotFoundException
+ TransducerHasWrongTypeException
+ TransducerHeaderException
+ TransducerIsCyclicException
+ TransducersAreNotAutomataException
+ TransducerTypeMismatchException
+
+"""
+
+from libhfst import HfstException,\
+HfstTransducerTypeMismatchException,\
+ImplementationTypeNotAvailableException,\
+FunctionNotImplementedException,\
+StreamNotReadableException,\
+StreamCannotBeWrittenException,\
+StreamIsClosedException,\
+EndOfStreamException,\
+TransducerIsCyclicException,\
+NotTransducerStreamException,\
+NotValidAttFormatException,\
+NotValidPrologFormatException,\
+NotValidLexcFormatException,\
+StateIsNotFinalException,\
+ContextTransducersAreNotAutomataException,\
+TransducersAreNotAutomataException,\
+StateIndexOutOfBoundsException,\
+TransducerHeaderException,\
+MissingOpenFstInputSymbolTableException,\
+TransducerTypeMismatchException,\
+EmptySetOfContextsException,\
+SpecifiedTypeRequiredException,\
+HfstFatalException,\
+TransducerHasWrongTypeException,\
+IncorrectUtf8CodingException,\
+EmptyStringException,\
+SymbolNotFoundException,\
+MetadataException,\
+FlagDiacriticsAreNotIdentitiesException
diff --git a/python/hfst/sfst_rules/__init__.py b/python/hfst/sfst_rules/__init__.py
new file mode 100644
index 0000000..8101844
--- /dev/null
+++ b/python/hfst/sfst_rules/__init__.py
@@ -0,0 +1,28 @@
+"""
+functions for creating transducers implementing two-level rules
+"""
+
+from libhfst import two_level_if, \
+two_level_only_if, \
+two_level_if_and_only_if, \
+replace_down, \
+replace_down_karttunen, \
+replace_right, \
+replace_left, \
+replace_up, \
+replace_down, \
+left_replace_up, \
+left_replace_up, \
+left_replace_down, \
+left_replace_down_karttunen, \
+left_replace_left, \
+left_replace_right, \
+restriction, \
+coercion, \
+restriction_and_coercion, \
+surface_restriction, \
+surface_coercion, \
+surface_restriction_and_coercion, \
+deep_restriction, \
+deep_coercion, \
+deep_restriction_and_coercion
diff --git a/python/hfst/xerox_rules/__init__.py b/python/hfst/xerox_rules/__init__.py
new file mode 100644
index 0000000..e424fb8
--- /dev/null
+++ b/python/hfst/xerox_rules/__init__.py
@@ -0,0 +1,35 @@
+"""
+
+Hfst Xerox-type rule functions and classes.
+
+"""
+
+from libhfst import Rule, \
+replace, replace_leftmost_longest_match, \
+replace_rightmost_longest_match, replace_leftmost_shortest_match, \
+replace_rightmost_shortest_match, replace_epenthesis, \
+before, after
+
+# these functions had to be renamed in the swig interface
+# to prevent name collision
+from libhfst import xerox_replace_left as replace_left
+from libhfst import xerox_restriction as restriction
+
+import libhfst
+
+class ReplaceType:
+ """
+ Replace type in Xerox-type rules.
+
+ Attributes:
+
+ REPL_UP: Match contexts on input level
+ REPL_DOWN: Match contexts on output level
+ REPL_RIGHT: Match left contexts on input level and right contexts on output level
+ REPL_LEFT: Match left contexts on output level and right contexts on input level
+
+ """
+ REPL_UP = libhfst.REPL_UP
+ REPL_DOWN = libhfst.REPL_DOWN
+ REPL_RIGHT = libhfst.REPL_RIGHT
+ REPL_LEFT = libhfst.REPL_LEFT
diff --git a/python/hfst_extensions.cc b/python/hfst_extensions.cc
new file mode 100644
index 0000000..c028bca
--- /dev/null
+++ b/python/hfst_extensions.cc
@@ -0,0 +1,38 @@
+namespace hfst {
+
+ // *** Add default implementation type *** //
+
+ hfst::ImplementationType type = hfst::TROPICAL_OPENFST_TYPE;
+ void set_default_fst_type(hfst::ImplementationType t) { type = t; }
+ hfst::ImplementationType get_default_fst_type() { return type; }
+
+ std::string fst_type_to_string(hfst::ImplementationType t) { std::string retval = hfst::implementation_type_to_string(t); return retval; }
+
+ bool is_diacritic(const std::string & symbol) { return hfst::FdOperation::is_diacritic(symbol); }
+
+ HfstTransducer * empty_transducer()
+ {
+ hfst::xre::XreCompiler comp(hfst::get_default_fst_type());
+ return hfst::hfst_regex(comp, "[0 - 0]", "");
+ }
+ HfstTransducer * copy_hfst_transducer(const hfst::HfstTransducer & t) { return new HfstTransducer(t); }
+ HfstTransducer * copy_hfst_transducer_from_basic_transducer(const hfst::implementations::HfstBasicTransducer & t) { return new HfstTransducer(t, type); }
+ HfstTransducer * copy_hfst_transducer_from_basic_transducer(const hfst::implementations::HfstBasicTransducer & t, hfst::ImplementationType impl) { return new HfstTransducer(t, impl); }
+
+ hfst::HfstOutputStream * create_hfst_output_stream(const std::string & filename, hfst::ImplementationType type, bool hfst_format)
+ {
+ if (filename == "") { return new hfst::HfstOutputStream(type, hfst_format); }
+ else { return new hfst::HfstOutputStream(filename, type, hfst_format); }
+ }
+
+ hfst_ol::PmatchContainer * create_pmatch_container(const std::string & filename)
+ {
+ std::ifstream instream(filename.c_str(),
+ std::ifstream::binary);
+ if (!instream.good()) {
+ return NULL;
+ }
+ return new hfst_ol::PmatchContainer(instream);
+ }
+
+}
diff --git a/python/hfst_file_extensions.cc b/python/hfst_file_extensions.cc
new file mode 100644
index 0000000..564a35d
--- /dev/null
+++ b/python/hfst_file_extensions.cc
@@ -0,0 +1,44 @@
+namespace hfst {
+
+ class HfstFile {
+ private:
+ FILE * file;
+ public:
+ HfstFile();
+ ~HfstFile();
+ void set_file(FILE * f);
+ FILE * get_file();
+ void close();
+ void write(const char * str);
+ bool is_eof(void);
+ };
+
+ HfstFile::HfstFile(): file(NULL){};
+ HfstFile::~HfstFile() {};
+ void HfstFile::set_file(FILE * f) { file = f; };
+ FILE * HfstFile::get_file() { return file; };
+ void HfstFile::close() { if (file != stdout && file != stderr && file != stdin) { fclose(file); } };
+ void HfstFile::write(const char * str) { fprintf(file, "%s", str); };
+ bool HfstFile::is_eof(void) { return (feof(file) != 0); };
+
+ HfstFile hfst_open(const char * filename, const char * mode) {
+ FILE * f = fopen(filename, mode);
+ if (f == NULL) { throw StreamNotReadableException("", "", 0); }
+ HfstFile file;
+ file.set_file(f);
+ return file;
+ };
+
+ HfstFile hfst_stdin() {
+ HfstFile file;
+ file.set_file(stdin);
+ return file;
+ };
+
+ HfstFile hfst_stdout() {
+ HfstFile file;
+ file.set_file(stdout);
+ return file;
+ };
+
+}
diff --git a/python/hfst_lexc_extensions.cc b/python/hfst_lexc_extensions.cc
new file mode 100644
index 0000000..5bd845a
--- /dev/null
+++ b/python/hfst_lexc_extensions.cc
@@ -0,0 +1,57 @@
+namespace hfst {
+
+// *** Wrapper variables for IOString outputs *** //
+
+std::string hfst_lexc_output("");
+std::string get_hfst_lexc_output() { return hfst::hfst_lexc_output; }
+
+hfst::HfstTransducer * hfst_compile_lexc(hfst::lexc::LexcCompiler & comp, const std::string & filename, const std::string & error_stream)
+{
+ hfst_lexc_output="";
+
+ if (error_stream == "cout")
+ {
+ comp.set_error_stream(&std::cout);
+ if (comp.getVerbosity() > 1)
+ std::cout << "Parsing the lexc file..." << std::endl;
+ comp.parse(filename.c_str());
+ if (comp.getVerbosity() > 1)
+ std::cout << "Compiling..." << std::endl;
+ hfst::HfstTransducer * retval = comp.compileLexical();
+ if (comp.getVerbosity() > 1)
+ std::cout << "Compilation done." << std::endl;
+ return retval;
+ }
+ else if (error_stream == "cerr")
+ {
+ comp.set_error_stream(&std::cerr);
+ if (comp.getVerbosity() > 1)
+ std::cerr << "Parsing the lexc file..." << std::endl;
+ comp.parse(filename.c_str());
+ if (comp.getVerbosity() > 1)
+ std::cerr << "Compiling..." << std::endl;
+ hfst::HfstTransducer * retval = comp.compileLexical();
+ if (comp.getVerbosity() > 1)
+ std::cerr << "Compilation done." << std::endl;
+ return retval;
+ }
+ else
+ {
+ std::ostringstream os(std::ostringstream::ate);
+ comp.set_error_stream(&os);
+ hfst::set_warning_stream(&os);
+ if (comp.getVerbosity() > 1)
+ os << "Parsing the lexc file..." << std::endl;
+ comp.parse(filename.c_str());
+ if (comp.getVerbosity() > 1)
+ os << "Compiling..." << std::endl;
+ hfst::HfstTransducer * retval = comp.compileLexical();
+ if (comp.getVerbosity() > 1)
+ os << "Compilation done." << std::endl;
+ hfst_lexc_output = os.str();
+ hfst::set_warning_stream(&std::cerr);
+ return retval;
+ }
+}
+
+}
diff --git a/python/hfst_lookup_extensions.cc b/python/hfst_lookup_extensions.cc
new file mode 100644
index 0000000..698df8c
--- /dev/null
+++ b/python/hfst_lookup_extensions.cc
@@ -0,0 +1,95 @@
+namespace hfst {
+
+std::string one_level_paths_to_string(const hfst::HfstOneLevelPaths & paths)
+{
+ std::ostringstream oss;
+ for(hfst::HfstOneLevelPaths::const_iterator it = paths.begin(); it != paths.end(); it++)
+ {
+ for (hfst::StringVector::const_iterator svit = it->second.begin(); svit != it->second.end(); svit++)
+ {
+ oss << *svit;
+ }
+ oss << "\t" << it->first << std::endl;
+ }
+ return oss.str();
+}
+
+hfst::HfstOneLevelPaths extract_output_side(const hfst::HfstTwoLevelPaths & paths)
+{
+ hfst::HfstOneLevelPaths result;
+ for(hfst::HfstTwoLevelPaths::const_iterator it = paths.begin(); it != paths.end(); it++)
+ {
+ hfst::StringVector sv;
+ for (hfst::StringPairVector::const_iterator svit = it->second.begin(); svit != it->second.end(); svit++)
+ {
+ sv.push_back(svit->second);
+ }
+ result.insert(std::pair<float, hfst::StringVector>(it->first, sv));
+ }
+ return result;
+}
+
+std::string two_level_paths_to_string(const hfst::HfstTwoLevelPaths & paths)
+{
+ std::ostringstream oss;
+ for(hfst::HfstTwoLevelPaths::const_iterator it = paths.begin(); it != paths.end(); it++)
+ {
+ std::string input("");
+ std::string output("");
+ for (hfst::StringPairVector::const_iterator svit = it->second.begin(); svit != it->second.end(); svit++)
+ {
+ input += svit->first;
+ output += svit->second;
+ }
+ oss << input << ":" << output << "\t" << it->first << std::endl;
+ }
+ return oss.str();
+}
+
+// *** Wrappers for lookup functions *** //
+
+HfstOneLevelPaths lookup_vector(const hfst::HfstTransducer * tr, bool fd, const StringVector& s, int limit = -1, double time_cutoff = 0.0) throw(FunctionNotImplementedException)
+{
+ if (tr->get_type() == hfst::HFST_OL_TYPE || tr->get_type() == hfst::HFST_OLW_TYPE)
+ {
+ if (fd)
+ { return *(tr->lookup_fd(s, limit, time_cutoff)); }
+ else
+ { return *(tr->lookup(s, limit, time_cutoff)); }
+ }
+ hfst::HfstTransducer input(s, tr->get_type());
+ input.compose(*(tr));
+ input.minimize();
+ hfst::HfstTwoLevelPaths result;
+ if (fd)
+ { input.extract_paths_fd(result, limit, -1); }
+ else
+ { input.extract_paths(result, limit, -1); }
+ return hfst::extract_output_side(result);
+}
+
+HfstOneLevelPaths lookup_string(const hfst::HfstTransducer * tr, bool fd, const std::string& s, int limit = -1, double time_cutoff = 0.0) throw(FunctionNotImplementedException)
+{
+ if (tr->get_type() == hfst::HFST_OL_TYPE || tr->get_type() == hfst::HFST_OLW_TYPE)
+ {
+ if (fd)
+ { return *(tr->lookup_fd(s, limit, time_cutoff)); }
+ else
+ { return *(tr->lookup(s, limit, time_cutoff)); }
+ }
+ hfst::StringSet alpha = tr->get_alphabet();
+ hfst::HfstTokenizer tok;
+ for (hfst::StringSet::const_iterator it = alpha.begin(); it != alpha.end(); it++)
+ { tok.add_multichar_symbol(*it); }
+ hfst::HfstTransducer input(s, tok, tr->get_type());
+ input.compose(*(tr));
+ input.minimize();
+ hfst::HfstTwoLevelPaths result;
+ if (fd)
+ { input.extract_paths_fd(result, limit, -1); }
+ else
+ { input.extract_paths(result, limit, -1); }
+ return hfst::extract_output_side(result);
+}
+
+}
diff --git a/python/hfst_pmatch_extensions.cc b/python/hfst_pmatch_extensions.cc
new file mode 100644
index 0000000..c731c78
--- /dev/null
+++ b/python/hfst_pmatch_extensions.cc
@@ -0,0 +1,93 @@
+#include "parsers/PmatchCompiler.h"
+
+namespace hfst {
+
+ //hfst::ImplementationType get_default_fst_type();
+
+// Mostly copied from file 'tools/src/hfst-pmatch2fst.cc'.
+// TODO: HfstTransducer pointers in variable 'definitions' need to be deleted manually?
+std::vector<hfst::HfstTransducer> compile_pmatch_expression(const std::string & pmatch)
+{
+ std::vector<hfst::HfstTransducer> retval;
+ hfst::pmatch::PmatchCompiler comp(hfst::TROPICAL_OPENFST_TYPE);
+ comp.set_verbose(false/*verbose*/);
+ comp.set_flatten(false/*flatten*/);
+ std::map<std::string, hfst::HfstTransducer*> definitions;
+ try {
+ definitions = comp.compile(pmatch);
+ } catch(...) {
+ throw HfstException(); // TODO
+ }
+
+ // A dummy transducer with an alphabet with all the symbols
+ hfst::HfstTransducer harmonizer(hfst::TROPICAL_OPENFST_TYPE);
+
+ // First we need to collect a unified alphabet from all the transducers.
+ hfst::StringSet symbols_seen;
+ for (std::map<std::string, hfst::HfstTransducer *>::const_iterator it =
+ definitions.begin(); it != definitions.end(); ++it) {
+ hfst::StringSet string_set = it->second->get_alphabet();
+ for (hfst::StringSet::const_iterator sym = string_set.begin();
+ sym != string_set.end(); ++sym) {
+ if (symbols_seen.count(*sym) == 0) {
+ harmonizer.disjunct(hfst::HfstTransducer(*sym, hfst::TROPICAL_OPENFST_TYPE));
+ symbols_seen.insert(*sym);
+ }
+ }
+ }
+ if (symbols_seen.size() == 0) {
+ // We don't recognise anything, go home early
+ std::cerr << "Empty ruleset, nothing to write\n";
+ throw HfstException(); // TODO
+ }
+
+ // Then we convert it...
+ harmonizer.convert(hfst::HFST_OLW_TYPE);
+ // Use these for naughty intermediate steps to make sure
+ // everything has the same alphabet
+ hfst::HfstBasicTransducer * intermediate_tmp;
+ hfst_ol::Transducer * harmonized_tmp;
+ hfst::HfstTransducer * output_tmp;
+
+ // When done compiling everything, look for TOP and output it first.
+ if (definitions.count("TOP") == 1) {
+ intermediate_tmp = hfst::implementations::ConversionFunctions::
+ hfst_transducer_to_hfst_basic_transducer(*definitions["TOP"]);
+ harmonized_tmp = hfst::implementations::ConversionFunctions::
+ hfst_basic_transducer_to_hfst_ol(intermediate_tmp,
+ true, // weighted
+ "", // no special options
+ &harmonizer); // harmonize with this
+ output_tmp = hfst::implementations::ConversionFunctions::
+ hfst_ol_to_hfst_transducer(harmonized_tmp);
+ output_tmp->set_name("TOP");
+ retval.push_back(*output_tmp);
+ delete definitions["TOP"];
+ definitions.erase("TOP");
+ delete intermediate_tmp;
+ delete output_tmp;
+
+ for (std::map<std::string, hfst::HfstTransducer *>::iterator it =
+ definitions.begin(); it != definitions.end(); ++it) {
+ intermediate_tmp = hfst::implementations::ConversionFunctions::
+ hfst_transducer_to_hfst_basic_transducer(*(it->second));
+ harmonized_tmp = hfst::implementations::ConversionFunctions::
+ hfst_basic_transducer_to_hfst_ol(intermediate_tmp,
+ true, // weighted
+ "", // no special options
+ &harmonizer); // harmonize with this
+ output_tmp = hfst::implementations::ConversionFunctions::
+ hfst_ol_to_hfst_transducer(harmonized_tmp);
+ output_tmp->set_name(it->first);
+ retval.push_back(*output_tmp);
+ delete it->second;
+ delete intermediate_tmp;
+ delete output_tmp;
+ }
+ } else {
+ std::cerr << "Empty ruleset, nothing to write\n";
+ throw HfstException(); // TODO
+ }
+ return retval;
+ }
+}
diff --git a/python/hfst_prolog_extensions.cc b/python/hfst_prolog_extensions.cc
new file mode 100644
index 0000000..23d9fbe
--- /dev/null
+++ b/python/hfst_prolog_extensions.cc
@@ -0,0 +1,28 @@
+namespace hfst
+{
+
+ bool parse_prolog_network_line(const std::string & line, hfst::implementations::HfstBasicTransducer * graph)
+ {
+ bool retval = hfst::implementations::HfstBasicTransducer::parse_prolog_network_line(line, *graph);
+ return retval;
+ }
+
+ bool parse_prolog_arc_line(const std::string & line, hfst::implementations::HfstBasicTransducer * graph)
+ {
+ bool retval = hfst::implementations::HfstBasicTransducer::parse_prolog_arc_line(line, *graph);
+ return retval;
+ }
+
+ bool parse_prolog_symbol_line(const std::string & line, hfst::implementations::HfstBasicTransducer * graph)
+ {
+ bool retval = hfst::implementations::HfstBasicTransducer::parse_prolog_symbol_line(line, *graph);
+ return retval;
+ }
+
+ bool parse_prolog_final_line(const std::string & line, hfst::implementations::HfstBasicTransducer * graph)
+ {
+ bool retval = hfst::implementations::HfstBasicTransducer::parse_prolog_final_line(line, *graph);
+ return retval;
+ }
+
+}
diff --git a/python/hfst_regex_extensions.cc b/python/hfst_regex_extensions.cc
new file mode 100644
index 0000000..6951066
--- /dev/null
+++ b/python/hfst_regex_extensions.cc
@@ -0,0 +1,34 @@
+namespace hfst {
+
+// *** Wrapper variables for IOString outputs *** //
+
+std::string hfst_regex_error_message("");
+std::string get_hfst_regex_error_message() { return hfst::hfst_regex_error_message; }
+
+hfst::HfstTransducer * hfst_regex(hfst::xre::XreCompiler & comp, const std::string & regex_string, const std::string & error_stream)
+{
+ hfst_regex_error_message="";
+
+ if (error_stream == "cout")
+ {
+ comp.set_error_stream(&std::cout);
+ return comp.compile(regex_string);
+ }
+ else if (error_stream == "cerr")
+ {
+ comp.set_error_stream(&std::cerr);
+ return comp.compile(regex_string);
+ }
+ else
+ {
+ std::ostringstream os(std::ostringstream::ate);
+ comp.set_error_stream(&os);
+ hfst::set_warning_stream(&os);
+ hfst::HfstTransducer * retval = comp.compile(regex_string);
+ hfst_regex_error_message = os.str();
+ hfst::set_warning_stream(&std::cerr);
+ return retval;
+ }
+}
+
+}
diff --git a/python/hfst_rules_extensions.cc b/python/hfst_rules_extensions.cc
new file mode 100644
index 0000000..dee3254
--- /dev/null
+++ b/python/hfst_rules_extensions.cc
@@ -0,0 +1,174 @@
+namespace hfst {
+// *** Wrappers for rule functions. *** //
+
+namespace hfst_rules {
+
+ HfstTransducer two_level_if(const HfstTransducerPair & context, const StringPairSet & mappings, const StringPairSet & alphabet)
+ {
+ hfst::HfstTransducerPair context_(context);
+ StringPairSet mappings_(mappings);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::two_level_if(context_, mappings_, alphabet_);
+ }
+ HfstTransducer two_level_only_if(const HfstTransducerPair &context, const StringPairSet &mappings, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPair context_(context);
+ StringPairSet mappings_(mappings);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::two_level_only_if(context_, mappings_, alphabet_);
+ }
+ HfstTransducer two_level_if_and_only_if(const HfstTransducerPair &context, const StringPairSet &mappings, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPair context_(context);
+ StringPairSet mappings_(mappings);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::two_level_if_and_only_if(context_, mappings_, alphabet_);
+ }
+ HfstTransducer replace_down(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPair context_(context);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::replace_down(context_, mapping_, optional, alphabet_);
+ }
+ HfstTransducer replace_down_karttunen(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPair context_(context);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::replace_down_karttunen(context_, mapping_, optional, alphabet_);
+ }
+ HfstTransducer replace_right(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPair context_(context);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::replace_right(context_, mapping_, optional, alphabet_);
+ }
+ HfstTransducer replace_left(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPair context_(context);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::replace_left(context_, mapping_, optional, alphabet_);
+ }
+ HfstTransducer replace_up(const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::replace_up(mapping_, optional, alphabet_);
+ }
+ HfstTransducer replace_down(const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::replace_down(mapping_, optional, alphabet_);
+ }
+ HfstTransducer left_replace_up(const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::left_replace_up(mapping_, optional, alphabet_);
+ }
+ HfstTransducer left_replace_up(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPair context_(context);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::left_replace_up(context_, mapping_, optional, alphabet_);
+ }
+ HfstTransducer left_replace_down(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPair context_(context);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::left_replace_down(context_, mapping_, optional, alphabet_);
+ }
+ HfstTransducer left_replace_down_karttunen(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPair context_(context);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::left_replace_down_karttunen(context_, mapping_, optional, alphabet_);
+ }
+ HfstTransducer left_replace_left(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPair context_(context);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::left_replace_left(context_, mapping_, optional, alphabet_);
+ }
+ HfstTransducer left_replace_right(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPair context_(context);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::left_replace_right(context_, mapping_, optional, alphabet_);
+ }
+ HfstTransducer restriction(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPairVector contexts_(contexts);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::restriction(contexts_, mapping_, alphabet_);
+ }
+ HfstTransducer coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPairVector contexts_(contexts);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::coercion(contexts_, mapping_, alphabet_);
+ }
+ HfstTransducer restriction_and_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPairVector contexts_(contexts);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::restriction_and_coercion(contexts_, mapping_, alphabet_);
+ }
+ HfstTransducer surface_restriction(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPairVector contexts_(contexts);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::surface_restriction(contexts_, mapping_, alphabet_);
+ }
+ HfstTransducer surface_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPairVector contexts_(contexts);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::surface_coercion(contexts_, mapping_, alphabet_);
+ }
+ HfstTransducer surface_restriction_and_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPairVector contexts_(contexts);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::surface_restriction_and_coercion(contexts_, mapping_, alphabet_);
+ }
+ HfstTransducer deep_restriction(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPairVector contexts_(contexts);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::deep_restriction(contexts_, mapping_, alphabet_);
+ }
+ HfstTransducer deep_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPairVector contexts_(contexts);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::deep_coercion(contexts_, mapping_, alphabet_);
+ }
+ HfstTransducer deep_restriction_and_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
+ {
+ hfst::HfstTransducerPairVector contexts_(contexts);
+ hfst::HfstTransducer mapping_(mapping);
+ StringPairSet alphabet_(alphabet);
+ return hfst::rules::deep_restriction_and_coercion(contexts_, mapping_, alphabet_);
+ }
+
+}
+
+}
diff --git a/python/hfst_sfst_extensions.cc b/python/hfst_sfst_extensions.cc
new file mode 100644
index 0000000..7d7f4be
--- /dev/null
+++ b/python/hfst_sfst_extensions.cc
@@ -0,0 +1,45 @@
+namespace hfst {
+
+// *** Wrapper variables for IOString outputs, TODO *** //
+
+std::string hfst_sfst_output("");
+std::string get_hfst_sfst_output() { return hfst::hfst_sfst_output; }
+
+hfst::HfstTransducer * hfst_compile_sfst(const std::string & filename, const std::string & error_stream, bool verbose, bool output_to_console)
+{
+ (void)error_stream;
+ (void)output_to_console;
+
+ FILE * inputfile = fopen(filename.c_str(), "rb");
+ if (inputfile == NULL)
+ return NULL;
+
+ bool unknown_in_use = hfst::get_unknown_symbols_in_use();
+ hfst::set_unknown_symbols_in_use(false);
+
+ hfst::SfstCompiler * compiler = new hfst::SfstCompiler(hfst::get_default_fst_type(), verbose);
+ compiler->set_input(inputfile);
+ compiler->set_filename(filename);
+
+ try
+ {
+ compiler->parse();
+ hfst::set_unknown_symbols_in_use(unknown_in_use);
+ fclose(inputfile);
+ return compiler->get_result();
+ }
+ catch (HfstException e)
+ {
+ hfst::set_unknown_symbols_in_use(unknown_in_use);
+ printf("\nAn error happened when compiling sfst file \"%s\"\n", filename.c_str());
+ return NULL;
+ }
+ catch(const char* p)
+ {
+ hfst::set_unknown_symbols_in_use(unknown_in_use);
+ printf("\nAn error happened when compiling sfst file \"%s\"\n", filename.c_str());
+ return NULL;
+ }
+}
+
+}
diff --git a/python/hfst_xfst_extensions.cc b/python/hfst_xfst_extensions.cc
new file mode 100644
index 0000000..eb7de47
--- /dev/null
+++ b/python/hfst_xfst_extensions.cc
@@ -0,0 +1,68 @@
+namespace hfst {
+
+// *** Wrapper variables for IOString outputs *** //
+
+std::string hfst_xfst_string_one("");
+char * get_hfst_xfst_string_one() { return strdup(hfst::hfst_xfst_string_one.c_str()); }
+std::string hfst_xfst_string_two("");
+char * get_hfst_xfst_string_two() { return strdup(hfst::hfst_xfst_string_two.c_str()); }
+
+int hfst_compile_xfst_to_string_one(hfst::xfst::XfstCompiler & comp, std::string input)
+{
+ hfst::hfst_xfst_string_one="";
+ hfst::hfst_xfst_string_two="";
+
+ std::ostringstream os(std::ostringstream::ate);
+ hfst::set_warning_stream(&os);
+ comp.set_output_stream(os);
+ comp.set_error_stream(os);
+ int retval = comp.parse_line(input);
+ hfst::hfst_xfst_string_one = os.str();
+ hfst::set_warning_stream(&std::cerr);
+ return retval;
+}
+
+int hfst_compile_xfst(hfst::xfst::XfstCompiler & comp, std::string input, const std::string & output_stream, const std::string & error_stream)
+{
+ hfst::hfst_xfst_string_one="";
+ hfst::hfst_xfst_string_two="";
+ std::ostringstream * os1 = NULL;
+ std::ostringstream * os2 = NULL;
+
+ if (output_stream == "cout")
+ comp.set_output_stream(std::cout);
+ else if (output_stream == "cerr")
+ comp.set_output_stream(std::cerr);
+ else {
+ os1 = new std::ostringstream(std::ostringstream::ate);
+ comp.set_output_stream(*os1);
+ }
+
+ if (error_stream == "cout") {
+ comp.set_error_stream(std::cout);
+ hfst::set_warning_stream(&std::cout);
+ }
+ else if (error_stream == "cerr")
+ comp.set_error_stream(std::cerr);
+ else {
+ os2 = new std::ostringstream(std::ostringstream::ate);
+ comp.set_error_stream(*os2);
+ hfst::set_warning_stream(os2);
+ }
+
+ int retval = comp.parse_line(input);
+ hfst::set_warning_stream(&std::cerr);
+
+ if (output_stream == "") {
+ hfst::hfst_xfst_string_one = os1->str();
+ delete os1;
+ }
+ if (error_stream == "") {
+ hfst::hfst_xfst_string_two = os2->str();
+ delete os2;
+ }
+
+ return retval;
+}
+
+}
diff --git a/python/libhfst.i b/python/libhfst.i
index ae02082..ad01b99 100644
--- a/python/libhfst.i
+++ b/python/libhfst.i
@@ -1,13 +1,19 @@
-// Copyright (c) 2016 University of Helsinki
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
+// Copyright (c) 2016 University of Helsinki
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
-// See the file COPYING included with this distribution for more
+// See the file COPYING included with this distribution for more
// information.
+// This is a swig interface file that is used to create python bindings for HFST.
+// Everything will be visible under module 'libhfst', but will be wrapped under
+// package 'hfst' and its subpackages 'hfst.exceptions' and 'hfst.rules' (see
+// folder 'hfst' in the current directory).
+
%module libhfst
+// Needed for type conversions between c++ and python.
%include "std_string.i"
%include "std_vector.i"
%include "std_pair.i"
@@ -15,12 +21,14 @@
%include "std_map.i"
%include "exception.i"
-%feature("autodoc", "3");
+// %feature("autodoc", "3");
+// We want warnings to be printed to standard error.
%init %{
hfst::set_warning_stream(&std::cerr);
%}
+// Make swig aware of what hfst offers.
%{
#define HFSTIMPORT
#include "HfstDataTypes.h"
@@ -30,622 +38,39 @@
#include "HfstExceptionDefs.h"
#include "HfstTokenizer.h"
#include "HfstFlagDiacritics.h"
-//#include "HfstRules.h" ???
+#include "HfstXeroxRules.h"
#include "parsers/XreCompiler.h"
#include "parsers/LexcCompiler.h"
-#include "parsers/PmatchCompiler.h"
#include "parsers/XfstCompiler.h"
-#include "implementations/HfstTransitionGraph.h"
+#include "parsers/SfstCompiler.h"
+#include "implementations/HfstBasicTransition.h"
+#include "implementations/HfstBasicTransducer.h"
#include "implementations/optimized-lookup/pmatch.h"
-
-// todo instead: #include "hfst_extensions.h"
-
-namespace hfst {
-
- class HfstFile {
- private:
- FILE * file;
- public:
- HfstFile();
- ~HfstFile();
- void set_file(FILE * f);
- FILE * get_file();
- void close();
- void write(const char * str);
- bool is_eof(void);
- };
-
- HfstFile::HfstFile(): file(NULL){};
- HfstFile::~HfstFile() {};
- void HfstFile::set_file(FILE * f) { file = f; };
- FILE * HfstFile::get_file() { return file; };
- void HfstFile::close() { if (file != stdout && file != stderr && file != stdin) { fclose(file); } };
- void HfstFile::write(const char * str) { fprintf(file, "%s", str); };
- bool HfstFile::is_eof(void) { return (feof(file) != 0); };
-
- HfstFile hfst_open(const char * filename, const char * mode) {
- FILE * f = fopen(filename, mode);
- HfstFile file;
- file.set_file(f);
- return file;
- };
-
- HfstFile hfst_stdin() {
- HfstFile file;
- file.set_file(stdin);
- return file;
- };
-
- HfstFile hfst_stdout() {
- HfstFile file;
- file.set_file(stdout);
- return file;
- };
-
-typedef std::vector<float> FloatVector;
-
-// Add Default implementation type
-hfst::ImplementationType type = hfst::TROPICAL_OPENFST_TYPE;
-void set_default_fst_type(hfst::ImplementationType t) { type = t; }
-hfst::ImplementationType get_default_fst_type() { return type; }
-
-std::string fst_type_to_string(hfst::ImplementationType t)
-{
- std::string retval = hfst::implementation_type_to_string(t);
- return retval;
-}
-bool is_diacritic(const std::string & symbol)
-{
- return hfst::FdOperation::is_diacritic(symbol);
-}
-
-// Copy functions
-HfstTransducer * copy_hfst_transducer(const hfst::HfstTransducer & t)
-{
- return new HfstTransducer(t);
-}
-HfstTransducer * copy_hfst_transducer_from_basic_transducer(const hfst::implementations::HfstBasicTransducer & t)
-{
- return new HfstTransducer(t, type);
-}
-HfstTransducer * copy_hfst_transducer_from_basic_transducer(const hfst::implementations::HfstBasicTransducer & t, hfst::ImplementationType impl)
-{
- return new HfstTransducer(t, impl);
-}
-
-// Mostly copied from file 'tools/src/hfst-pmatch2fst.cc'.
-// TODO: HfstTransducer pointers in variable 'definitions' need to be deleted manually?
-std::vector<hfst::HfstTransducer> compile_pmatch_expression(const std::string & pmatch)
-{
- std::vector<hfst::HfstTransducer> retval;
- hfst::pmatch::PmatchCompiler comp(get_default_fst_type());
- comp.set_verbose(false/*verbose*/);
- comp.set_flatten(false/*flatten*/);
- std::map<std::string, hfst::HfstTransducer*> definitions = comp.compile(pmatch);
-
- // A dummy transducer with an alphabet with all the symbols
- hfst::HfstTransducer harmonizer(get_default_fst_type());
-
- // First we need to collect a unified alphabet from all the transducers.
- hfst::StringSet symbols_seen;
- for (std::map<std::string, hfst::HfstTransducer *>::const_iterator it =
- definitions.begin(); it != definitions.end(); ++it) {
- hfst::StringSet string_set = it->second->get_alphabet();
- for (hfst::StringSet::const_iterator sym = string_set.begin();
- sym != string_set.end(); ++sym) {
- if (symbols_seen.count(*sym) == 0) {
- harmonizer.disjunct(hfst::HfstTransducer(*sym, get_default_fst_type()));
- symbols_seen.insert(*sym);
- }
- }
- }
- if (symbols_seen.size() == 0) {
- // We don't recognise anything, go home early
- std::cerr << "Empty ruleset, nothing to write\n";
- throw HfstException(); // TODO
- }
-
- // Then we convert it...
- harmonizer.convert(hfst::HFST_OLW_TYPE);
- // Use these for naughty intermediate steps to make sure
- // everything has the same alphabet
- hfst::HfstBasicTransducer * intermediate_tmp;
- hfst_ol::Transducer * harmonized_tmp;
- hfst::HfstTransducer * output_tmp;
-
- // When done compiling everything, look for TOP and output it first.
- if (definitions.count("TOP") == 1) {
- intermediate_tmp = hfst::implementations::ConversionFunctions::
- hfst_transducer_to_hfst_basic_transducer(*definitions["TOP"]);
- harmonized_tmp = hfst::implementations::ConversionFunctions::
- hfst_basic_transducer_to_hfst_ol(intermediate_tmp,
- true, // weighted
- "", // no special options
- &harmonizer); // harmonize with this
- output_tmp = hfst::implementations::ConversionFunctions::
- hfst_ol_to_hfst_transducer(harmonized_tmp);
- output_tmp->set_name("TOP");
- retval.push_back(*output_tmp);
- delete definitions["TOP"];
- definitions.erase("TOP");
- delete intermediate_tmp;
- delete output_tmp;
-
- for (std::map<std::string, hfst::HfstTransducer *>::iterator it =
- definitions.begin(); it != definitions.end(); ++it) {
- intermediate_tmp = hfst::implementations::ConversionFunctions::
- hfst_transducer_to_hfst_basic_transducer(*(it->second));
- harmonized_tmp = hfst::implementations::ConversionFunctions::
- hfst_basic_transducer_to_hfst_ol(intermediate_tmp,
- true, // weighted
- "", // no special options
- &harmonizer); // harmonize with this
- output_tmp = hfst::implementations::ConversionFunctions::
- hfst_ol_to_hfst_transducer(harmonized_tmp);
- output_tmp->set_name(it->first);
- retval.push_back(*output_tmp);
- delete it->second;
- delete intermediate_tmp;
- delete output_tmp;
- }
- } else {
- std::cerr << "Empty ruleset, nothing to write\n";
- throw HfstException(); // TODO
- }
- return retval;
-}
-
-
-/* Wrapper variables for an IOString output. */
-std::string hfst_regex_error_message("");
-std::string get_hfst_regex_error_message() { return hfst::hfst_regex_error_message; }
-
-hfst::HfstTransducer * hfst_regex(hfst::xre::XreCompiler & comp, const std::string & regex_string, const std::string & error_stream)
-{
- hfst_regex_error_message="";
-
- if (error_stream == "cout")
- {
- comp.set_error_stream(&std::cout);
- return comp.compile(regex_string);
- }
- else if (error_stream == "cerr")
- {
- comp.set_error_stream(&std::cerr);
- return comp.compile(regex_string);
- }
- else
- {
- std::ostringstream os(std::ostringstream::ate);
- comp.set_error_stream(&os);
- hfst::set_warning_stream(&os);
- hfst::HfstTransducer * retval = comp.compile(regex_string);
- hfst_regex_error_message = os.str();
- hfst::set_warning_stream(&std::cerr);
- return retval;
- }
-}
-
-/* Wrapper variables for an IOString output. */
-std::string hfst_lexc_output("");
-std::string get_hfst_lexc_output() { return hfst::hfst_lexc_output; }
-
-hfst::HfstTransducer * hfst_compile_lexc(hfst::lexc::LexcCompiler & comp, const std::string & filename, const std::string & error_stream)
-{
- hfst_lexc_output="";
-
- if (error_stream == "cout")
- {
- comp.set_error_stream(&std::cout);
- if (comp.getVerbosity() > 1)
- std::cout << "Parsing the lexc file..." << std::endl;
- comp.parse(filename.c_str());
- if (comp.getVerbosity() > 1)
- std::cout << "Compiling..." << std::endl;
- hfst::HfstTransducer * retval = comp.compileLexical();
- if (comp.getVerbosity() > 1)
- std::cout << "Compilation done." << std::endl;
- return retval;
- }
- else if (error_stream == "cerr")
- {
- comp.set_error_stream(&std::cerr);
- if (comp.getVerbosity() > 1)
- std::cerr << "Parsing the lexc file..." << std::endl;
- comp.parse(filename.c_str());
- if (comp.getVerbosity() > 1)
- std::cerr << "Compiling..." << std::endl;
- hfst::HfstTransducer * retval = comp.compileLexical();
- if (comp.getVerbosity() > 1)
- std::cerr << "Compilation done." << std::endl;
- return retval;
- }
- else
- {
- std::ostringstream os(std::ostringstream::ate);
- comp.set_error_stream(&os);
- hfst::set_warning_stream(&os);
- if (comp.getVerbosity() > 1)
- os << "Parsing the lexc file..." << std::endl;
- comp.parse(filename.c_str());
- if (comp.getVerbosity() > 1)
- os << "Compiling..." << std::endl;
- hfst::HfstTransducer * retval = comp.compileLexical();
- if (comp.getVerbosity() > 1)
- os << "Compilation done." << std::endl;
- hfst_lexc_output = os.str();
- hfst::set_warning_stream(&std::cerr);
- return retval;
- }
-}
-
-
-/* Wrapper variables for an IOString output. */
-std::string hfst_xfst_string_one("");
-char * get_hfst_xfst_string_one() { return strdup(hfst::hfst_xfst_string_one.c_str()); }
-std::string hfst_xfst_string_two("");
-char * get_hfst_xfst_string_two() { return strdup(hfst::hfst_xfst_string_two.c_str()); }
-
-int hfst_compile_xfst_to_string_one(hfst::xfst::XfstCompiler & comp, std::string input)
-{
- hfst::hfst_xfst_string_one="";
- hfst::hfst_xfst_string_two="";
-
- std::ostringstream os(std::ostringstream::ate);
- hfst::set_warning_stream(&os);
- comp.set_output_stream(os);
- comp.set_error_stream(os);
- int retval = comp.parse_line(input);
- hfst::hfst_xfst_string_one = os.str();
- hfst::set_warning_stream(&std::cerr);
- return retval;
-}
-
-int hfst_compile_xfst(hfst::xfst::XfstCompiler & comp, std::string input, const std::string & output_stream, const std::string & error_stream)
-{
- hfst::hfst_xfst_string_one="";
- hfst::hfst_xfst_string_two="";
- std::ostringstream * os1 = NULL;
- std::ostringstream * os2 = NULL;
-
- if (output_stream == "cout")
- comp.set_output_stream(std::cout);
- else if (output_stream == "cerr")
- comp.set_output_stream(std::cerr);
- else {
- os1 = new std::ostringstream(std::ostringstream::ate);
- comp.set_output_stream(*os1);
- }
-
- if (error_stream == "cout") {
- comp.set_error_stream(std::cout);
- hfst::set_warning_stream(&std::cout);
- }
- else if (error_stream == "cerr")
- comp.set_error_stream(std::cerr);
- else {
- os2 = new std::ostringstream(std::ostringstream::ate);
- comp.set_error_stream(*os2);
- hfst::set_warning_stream(os2);
- }
-
- int retval = comp.parse_line(input);
- hfst::set_warning_stream(&std::cerr);
-
- if (output_stream == "") {
- hfst::hfst_xfst_string_one = os1->str();
- delete os1;
- }
- if (error_stream == "") {
- hfst::hfst_xfst_string_two = os2->str();
- delete os2;
- }
-
- return retval;
-}
-
-
-
-hfst_ol::PmatchContainer * create_pmatch_container(const std::string & filename)
-{
- std::ifstream instream(filename.c_str(),
- std::ifstream::binary);
- if (!instream.good()) {
- return NULL;
- }
- return new hfst_ol::PmatchContainer(instream);
-}
-
-hfst::HfstOutputStream * create_hfst_output_stream(const std::string & filename, hfst::ImplementationType type, bool hfst_format)
-{
- if (filename == "") { return new hfst::HfstOutputStream(type, hfst_format); }
- else { return new hfst::HfstOutputStream(filename, type, hfst_format); }
-}
-
-hfst::HfstTransducer * read_att(hfst::HfstFile & f, std::string epsilon="@_EPSILON_SYMBOL_@")
-{
- return new hfst::HfstTransducer(f.get_file(), type, epsilon);
-}
-
-hfst::HfstTransducer * read_prolog(hfst::HfstFile & f)
-{
- unsigned int linecount = 0;
- hfst::implementations::HfstBasicTransducer fsm = hfst::implementations::HfstBasicTransducer::read_in_prolog_format(f.get_file(), linecount);
- return new hfst::HfstTransducer(fsm, hfst::type);
-}
-
-std::string one_level_paths_to_string(const hfst::HfstOneLevelPaths & paths)
-{
- std::ostringstream oss;
- for(hfst::HfstOneLevelPaths::const_iterator it = paths.begin(); it != paths.end(); it++)
- {
- for (hfst::StringVector::const_iterator svit = it->second.begin(); svit != it->second.end(); svit++)
- {
- oss << *svit;
- }
- oss << "\t" << it->first << std::endl;
- }
- return oss.str();
-}
-
-hfst::HfstOneLevelPaths extract_output_side(const hfst::HfstTwoLevelPaths & paths)
-{
- hfst::HfstOneLevelPaths result;
- for(hfst::HfstTwoLevelPaths::const_iterator it = paths.begin(); it != paths.end(); it++)
- {
- hfst::StringVector sv;
- for (hfst::StringPairVector::const_iterator svit = it->second.begin(); svit != it->second.end(); svit++)
- {
- sv.push_back(svit->second);
- }
- result.insert(std::pair<float, hfst::StringVector>(it->first, sv));
- }
- return result;
-}
-
-std::string two_level_paths_to_string(const hfst::HfstTwoLevelPaths & paths)
-{
- std::ostringstream oss;
- for(hfst::HfstTwoLevelPaths::const_iterator it = paths.begin(); it != paths.end(); it++)
- {
- std::string input("");
- std::string output("");
- for (hfst::StringPairVector::const_iterator svit = it->second.begin(); svit != it->second.end(); svit++)
- {
- input += svit->first;
- output += svit->second;
- }
- oss << input << ":" << output << "\t" << it->first << std::endl;
- }
- return oss.str();
-}
-
-/** Wrappers for lookup functions **/
-
-HfstOneLevelPaths lookup_vector_(const hfst::HfstTransducer * tr, bool fd, const StringVector& s, int limit = -1, double time_cutoff = 0.0) throw(FunctionNotImplementedException)
-{
- if (tr->get_type() == hfst::HFST_OL_TYPE || tr->get_type() == hfst::HFST_OLW_TYPE)
- {
- if (fd)
- { return *(tr->lookup_fd(s, limit, time_cutoff)); }
- else
- { return *(tr->lookup(s, limit, time_cutoff)); }
- }
- hfst::HfstTransducer input(s, tr->get_type());
- input.compose(*(tr));
- input.minimize();
- hfst::HfstTwoLevelPaths result;
- if (fd)
- { input.extract_paths_fd(result, limit, -1); }
- else
- { input.extract_paths(result, limit, -1); }
- return hfst::extract_output_side(result);
-}
-
-HfstOneLevelPaths lookup_string_(const hfst::HfstTransducer * tr, bool fd, const std::string& s, int limit = -1, double time_cutoff = 0.0) throw(FunctionNotImplementedException)
-{
- if (tr->get_type() == hfst::HFST_OL_TYPE || tr->get_type() == hfst::HFST_OLW_TYPE)
- {
- if (fd)
- { return *(tr->lookup_fd(s, limit, time_cutoff)); }
- else
- { return *(tr->lookup(s, limit, time_cutoff)); }
- }
- hfst::StringSet alpha = tr->get_alphabet();
- hfst::HfstTokenizer tok;
- for (hfst::StringSet::const_iterator it = alpha.begin(); it != alpha.end(); it++)
- { tok.add_multichar_symbol(*it); }
- hfst::HfstTransducer input(s, tok, tr->get_type());
- input.compose(*(tr));
- input.minimize();
- hfst::HfstTwoLevelPaths result;
- if (fd)
- { input.extract_paths_fd(result, limit, -1); }
- else
- { input.extract_paths(result, limit, -1); }
- return hfst::extract_output_side(result);
-}
-
-/** Wrappers for rule functions. **/
-
-namespace hfst_rules {
-
- HfstTransducer two_level_if(const HfstTransducerPair & context, const StringPairSet & mappings, const StringPairSet & alphabet)
- {
- hfst::HfstTransducerPair context_(context);
- StringPairSet mappings_(mappings);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::two_level_if(context_, mappings_, alphabet_);
- }
- HfstTransducer two_level_only_if(const HfstTransducerPair &context, const StringPairSet &mappings, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPair context_(context);
- StringPairSet mappings_(mappings);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::two_level_only_if(context_, mappings_, alphabet_);
- }
- HfstTransducer two_level_if_and_only_if(const HfstTransducerPair &context, const StringPairSet &mappings, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPair context_(context);
- StringPairSet mappings_(mappings);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::two_level_if_and_only_if(context_, mappings_, alphabet_);
- }
- HfstTransducer replace_down(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPair context_(context);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::replace_down(context_, mapping_, optional, alphabet_);
- }
- HfstTransducer replace_down_karttunen(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPair context_(context);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::replace_down_karttunen(context_, mapping_, optional, alphabet_);
- }
- HfstTransducer replace_right(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPair context_(context);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::replace_right(context_, mapping_, optional, alphabet_);
- }
- HfstTransducer replace_left(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPair context_(context);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::replace_left(context_, mapping_, optional, alphabet_);
- }
- HfstTransducer replace_up(const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
- {
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::replace_up(mapping_, optional, alphabet_);
- }
- HfstTransducer replace_down(const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
- {
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::replace_down(mapping_, optional, alphabet_);
- }
- HfstTransducer left_replace_up(const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
- {
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::left_replace_up(mapping_, optional, alphabet_);
- }
- HfstTransducer left_replace_up(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPair context_(context);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::left_replace_up(context_, mapping_, optional, alphabet_);
- }
- HfstTransducer left_replace_down(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPair context_(context);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::left_replace_down(context_, mapping_, optional, alphabet_);
- }
- HfstTransducer left_replace_down_karttunen(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPair context_(context);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::left_replace_down_karttunen(context_, mapping_, optional, alphabet_);
- }
- HfstTransducer left_replace_left(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPair context_(context);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::left_replace_left(context_, mapping_, optional, alphabet_);
- }
- HfstTransducer left_replace_right(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPair context_(context);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::left_replace_right(context_, mapping_, optional, alphabet_);
- }
- HfstTransducer restriction(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPairVector contexts_(contexts);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::restriction(contexts_, mapping_, alphabet_);
- }
- HfstTransducer coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPairVector contexts_(contexts);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::coercion(contexts_, mapping_, alphabet_);
- }
- HfstTransducer restriction_and_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPairVector contexts_(contexts);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::restriction_and_coercion(contexts_, mapping_, alphabet_);
- }
- HfstTransducer surface_restriction(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPairVector contexts_(contexts);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::surface_restriction(contexts_, mapping_, alphabet_);
- }
- HfstTransducer surface_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPairVector contexts_(contexts);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::surface_coercion(contexts_, mapping_, alphabet_);
- }
- HfstTransducer surface_restriction_and_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPairVector contexts_(contexts);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::surface_restriction_and_coercion(contexts_, mapping_, alphabet_);
- }
- HfstTransducer deep_restriction(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPairVector contexts_(contexts);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::deep_restriction(contexts_, mapping_, alphabet_);
- }
- HfstTransducer deep_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPairVector contexts_(contexts);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::deep_coercion(contexts_, mapping_, alphabet_);
- }
- HfstTransducer deep_restriction_and_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet)
- {
- hfst::HfstTransducerPairVector contexts_(contexts);
- hfst::HfstTransducer mapping_(mapping);
- StringPairSet alphabet_(alphabet);
- return hfst::rules::deep_restriction_and_coercion(contexts_, mapping_, alphabet_);
- }
-
-}
-
-}
-
+namespace hfst { typedef std::vector<hfst::xeroxRules::Rule> HfstRuleVector; }
+
+// Most of C++ extension code is located in separate files.
+#include "hfst_regex_extensions.cc"
+#include "hfst_extensions.cc"
+#include "hfst_lexc_extensions.cc"
+#include "hfst_xfst_extensions.cc"
+#include "hfst_pmatch_extensions.cc"
+#include "hfst_sfst_extensions.cc"
+#include "hfst_lookup_extensions.cc"
+#include "hfst_rules_extensions.cc"
+#include "hfst_prolog_extensions.cc"
%}
#ifdef _MSC_VER
%include <windows.h>
#endif
+// Templates needed for conversion between c++ and python datatypes.
+//
+// Note that templating order matters; simple templates used as part of
+// more complex templates must be defined first, e.g. StringPair must be
+// defined before StringPairSet. Also templates that are not used as such
+// but are used as part of other templates must be defined.
+
%include "typemaps.i"
namespace std {
@@ -658,71 +83,69 @@ namespace std {
%template(HfstTransducerVector) vector<hfst::HfstTransducer>;
%template(HfstSymbolSubstitutions) map<string, string>;
%template(HfstSymbolPairSubstitutions) map<pair<string, string>, pair<string, string> >;
-%template(FooBarBaz) vector<hfst::implementations::HfstBasicTransition>;
+// needed for HfstBasicTransducer.states()
%template(BarBazFoo) vector<unsigned int>;
-%template(HfstBasicStates) vector<vector<hfst::implementations::HfstBasicTransition> >;
+%template(HfstBasicTransitions) vector<hfst::implementations::HfstBasicTransition>;
%template(HfstOneLevelPath) pair<float, vector<string> >;
%template(HfstOneLevelPaths) set<pair<float, vector<string> > >;
%template(HfstTwoLevelPath) pair<float, vector<pair<string, string > > >;
%template(HfstTwoLevelPaths) set<pair<float, vector<pair<string, string > > > >;
%template(HfstTransducerPair) pair<hfst::HfstTransducer, hfst::HfstTransducer>;
%template(HfstTransducerPairVector) vector<pair<hfst::HfstTransducer, hfst::HfstTransducer> >;
+%template(HfstRuleVector) vector<hfst::xeroxRules::Rule>;
}
-//%ignore hfst::HfstTransducer::lookup_fd(const std::string & s) const;
+
+%include "docstrings.i"
+
+// ****************************************************** //
+// ********** WHAT IS MADE AVAILABLE ON PYTHON ********** //
+// ****************************************************** //
+
+// *** HfstException and its subclasses (will be wrapped under module hfst.exceptions). *** //
class HfstException
{
-public:
- HfstException();
- HfstException(const std::string&, const std::string&, size_t);
- ~HfstException();
+ public:
+ HfstException();
+ HfstException(const std::string&, const std::string&, size_t);
+ ~HfstException();
+ std::string what() const;
};
-class HfstTransducerTypeMismatchException : public HfstException { public: HfstTransducerTypeMismatchException(const std::string&, const std::string&, size_t); ~HfstTransducerTypeMismatchException(); };
-class ImplementationTypeNotAvailableException : public HfstException { public: ImplementationTypeNotAvailableException(const std::string&, const std::string&, size_t); ~ImplementationTypeNotAvailableException(); };
-class FunctionNotImplementedException : public HfstException { public: FunctionNotImplementedException(const std::string&, const std::string&, size_t); ~FunctionNotImplementedException(); };
-class StreamNotReadableException : public HfstException { public: StreamNotReadableException(const std::string&, const std::string&, size_t); ~StreamNotReadableException(); };
-class StreamCannotBeWrittenException : public HfstException { public: StreamCannotBeWrittenException(const std::string&, const std::string&, size_t); ~StreamCannotBeWrittenException(); };
-class StreamIsClosedException : public HfstException { public: StreamIsClosedException(const std::string&, const std::string&, size_t); ~StreamIsClosedException(); };
-class EndOfStreamException : public HfstException { public: EndOfStreamException(const std::string&, const std::string&, size_t); ~EndOfStreamException(); };
-class TransducerIsCyclicException : public HfstException { public: TransducerIsCyclicException(const std::string&, const std::string&, size_t); ~TransducerIsCyclicException(); };
-class NotTransducerStreamException : public HfstException { public: NotTransducerStreamException(const std::string&, const std::string&, size_t); ~NotTransducerStreamException(); };
-class NotValidAttFormatException : public HfstException { public: NotValidAttFormatException(const std::string&, const std::string&, size_t); ~NotValidAttFormatException(); };
-class NotValidPrologFormatException : public HfstException { public: NotValidPrologFormatException(const std::string&, const std::string&, size_t); ~NotValidPrologFormatException(); };
-class NotValidLexcFormatException : public HfstException { public: NotValidLexcFormatException(const std::string&, const std::string&, size_t); ~NotValidLexcFormatException(); };
-class StateIsNotFinalException : public HfstException { public: StateIsNotFinalException(const std::string&, const std::string&, size_t); ~StateIsNotFinalException(); };
-class ContextTransducersAreNotAutomataException : public HfstException { public: ContextTransducersAreNotAutomataException(const std::string&, const std::string&, size_t); ~ContextTransducersAreNotAutomataException(); };
-class TransducersAreNotAutomataException : public HfstException { public: TransducersAreNotAutomataException(const std::string&, const std::string&, size_t); ~TransducersAreNotAutomataException(); };
-class StateIndexOutOfBoundsException : public HfstException { public: StateIndexOutOfBoundsException(const std::string&, const std::string&, size_t); ~StateIndexOutOfBoundsException(); };
-class TransducerHeaderException : public HfstException { public: TransducerHeaderException(const std::string&, const std::string&, size_t); ~TransducerHeaderException(); };
-class MissingOpenFstInputSymbolTableException : public HfstException { public: MissingOpenFstInputSymbolTableException(const std::string&, const std::string&, size_t); ~MissingOpenFstInputSymbolTableException(); };
-class TransducerTypeMismatchException : public HfstException { public: TransducerTypeMismatchException(const std::string&, const std::string&, size_t); ~TransducerTypeMismatchException(); };
-class EmptySetOfContextsException : public HfstException { public: EmptySetOfContextsException(const std::string&, const std::string&, size_t); ~EmptySetOfContextsException(); };
-class SpecifiedTypeRequiredException : public HfstException { public: SpecifiedTypeRequiredException(const std::string&, const std::string&, size_t); ~SpecifiedTypeRequiredException(); };
-class HfstFatalException : public HfstException { public: HfstFatalException(const std::string&, const std::string&, size_t); ~HfstFatalException(); };
-class TransducerHasWrongTypeException : public HfstException { public: TransducerHasWrongTypeException(const std::string&, const std::string&, size_t); ~TransducerHasWrongTypeException(); };
-class IncorrectUtf8CodingException : public HfstException { public: IncorrectUtf8CodingException(const std::string&, const std::string&, size_t); ~IncorrectUtf8CodingException(); };
-class EmptyStringException : public HfstException { public: EmptyStringException(const std::string&, const std::string&, size_t); ~EmptyStringException(); };
-class SymbolNotFoundException : public HfstException { public: SymbolNotFoundException(const std::string&, const std::string&, size_t); ~SymbolNotFoundException(); };
-class MetadataException : public HfstException { public: MetadataException(const std::string&, const std::string&, size_t); ~MetadataException(); };
-class FlagDiacriticsAreNotIdentitiesException : public HfstException { public: FlagDiacriticsAreNotIdentitiesException(const std::string&, const std::string&, size_t); ~FlagDiacriticsAreNotIdentitiesException(); };
+class HfstTransducerTypeMismatchException : public HfstException { public: HfstTransducerTypeMismatchException(const std::string&, const std::string&, size_t); ~HfstTransducerTypeMismatchException(); std::string what() const; };
+class ImplementationTypeNotAvailableException : public HfstException { public: ImplementationTypeNotAvailableException(const std::string&, const std::string&, size_t, hfst::ImplementationType type); ~ImplementationTypeNotAvailableException(); std::string what() const; hfst::ImplementationType get_type() const; };
+class FunctionNotImplementedException : public HfstException { public: FunctionNotImplementedException(const std::string&, const std::string&, size_t); ~FunctionNotImplementedException(); std::string what() const; };
+class StreamNotReadableException : public HfstException { public: StreamNotReadableException(const std::string&, const std::string&, size_t); ~StreamNotReadableException(); std::string what() const; };
+class StreamCannotBeWrittenException : public HfstException { public: StreamCannotBeWrittenException(const std::string&, const std::string&, size_t); ~StreamCannotBeWrittenException(); std::string what() const; };
+class StreamIsClosedException : public HfstException { public: StreamIsClosedException(const std::string&, const std::string&, size_t); ~StreamIsClosedException(); std::string what() const; };
+class EndOfStreamException : public HfstException { public: EndOfStreamException(const std::string&, const std::string&, size_t); ~EndOfStreamException(); std::string what() const; };
+class TransducerIsCyclicException : public HfstException { public: TransducerIsCyclicException(const std::string&, const std::string&, size_t); ~TransducerIsCyclicException(); std::string what() const; };
+class NotTransducerStreamException : public HfstException { public: NotTransducerStreamException(const std::string&, const std::string&, size_t); ~NotTransducerStreamException(); std::string what() const; };
+class NotValidAttFormatException : public HfstException { public: NotValidAttFormatException(const std::string&, const std::string&, size_t); ~NotValidAttFormatException(); std::string what() const; };
+class NotValidPrologFormatException : public HfstException { public: NotValidPrologFormatException(const std::string&, const std::string&, size_t); ~NotValidPrologFormatException(); std::string what() const; };
+class NotValidLexcFormatException : public HfstException { public: NotValidLexcFormatException(const std::string&, const std::string&, size_t); ~NotValidLexcFormatException(); std::string what() const; };
+class StateIsNotFinalException : public HfstException { public: StateIsNotFinalException(const std::string&, const std::string&, size_t); ~StateIsNotFinalException(); std::string what() const; };
+class ContextTransducersAreNotAutomataException : public HfstException { public: ContextTransducersAreNotAutomataException(const std::string&, const std::string&, size_t); ~ContextTransducersAreNotAutomataException(); std::string what() const; };
+class TransducersAreNotAutomataException : public HfstException { public: TransducersAreNotAutomataException(const std::string&, const std::string&, size_t); ~TransducersAreNotAutomataException(); std::string what() const; };
+class StateIndexOutOfBoundsException : public HfstException { public: StateIndexOutOfBoundsException(const std::string&, const std::string&, size_t); ~StateIndexOutOfBoundsException(); std::string what() const; };
+class TransducerHeaderException : public HfstException { public: TransducerHeaderException(const std::string&, const std::string&, size_t); ~TransducerHeaderException(); std::string what() const; };
+class MissingOpenFstInputSymbolTableException : public HfstException { public: MissingOpenFstInputSymbolTableException(const std::string&, const std::string&, size_t); ~MissingOpenFstInputSymbolTableException(); std::string what() const; };
+class TransducerTypeMismatchException : public HfstException { public: TransducerTypeMismatchException(const std::string&, const std::string&, size_t); ~TransducerTypeMismatchException(); std::string what() const; };
+class EmptySetOfContextsException : public HfstException { public: EmptySetOfContextsException(const std::string&, const std::string&, size_t); ~EmptySetOfContextsException(); std::string what() const; };
+class SpecifiedTypeRequiredException : public HfstException { public: SpecifiedTypeRequiredException(const std::string&, const std::string&, size_t); ~SpecifiedTypeRequiredException(); std::string what() const; };
+class HfstFatalException : public HfstException { public: HfstFatalException(const std::string&, const std::string&, size_t); ~HfstFatalException(); std::string what() const; };
+class TransducerHasWrongTypeException : public HfstException { public: TransducerHasWrongTypeException(const std::string&, const std::string&, size_t); ~TransducerHasWrongTypeException(); std::string what() const; };
+class IncorrectUtf8CodingException : public HfstException { public: IncorrectUtf8CodingException(const std::string&, const std::string&, size_t); ~IncorrectUtf8CodingException(); std::string what() const; };
+class EmptyStringException : public HfstException { public: EmptyStringException(const std::string&, const std::string&, size_t); ~EmptyStringException(); std::string what() const; };
+class SymbolNotFoundException : public HfstException { public: SymbolNotFoundException(const std::string&, const std::string&, size_t); ~SymbolNotFoundException(); std::string what() const; };
+class MetadataException : public HfstException { public: MetadataException(const std::string&, const std::string&, size_t); ~MetadataException(); std::string what() const; };
+class FlagDiacriticsAreNotIdentitiesException : public HfstException { public: FlagDiacriticsAreNotIdentitiesException(const std::string&, const std::string&, size_t); ~FlagDiacriticsAreNotIdentitiesException(); std::string what() const; };
namespace hfst
{
-class HfstFile {
- public:
- HfstFile();
- ~HfstFile();
- void write(const char * str);
- void close();
- bool is_eof(void);
-};
-
-HfstFile hfst_stdout();
-HfstFile hfst_stdin();
-HfstFile hfst_open(const char * filename, const char * mode);
+// Needed for conversion between c++ and python datatypes.
typedef std::vector<std::string> StringVector;
typedef std::pair<std::string, std::string> StringPair;
@@ -739,88 +162,110 @@ typedef std::map<std::pair<std::string, std::string>, std::pair<std::string, std
typedef std::vector<hfst::HfstTransducer> HfstTransducerVector;
typedef std::pair<hfst::HfstTransducer, hfst::HfstTransducer> HfstTransducerPair;
typedef std::vector<std::pair<hfst::HfstTransducer, hfst::HfstTransducer> > HfstTransducerPairVector;
+typedef std::vector<hfst::xeroxRules::Rule> HfstRuleVector;
+
+// *** Some enumerations *** //
enum ImplementationType
-{
- SFST_TYPE,
- TROPICAL_OPENFST_TYPE,
- LOG_OPENFST_TYPE,
- FOMA_TYPE,
- XFSM_TYPE,
- HFST_OL_TYPE,
- HFST_OLW_TYPE,
- HFST2_TYPE,
- UNSPECIFIED_TYPE,
- ERROR_TYPE
-};
+{ SFST_TYPE, TROPICAL_OPENFST_TYPE, LOG_OPENFST_TYPE, FOMA_TYPE,
+ XFSM_TYPE, HFST_OL_TYPE, HFST_OLW_TYPE, HFST2_TYPE,
+ UNSPECIFIED_TYPE, ERROR_TYPE };
-enum PushType { TO_INITIAL_STATE, TO_FINAL_STATE };
+ namespace xeroxRules
+ {
+ enum ReplaceType {REPL_UP, REPL_DOWN, REPL_RIGHT, REPL_LEFT};
-//std::ostream * get_warning_stream();
-//void set_warning_stream(std::ostream * os);
+ class Rule
+ {
+ public:
+ Rule();
+ Rule(const hfst::HfstTransducerPairVector&);
+ Rule(const hfst::HfstTransducerPairVector&, const hfst::HfstTransducerPairVector&, ReplaceType);
+ Rule(const Rule&);
+ hfst::HfstTransducerPairVector get_mapping() const;
+ hfst::HfstTransducerPairVector get_context() const;
+ ReplaceType get_replType() const;
+ void encodeFlags();
+ // friend std::ostream& operator<<(std::ostream &out, const Rule &r);
+ };
-/*
-%typemap(out) HfstOneLevelPaths* {
- $result = PyList_New((*$1).size());
- unsigned int i = 0;
- for (hfst::HfstOneLevelPaths::const_iterator it = (*$1).begin(); it != (*$1).end(); it++)
- {
- std::string result_string("");
- for (hfst::StringVector::const_iterator svit = it->second.begin(); svit != it->second.end(); svit++)
- {
- result_string += *svit;
- }
- PyObject * res = PyTuple_New(2);
- PyTuple_SetItem(res, 0, PyString_FromString(result_string.c_str()));
- PyTuple_SetItem(res, 1, PyFloat_FromDouble(it->first));
- PyList_SetItem($result, i, res);
- i++;
- }
-}
-*/
+ // replace up, left, right, down
+ hfst::HfstTransducer replace(const Rule &rule, bool optional);
+ // for parallel rules
+ hfst::HfstTransducer replace(const HfstRuleVector &ruleVector, bool optional);
+ // replace up, left, right, down
+%rename("xerox_replace_left") replace_left(const Rule &rule, bool optional);
+ hfst::HfstTransducer replace_left(const Rule &rule, bool optional);
+ // for parallel rules
+%rename("xerox_replace_left") replace_left(const HfstRuleVector &ruleVector, bool optional);
+ hfst::HfstTransducer replace_left(const HfstRuleVector &ruleVector, bool optional);
+ // left to right
+ hfst::HfstTransducer replace_leftmost_longest_match(const Rule &rule);
+ // left to right
+ hfst::HfstTransducer replace_leftmost_longest_match(const HfstRuleVector &ruleVector);
+ // right to left
+ hfst::HfstTransducer replace_rightmost_longest_match(const Rule &rule);
+
+ // right to left
+ hfst::HfstTransducer replace_rightmost_longest_match(const HfstRuleVector &ruleVector);
+
+ hfst::HfstTransducer replace_leftmost_shortest_match(const Rule &rule);
+
+ hfst::HfstTransducer replace_leftmost_shortest_match(const HfstRuleVector &ruleVector);
+ hfst::HfstTransducer replace_rightmost_shortest_match(const Rule &rule);
+ hfst::HfstTransducer replace_rightmost_shortest_match(const HfstRuleVector &ruleVector);
+
+ // replace up, left, right, down
+ hfst::HfstTransducer replace_epenthesis(const Rule &rule, bool optional);
+ // replace up, left, right, down
+ hfst::HfstTransducer replace_epenthesis(const HfstRuleVector &ruleVector, bool optional);
+
+ // Restriction function "=>"
+%rename("xerox_restriction") restriction(const hfst::HfstTransducer &automata, const hfst::HfstTransducerPairVector &context);
+ hfst::HfstTransducer restriction(const hfst::HfstTransducer &automata, const hfst::HfstTransducerPairVector &context);
+ hfst::HfstTransducer before(const hfst::HfstTransducer &left, const hfst::HfstTransducer &right);
+ hfst::HfstTransducer after(const hfst::HfstTransducer &left, const hfst::HfstTransducer &right);
-/* instead, run:
- sed -i 's/class HfstException(_object):/class HfstException(Exception):/' libhfst.py
- after build to make HfstException and its subclasses subclasses of Python's Exception.
+ }
-%typemap(throws) HfstTransducerTypeMismatchException %{ PyErr_SetString(PyExc_RuntimeError, "HfstTransducerTypeMismatchException"); SWIG_fail; %}
-... etc for all exception classes
-*/
+// *** Some other functions *** //
bool is_diacritic(const std::string & symbol);
+hfst::HfstTransducerVector compile_pmatch_expression(const std::string & pmatch) throw(HfstException);
+// internal functions
%pythoncode %{
- def is_string(s):
+ def _is_string(s):
if isinstance(s, str):
return True
else:
- return False
- def is_string_pair(sp):
+ return False
+ def _is_string_pair(sp):
if not isinstance(sp, tuple):
return False
if len(sp) != 2:
return False
- if not is_string(sp[0]):
+ if not _is_string(sp[0]):
return False
- if not is_string(sp[1]):
+ if not _is_string(sp[1]):
return False
return True
- def is_string_vector(sv):
+ def _is_string_vector(sv):
if not isinstance(sv, tuple):
return False
for s in sv:
- if not is_string(s):
+ if not _is_string(s):
return False
return True
- def is_string_pair_vector(spv):
+ def _is_string_pair_vector(spv):
if not isinstance(spv, tuple):
return False
for sp in spv:
- if not is_string_pair(sp):
+ if not _is_string_pair(sp):
return False
return True
- def two_level_paths_to_dict(tlps):
+ def _two_level_paths_to_dict(tlps):
retval = {}
for tlp in tlps:
input = ""
@@ -834,7 +279,7 @@ bool is_diacritic(const std::string & symbol);
retval[input] = [(output, tlp[0])]
return retval
- def one_level_paths_to_tuple(olps):
+ def _one_level_paths_to_tuple(olps):
retval = []
for olp in olps:
path = ""
@@ -844,103 +289,35 @@ bool is_diacritic(const std::string & symbol);
return tuple(retval)
%}
-// NOTE: all functions returning an HfstTransducer& are commented out and extended by replacing them with equivalent functions that return void.
-// This is done in order to avoid use of references that are not handled well by swig/python.
+// *** HfstTransducer *** //
+
+// NOTE: all functions returning an HfstTransducer& are commented out and extended
+// by replacing them with equivalent functions that return void. This is done in
+// order to avoid use of references that are not handled well by swig/python.
+// Some constructors and the destructor are also redefined.
-class HfstTransducer
+class HfstTransducer
{
public:
-HfstTransducer();
-// Redefined with %extend:
-//HfstTransducer(const hfst::HfstTransducer &);
-//HfstTransducer(const hfst::implementations::HfstBasicTransducer &, hfst::ImplementationType);
-//~HfstTransducer();
-
-void set_name(const std::string &name);
-std::string get_name() const;
-hfst::ImplementationType get_type() const;
-void set_property(const std::string& property, const std::string& value);
-std::string get_property(const std::string& property) const;
-const std::map<std::string,std::string>& get_properties() const;
-
-/* Basic binary operations */
-//HfstTransducer & concatenate(const HfstTransducer&, bool harmonize=true) throw(TransducerTypeMismatchException);
-//HfstTransducer & disjunct(const HfstTransducer&, bool harmonize=true) throw(TransducerTypeMismatchException);
-//HfstTransducer & subtract(const HfstTransducer&, bool harmonize=true) throw(TransducerTypeMismatchException);
-//HfstTransducer & intersect(const HfstTransducer&, bool harmonize=true) throw(TransducerTypeMismatchException);
-//HfstTransducer & compose(const HfstTransducer&, bool harmonize=true) throw(TransducerTypeMismatchException);
-
-/* More binary operations */
-//HfstTransducer & compose_intersect(const HfstTransducerVector &v, bool invert=false, bool harmonize=true);
-//HfstTransducer & priority_union(const HfstTransducer &another);
-//HfstTransducer & lenient_composition(const HfstTransducer &another, bool harmonize=true);
-//HfstTransducer & cross_product(const HfstTransducer &another, bool harmonize=true) throw(TransducersAreNotAutomataException);
-//HfstTransducer & shuffle(const HfstTransducer &another, bool harmonize=true);
-
-/* Testing */
-bool compare(const HfstTransducer&, bool harmonize=true) const throw(TransducerTypeMismatchException);
-unsigned int number_of_states() const;
-unsigned int number_of_arcs() const;
-StringSet get_alphabet() const;
-bool is_cyclic() const;
-bool is_automaton() const;
-bool is_infinitely_ambiguous() const;
-bool is_lookup_infinitely_ambiguous(const std::string &) const;
-bool has_flag_diacritics() const;
-
-void insert_to_alphabet(const std::string &);
-void remove_from_alphabet(const std::string &);
-
-static bool is_implementation_type_available(hfst::ImplementationType type);
-
-/* Optimization */
-//HfstTransducer & remove_epsilons();
-//HfstTransducer & determinize();
-//HfstTransducer & minimize();
-//HfstTransducer & prune();
-//HfstTransducer & eliminate_flags();
-//HfstTransducer & eliminate_flag(const std::string&) throw(HfstException);
-//HfstTransducer & n_best(unsigned int n);
-//HfstTransducer & convert(ImplementationType impl);
-
-/* Repeat */
-//HfstTransducer & repeat_star();
-//HfstTransducer & repeat_plus();
-//HfstTransducer & repeat_n(unsigned int);
-//HfstTransducer & repeat_n_to_k(unsigned int, unsigned int);
-//HfstTransducer & repeat_n_minus(unsigned int);
-//HfstTransducer & repeat_n_plus(unsigned int);
-
-/* Other basic operations */
-//HfstTransducer & invert();
-//HfstTransducer & reverse();
-//HfstTransducer & input_project();
-//HfstTransducer & output_project();
-//HfstTransducer & optionalize();
-
-/* Insert freely, substitute */
-
-//HfstTransducer & insert_freely(const StringPair &symbol_pair, bool harmonize=true);
-//HfstTransducer & insert_freely(const HfstTransducer &tr, bool harmonize=true);
-
-//HfstTransducer & substitute(bool (*func)(const StringPair &sp, hfst::StringPairSet &sps));
-
-//HfstTransducer & substitute_symbol(const std::string &old_symbol, const std::string &new_symbol, bool input_side=true, bool output_side=true);
-//HfstTransducer & substitute_symbol_pair(const StringPair &old_symbol_pair, const StringPair &new_symbol_pair);
-//HfstTransducer & substitute_symbol_pair_with_set(const StringPair &old_symbol_pair, const hfst::StringPairSet &new_symbol_pair_set);
-//HfstTransducer & substitute_symbol_pair_with_transducer(const StringPair &symbol_pair, HfstTransducer &transducer, bool harmonize=true);
-//HfstTransducer & substitute_symbols(const hfst::HfstSymbolSubstitutions &substitutions); // alias for the previous function which is shadowed
-//HfstTransducer & substitute_symbol_pairs(const hfst::HfstSymbolPairSubstitutions &substitutions); // alias for the previous function which is shadowed
-
-
-/* Weight handling */
-//HfstTransducer & set_final_weights(float weight, bool increment=false);
-// Can 'transform_weights' be wrapped? It maybe needs to be rewritten in python.
-//HfstTransducer & push_weights(hfst::PushType type);
-
-//void extract_shortest_paths(HfstTwoLevelPaths &results) const;
-//bool extract_longest_paths(HfstTwoLevelPaths &results, bool obey_flags=true) const;
-int longest_path_size(bool obey_flags=true) const;
+ void set_name(const std::string &name);
+ std::string get_name() const;
+ hfst::ImplementationType get_type() const;
+ void set_property(const std::string& property, const std::string& value);
+ std::string get_property(const std::string& property) const;
+ const std::map<std::string,std::string>& get_properties() const;
+ bool compare(const HfstTransducer&, bool harmonize=true) const throw(TransducerTypeMismatchException);
+ unsigned int number_of_states() const;
+ unsigned int number_of_arcs() const;
+ StringSet get_alphabet() const;
+ bool is_cyclic() const;
+ bool is_automaton() const;
+ bool is_infinitely_ambiguous() const;
+ bool is_lookup_infinitely_ambiguous(const std::string &) const;
+ bool has_flag_diacritics() const;
+ void insert_to_alphabet(const std::string &);
+ void remove_from_alphabet(const std::string &);
+ static bool is_implementation_type_available(hfst::ImplementationType type);
+ int longest_path_size(bool obey_flags=true) const;
%extend {
@@ -977,49 +354,31 @@ int longest_path_size(bool obey_flags=true) const;
void optionalize() { self->optionalize(); }
void insert_freely(const StringPair &symbol_pair, bool harmonize=true) { self->insert_freely(symbol_pair, harmonize); }
void insert_freely(const HfstTransducer &tr, bool harmonize=true) { self->insert_freely(tr, harmonize); }
- void substitute_symbol(const std::string &old_symbol, const std::string &new_symbol, bool input_side=true, bool output_side=true) { self->substitute_symbol(old_symbol, new_symbol, input_side, output_side); }
- void substitute_symbol_pair(const StringPair &old_symbol_pair, const StringPair &new_symbol_pair) { self->substitute_symbol_pair(old_symbol_pair, new_symbol_pair); }
- void substitute_symbol_pair_with_set(const StringPair &old_symbol_pair, const hfst::StringPairSet &new_symbol_pair_set) { self->substitute_symbol_pair_with_set(old_symbol_pair, new_symbol_pair_set); }
- void substitute_symbol_pair_with_transducer(const StringPair &symbol_pair, HfstTransducer &transducer, bool harmonize=true) { self->substitute_symbol_pair_with_transducer(symbol_pair, transducer, harmonize); }
- void substitute_symbols(const hfst::HfstSymbolSubstitutions &substitutions) { self->substitute_symbols(substitutions); } // alias for the previous function which is shadowed
- void substitute_symbol_pairs(const hfst::HfstSymbolPairSubstitutions &substitutions) { self->substitute_symbol_pairs(substitutions); } // alias for the previous function which is shadowed
+ void _substitute_symbol(const std::string &old_symbol, const std::string &new_symbol, bool input_side=true, bool output_side=true) { self->substitute_symbol(old_symbol, new_symbol, input_side, output_side); }
+ void _substitute_symbol_pair(const StringPair &old_symbol_pair, const StringPair &new_symbol_pair) { self->substitute_symbol_pair(old_symbol_pair, new_symbol_pair); }
+ void _substitute_symbol_pair_with_set(const StringPair &old_symbol_pair, const hfst::StringPairSet &new_symbol_pair_set) { self->substitute_symbol_pair_with_set(old_symbol_pair, new_symbol_pair_set); }
+ void _substitute_symbol_pair_with_transducer(const StringPair &symbol_pair, HfstTransducer &transducer, bool harmonize=true) { self->substitute_symbol_pair_with_transducer(symbol_pair, transducer, harmonize); }
+ void _substitute_symbols(const hfst::HfstSymbolSubstitutions &substitutions) { self->substitute_symbols(substitutions); } // alias for the previous function which is shadowed
+ void _substitute_symbol_pairs(const hfst::HfstSymbolPairSubstitutions &substitutions) { self->substitute_symbol_pairs(substitutions); } // alias for the previous function which is shadowed
void set_final_weights(float weight, bool increment=false) { self->set_final_weights(weight, increment); };
- void push_weights(hfst::PushType type) { self->push_weights(type); };
- // And some aliases:
+ void push_weights_to_start() { self->push_weights(hfst::TO_INITIAL_STATE); };
+ void push_weights_to_end() { self->push_weights(hfst::TO_FINAL_STATE); };
- // 'union' is a reserved word in python, so it cannot be used as an alias for function 'disjunct'
+ // And some aliases:
+ // 'union' is a reserved word in python, so it cannot be used as an alias for function 'disjunct'
void minus(const HfstTransducer& t, bool harmonize=true) { $self->subtract(t, harmonize); }
void conjunct(const HfstTransducer& t, bool harmonize=true) { $self->intersect(t, harmonize); }
// Then the actual extensions:
- hfst::HfstTwoLevelPaths extract_shortest_paths_()
- {
- hfst::HfstTwoLevelPaths results;
- $self->extract_shortest_paths(results);
- return results;
- }
-
- hfst::HfstTwoLevelPaths extract_longest_paths_(bool obey_flags)
- {
- hfst::HfstTwoLevelPaths results;
- $self->extract_longest_paths(results, obey_flags);
- return results;
- }
+ void lookup_optimize() { self->convert(hfst::HFST_OLW_TYPE); }
+ void remove_optimization() { self->convert(hfst::get_default_fst_type()); }
- HfstTransducer(const hfst::HfstTransducer & t)
- {
- return hfst::copy_hfst_transducer(t);
- }
- HfstTransducer(const hfst::implementations::HfstBasicTransducer & t)
- {
- return hfst::copy_hfst_transducer_from_basic_transducer(t);
- }
- HfstTransducer(const hfst::implementations::HfstBasicTransducer & t, hfst::ImplementationType impl)
- {
- return hfst::copy_hfst_transducer_from_basic_transducer(t, impl);
- }
+ HfstTransducer() { return hfst::empty_transducer(); }
+ HfstTransducer(const hfst::HfstTransducer & t) { return hfst::copy_hfst_transducer(t); }
+ HfstTransducer(const hfst::implementations::HfstBasicTransducer & t) { return hfst::copy_hfst_transducer_from_basic_transducer(t); }
+ HfstTransducer(const hfst::implementations::HfstBasicTransducer & t, hfst::ImplementationType impl) { return hfst::copy_hfst_transducer_from_basic_transducer(t, impl); }
~HfstTransducer()
{
if ($self->get_type() == hfst::UNSPECIFIED_TYPE || $self->get_type() == hfst::ERROR_TYPE)
@@ -1028,80 +387,173 @@ int longest_path_size(bool obey_flags=true) const;
}
delete $self;
}
-
+ // For python's 'print'
char *__str__() {
- static char tmp[1024];
- $self->write_in_att_format(tmp);
- return tmp;
- }
- void write(hfst::HfstOutputStream & os) {
- (void) os.redirect(*$self);
- //return *$self;
+ std::ostringstream oss;
+ hfst::implementations::HfstBasicTransducer fsm(*$self);
+ fsm.write_in_att_format(oss,true);
+ return strdup(oss.str().c_str());
}
+ void write(hfst::HfstOutputStream & os) { (void) os.redirect(*$self); }
- void write_att(hfst::HfstFile & f, bool write_weights=true)
+ hfst::HfstTwoLevelPaths _extract_shortest_paths()
{
- $self->write_in_att_format(f.get_file(), write_weights);
+ hfst::HfstTwoLevelPaths results;
+ $self->extract_shortest_paths(results);
+ return results;
}
-
- void write_prolog(hfst::HfstFile & f, const std::string & name, bool write_weights=true)
+ hfst::HfstTwoLevelPaths _extract_longest_paths(bool obey_flags)
{
- $self->write_in_prolog_format(f.get_file(), name, write_weights);
+ hfst::HfstTwoLevelPaths results;
+ $self->extract_longest_paths(results, obey_flags);
+ return results;
}
-
- hfst::HfstTwoLevelPaths extract_paths_(int max_num=-1, int cycles=-1) const throw(TransducerIsCyclicException)
+ hfst::HfstTwoLevelPaths _extract_paths(int max_num=-1, int cycles=-1) const throw(TransducerIsCyclicException)
{
hfst::HfstTwoLevelPaths results;
$self->extract_paths(results, max_num, cycles);
return results;
}
-
- hfst::HfstTwoLevelPaths extract_paths_fd_(int max_num=-1, int cycles=-1, bool filter_fd=true) const throw(TransducerIsCyclicException)
+ hfst::HfstTwoLevelPaths _extract_paths_fd(int max_num=-1, int cycles=-1, bool filter_fd=true) const throw(TransducerIsCyclicException)
{
hfst::HfstTwoLevelPaths results;
$self->extract_paths_fd(results, max_num, cycles, filter_fd);
return results;
}
-
- hfst::HfstTwoLevelPaths extract_random_paths_(int max_num) const
+ hfst::HfstTwoLevelPaths _extract_random_paths(int max_num) const
{
hfst::HfstTwoLevelPaths results;
$self->extract_random_paths(results, max_num);
return results;
}
-
- hfst::HfstTwoLevelPaths extract_random_paths_fd_(int max_num, bool filter_fd) const
+ hfst::HfstTwoLevelPaths _extract_random_paths_fd(int max_num, bool filter_fd) const
{
hfst::HfstTwoLevelPaths results;
$self->extract_random_paths_fd(results, max_num, filter_fd);
return results;
}
-HfstOneLevelPaths lookup_vector(const StringVector& s, int limit = -1, double time_cutoff = 0.0) const throw(FunctionNotImplementedException)
-{
- return hfst::lookup_vector_($self, false /*fd*/, s, limit, time_cutoff);
-}
-
-HfstOneLevelPaths lookup_fd_vector(const StringVector& s, int limit = -1, double time_cutoff = 0.0) const throw(FunctionNotImplementedException)
-{
- return hfst::lookup_vector_($self, true /*fd*/, s, limit, time_cutoff);
-}
-
-HfstOneLevelPaths lookup_fd_string(const std::string& s, int limit = -1, double time_cutoff = 0.0) const throw(FunctionNotImplementedException)
-{
- return hfst::lookup_string_($self, true /*fd*/, s, limit, time_cutoff);
-}
-
-HfstOneLevelPaths lookup_string(const std::string & s, int limit = -1, double time_cutoff = 0.0) const throw(FunctionNotImplementedException)
-{
- return hfst::lookup_string_($self, false /*fd*/, s, limit, time_cutoff);
-}
-
+ HfstOneLevelPaths _lookup_vector(const StringVector& s, int limit = -1, double time_cutoff = 0.0) const throw(FunctionNotImplementedException)
+ {
+ return hfst::lookup_vector($self, false /*fd*/, s, limit, time_cutoff);
+ }
+ HfstOneLevelPaths _lookup_fd_vector(const StringVector& s, int limit = -1, double time_cutoff = 0.0) const throw(FunctionNotImplementedException)
+ {
+ return hfst::lookup_vector($self, true /*fd*/, s, limit, time_cutoff);
+ }
+ HfstOneLevelPaths _lookup_fd_string(const std::string& s, int limit = -1, double time_cutoff = 0.0) const throw(FunctionNotImplementedException)
+ {
+ return hfst::lookup_string($self, true /*fd*/, s, limit, time_cutoff);
+ }
+ HfstOneLevelPaths _lookup_string(const std::string & s, int limit = -1, double time_cutoff = 0.0) const throw(FunctionNotImplementedException)
+ {
+ return hfst::lookup_string($self, false /*fd*/, s, limit, time_cutoff);
+ }
%pythoncode %{
+ def copy(self):
+ """
+ Return a deep copy of the transducer.
+ """
+ return HfstTransducer(self)
+
+ def write_to_file(self, filename_):
+ """
+ Write the transducer in binary format to file *filename_*.
+ """
+ ostr = HfstOutputStream(filename=filename_, type=self.get_type(), hfst_format=True)
+ ostr.write(self)
+ ostr.close()
+
+ def read_from_file(filename_):
+ """
+ Read a binary transducer from file *filename_*.
+ """
+ istr = HfstInputStream(filename_)
+ tr = istr.read()
+ istr.close()
+ return tr
+
+ def write_prolog(self, f, write_weights=True):
+ """
+ Write the transducer in prolog format with name *name* to file *f*,
+ *write_weights* defined whether weights are written.
+
+ Parameters
+ ----------
+ * `f` :
+ A python file where the transducer is written.
+ * `write_weights` :
+ Whether weights are written.
+ """
+ fsm = HfstBasicTransducer(self)
+ fsm.name = self.get_name()
+ prologstr = fsm.get_prolog_string(write_weights)
+ f.write(prologstr)
+
+ def write_xfst(self, f, write_weights=True):
+ """
+ Write the transducer in xfst format to file *f*, *write_weights* defined whether
+ weights are written.
+
+ Parameters
+ ----------
+ * `f` :
+ A python file where transducer is written.
+ * `write_weights` :
+ Whether weights are written.
+ """
+ fsm = HfstBasicTransducer(self)
+ fsm.name = self.get_name()
+ xfststr = fsm.get_xfst_string(write_weights)
+ f.write(xfst)
+
+ def write_att(self, f, write_weights=True):
+ """
+ Write the transducer in AT&T format to file *f*, *write_weights* defined whether
+ weights are written.
+
+ Parameters
+ ----------
+ * `f` :
+ A python file where transducer is written.
+ * `write_weights` :
+ Whether weights are written.
+ """
+ fsm = HfstBasicTransducer(self)
+ fsm.name = self.get_name()
+ attstr = fsm.get_att_string(write_weights)
+ f.write(attstr)
+
def lookup(self, input, **kvargs):
-
+ """
+ Lookup string *input*.
+
+ Parameters
+ ----------
+ * `input` :
+ The input. A string or a pre-tokenized tuple of symbols (i.e. a tuple of strings).
+ * `kvargs` :
+ Possible parameters and their default values are: obey_flags=True,
+ max_number=-1, time_cutoff=0.0, output='tuple'
+ * `obey_flags` :
+ Whether flag diacritics are obeyed. Always True for HFST_OL(W)_TYPE transducers.
+ * `max_number` :
+ Maximum number of results returned, defaults to -1, i.e. infinity.
+ * `time_cutoff` :
+ How long the function can search for results before returning, expressed in
+ seconds. Defaults to 0.0, i.e. infinitely. Always 0.0 for transducers that are
+ not of HFST_OL(W)_TYPE.
+ * `output` :
+ Possible values are 'tuple', 'text' and 'raw', 'tuple' being the default.
+
+ Note: This function has an efficient implementation only for optimized lookup format
+ (hfst.ImplementationType.HFST_OL_TYPE or hfst.ImplementationType.HFST_OLW_TYPE). Other formats perform the
+ lookup via composition. Consider converting the transducer to optimized lookup format
+ or to a HfstBasicTransducer. Conversion to HFST_OL(W)_TYPE might take a while but the
+ lookup is fast. Conversion to HfstBasicTransducer is quick but lookup is slower.
+ """
obey_flags=True
max_number=-1
time_cutoff=0.0
@@ -1137,31 +589,45 @@ HfstOneLevelPaths lookup_string(const std::string & s, int limit = -1, double ti
if isinstance(input, tuple):
if obey_flags:
- retval=self.lookup_fd_vector(input, max_number, time_cutoff)
+ retval=self._lookup_fd_vector(input, max_number, time_cutoff)
else:
- retval=self.lookup_vector(input, max_number, time_cutoff)
+ retval=self._lookup_vector(input, max_number, time_cutoff)
elif isinstance(input, str):
if obey_flags:
- retval=self.lookup_fd_string(input, max_number, time_cutoff)
+ retval=self._lookup_fd_string(input, max_number, time_cutoff)
else:
- retval=self.lookup_string(input, max_number, time_cutoff)
+ retval=self._lookup_string(input, max_number, time_cutoff)
else:
try:
if obey_flags:
- retval=self.lookup_fd_string(str(input), max_number, time_cutoff)
+ retval=self._lookup_fd_string(str(input), max_number, time_cutoff)
else:
- retval=self.lookup_string(str(input), max_number, time_cutoff)
+ retval=self._lookup_string(str(input), max_number, time_cutoff)
except:
raise RuntimeError('Input argument must be string or tuple.')
if output == 'text':
return one_level_paths_to_string(retval)
elif output == 'tuple':
- return one_level_paths_to_tuple(retval)
+ return _one_level_paths_to_tuple(retval)
else:
return retval
def extract_longest_paths(self, **kvargs):
+ """
+ Extract longest paths of the transducer.
+
+ Parameters
+ ----------
+ * `kvargs` :
+ Possible parameters and their default values are: obey_flags=True,
+ output='dict'
+ * `obey_flags` :
+ Whether flag diacritics are obeyed. The default is True.
+ * `output` :
+ Possible values are 'dict', 'text' and 'raw', 'dict' being the default.
+
+ """
obey_flags=True
output='dict' # 'dict' (default), 'text', 'raw'
@@ -1187,16 +653,27 @@ HfstOneLevelPaths lookup_string(const std::string & s, int limit = -1, double ti
else:
print('Warning: ignoring unknown argument %s.' % (k))
- retval = self.extract_longest_paths_(obey_flags)
+ retval = self._extract_longest_paths(obey_flags)
if output == 'text':
return two_level_paths_to_string(retval)
elif output == 'dict':
- return two_level_paths_to_dict(retval)
+ return _two_level_paths_to_dict(retval)
else:
return retval
def extract_shortest_paths(self, **kvargs):
+ """
+ Extract shortest paths of the transducer.
+
+ Parameters
+ ----------
+ * `kvargs` :
+ Possible parameters and their default values are: obey_flags=True.
+ * `output` :
+ Possible values are 'dict', 'text' and 'raw', 'dict' being the default.
+
+ """
output='dict' # 'dict' (default), 'text', 'raw'
for k,v in kvargs.items():
@@ -1213,17 +690,89 @@ HfstOneLevelPaths lookup_string(const std::string & s, int limit = -1, double ti
else:
print('Warning: ignoring unknown argument %s.' % (k))
- retval = self.extract_shortest_paths_()
+ retval = self._extract_shortest_paths()
if output == 'text':
return two_level_paths_to_string(retval)
elif output == 'dict':
- return two_level_paths_to_dict(retval)
+ return _two_level_paths_to_dict(retval)
else:
return retval
def extract_paths(self, **kvargs):
-
+ """
+
+ Extract paths that are recognized by the transducer.
+
+ Parameters
+ ----------
+ * `kvargs` :
+ Arguments recognized are filter_flags, max_cycles, max_number, obey_flags,
+ output, random.
+ * `filter_flags` :
+ Whether flags diacritics are filtered out from the result (default True).
+ * `max_cycles` :
+ Indicates how many times a cycle will be followed, with negative numbers
+ indicating unlimited (default -1 i.e. unlimited).
+ * `max_number` :
+ The total number of resulting strings is capped at this value, with 0 or
+ negative indicating unlimited (default -1 i.e. unlimited).
+ * `obey_flags` :
+ Whether flag diacritics are validated (default True).
+ * `output` :
+ Output format. Values recognized: 'text' (as a string, separated by
+ newlines), 'raw' (a dictionary that maps each input string into a list of
+ tuples of an output string and a weight), 'dict' (a dictionary that maps
+ each input string into a tuple of tuples of an output string and a weight,
+ the default).
+ * `random` :
+ Whether result strings are fetched randomly (default False).
+
+ Returns
+ -------
+ The extracted strings. *output* controls how they are represented.
+
+ pre: The transducer must be acyclic, if both *max_number* and *max_cycles* have
+ unlimited values. Else a hfst.exceptions.TransducerIsCyclicException will be
+ thrown.
+
+ An example:
+
+ >>> tr = hfst.regex('a:b+ (a:c+)')
+ >>> print(tr)
+ 0 1 a b 0.000000
+ 1 1 a b 0.000000
+ 1 2 a c 0.000000
+ 1 0.000000
+ 2 2 a c 0.000000
+ 2 0.000000
+
+ >>> print(tr.extract_paths(max_cycles=1, output='text'))
+ a:b 0
+ aa:bb 0
+ aaa:bbc 0
+ aaaa:bbcc 0
+ aa:bc 0
+ aaa:bcc 0
+
+ >>> print(tr.extract_paths(max_number=4, output='text'))
+ a:b 0
+ aa:bc 0
+ aaa:bcc 0
+ aaaa:bccc 0
+
+ >>> print(tr.extract_paths(max_cycles=1, max_number=4, output='text'))
+ a:b 0
+ aa:bb 0
+ aa:bc 0
+ aaa:bcc 0
+
+ Exceptions
+ ----------
+ * `TransducerIsCyclicException` :
+
+ See also: hfst.HfstTransducer.n_best
+ """
obey_flags=True
filter_flags=True
max_cycles=-1
@@ -1277,24 +826,51 @@ HfstOneLevelPaths lookup_string(const std::string & s, int limit = -1, double ti
if obey_flags :
if random :
- retval=self.extract_random_paths_fd_(max_number, filter_flags)
+ retval=self._extract_random_paths_fd(max_number, filter_flags)
else :
- retval=self.extract_paths_fd_(max_number, max_cycles)
+ retval=self._extract_paths_fd(max_number, max_cycles)
else :
if random :
- retval=self.extract_random_paths_(max_number)
- else :
- retval=self.extract_paths_(max_number, max_cycles)
+ retval=self._extract_random_paths(max_number)
+ else :
+ retval=self._extract_paths(max_number, max_cycles)
if output == 'text':
return two_level_paths_to_string(retval)
elif output == 'dict':
- return two_level_paths_to_dict(retval)
+ return _two_level_paths_to_dict(retval)
else:
return retval
def substitute(self, s, S=None, **kvargs):
-
+ """
+ Substitute symbols or transitions in the transducer.
+
+ Parameters
+ ----------
+ * `s` :
+ The symbol or transition to be substituted. Can also be a dictionary of
+ substitutions, if S == None.
+ * `S` :
+ The symbol, transition, a tuple of transitions or a transducer
+ (hfst.HfstTransducer) that substitutes *s*.
+ * `kvargs` :
+ Arguments recognized are 'input' and 'output', their values can be False or
+ True, True being the default. These arguments are valid only if *s* and *S*
+ are strings, else they are ignored.
+ * `input` :
+ Whether substitution is performed on input side, defaults to True. Valid
+ only if *s* and *S* are strings.
+ * `output` :
+ Whether substitution is performed on output side, defaults to True. Valid
+ only if *s* and \\ S are strings.
+
+ For more information, see hfst.HfstBasicTransducer.substitute. The function
+ works similarly, with the exception of argument *S*, which must be
+ hfst.HfstTransducer instead of hfst.HfstBasicTransducer.
+
+ See also: hfst.HfstBasicTransducer.substitute
+ """
if S == None:
if not isinstance(s, dict):
raise RuntimeError('Sole input argument must be a dictionary.')
@@ -1302,30 +878,30 @@ HfstOneLevelPaths lookup_string(const std::string & s, int limit = -1, double ti
subst_type=""
for k, v in s.items():
- if is_string(k):
+ if _is_string(k):
if subst_type == "":
subst_type="string"
elif subst_type == "string pair":
raise RuntimeError('')
- if not is_string(v):
+ if not _is_string(v):
raise RuntimeError('')
- elif is_string_pair(k):
+ elif _is_string_pair(k):
if subst_type == "":
subst_type="string pair"
elif subst_type == "string":
raise RuntimeError('')
- if not is_string_pair(v):
+ if not _is_string_pair(v):
raise RuntimeError('')
else:
raise RuntimeError('')
if subst_type == "string":
- return self.substitute_symbols(s)
+ return self._substitute_symbols(s)
else:
- return self.substitute_symbol_pairs(s)
+ return self._substitute_symbol_pairs(s)
- if is_string(s):
- if is_string(S):
+ if _is_string(s):
+ if _is_string(S):
input=True
output=True
for k,v in kvargs.items():
@@ -1337,16 +913,16 @@ HfstOneLevelPaths lookup_string(const std::string & s, int limit = -1, double ti
output=False
else:
raise RuntimeError('Free argument not recognized.')
- return self.substitute_symbol(s, S, input, output)
+ return self._substitute_symbol(s, S, input, output)
else:
raise RuntimeError('...')
- elif is_string_pair(s):
- if is_string_pair(S):
- return self.substitute_symbol_pair(s, S)
- elif is_string_pair_vector(S):
- return self.substitute_symbol_pair_with_set(s, S)
+ elif _is_string_pair(s):
+ if _is_string_pair(S):
+ return self._substitute_symbol_pair(s, S)
+ elif _is_string_pair_vector(S):
+ return self._substitute_symbol_pair_with_set(s, S)
elif isinstance(S, HfstTransducer):
- return self.substitute_symbol_pair_with_transducer(s, S, True)
+ return self._substitute_symbol_pair_with_transducer(s, S, True)
else:
raise RuntimeError('...')
else:
@@ -1355,34 +931,66 @@ HfstOneLevelPaths lookup_string(const std::string & s, int limit = -1, double ti
};
-};
+}; // class HfstTransducer
+
+
+// *** HfstOutputStream *** //
hfst::HfstOutputStream * create_hfst_output_stream(const std::string & filename, hfst::ImplementationType type, bool hfst_format);
class HfstOutputStream
{
public:
-//HfstOutputStream(ImplementationType type, bool hfst_format=true);
-//HfstOutputStream(const std::string &filename, ImplementationType type, bool hfst_format=true);
~HfstOutputStream(void);
HfstOutputStream &flush();
-//HfstOutputStream &operator<< (HfstTransducer &transducer);
-//HfstOutputStream& redirect (HfstTransducer &transducer);
void close(void);
-
%extend {
-void write(hfst::HfstTransducer & transducer) throw(StreamIsClosedException)
-{
- $self->redirect(transducer);
-}
-
+void write(hfst::HfstTransducer transducer) throw(StreamIsClosedException) { $self->redirect(transducer); }
HfstOutputStream() { return new hfst::HfstOutputStream(hfst::get_default_fst_type()); }
%pythoncode %{
def __init__(self, **kvargs):
+ """
+ Open a stream for writing binary transducers. Note: hfst.HfstTransducer.write_to_file
+ is probably the easiest way to write a single binary transducer to a file.
+
+ Parameters
+ ----------
+ * `kvargs` :
+ Arguments recognized are filename, hfst_format, type.
+ * `filename` :
+ The name of the file where transducers are written. If the file exists, it
+ is overwritten. If *filename* is not given, transducers are written to
+ standard output.
+ * `hfst_format` :
+ Whether transducers are written in hfst format (default is True) or as such
+ in their backend format.
+ * `type` :
+ The type of the transducers that will be written to the stream. Default is
+ hfst.get_default_fst_type().
+
+ Examples:
+
+ # a stream for writing default type transducers in hfst format to standard output
+ ostr = hfst.HfstOutputStream()
+ transducer = hfst.regex('foo:bar::0.5')
+ ostr.write(transducer)
+ ostr.flush()
+
+ # a stream for writing native sfst type transducers to a file
+ ostr = hfst.HfstOutputStream(filename='transducer.sfst', hfst_format=False, type=hfst.ImplementationType.SFST_TYPE)
+ transducer1 = hfst.regex('foo:bar')
+ transducer1.convert(hfst.ImplementationType.SFST_TYPE) # if not set as the default type
+ transducer2 = hfst.regex('bar:baz')
+ transducer2.convert(hfst.ImplementationType.SFST_TYPE) # if not set as the default type
+ ostr.write(transducer1)
+ ostr.write(transducer2)
+ ostr.flush()
+ ostr.close()
+ """
filename = ""
hfst_format = True
type = _libhfst.get_default_fst_type()
@@ -1401,7 +1009,9 @@ def __init__(self, **kvargs):
}
-};
+}; // class HfstOutputStream
+
+// *** HfstInputStream *** //
class HfstInputStream
{
@@ -1417,48 +1027,42 @@ public:
%extend {
-hfst::HfstTransducer * read() throw (EndOfStreamException)
-{
- return new hfst::HfstTransducer(*($self));
-}
+hfst::HfstTransducer * read() throw (EndOfStreamException) { return new hfst::HfstTransducer(*($self)); }
%pythoncode %{
def __iter__(self):
+ """
+ Return *self*. Needed for 'for ... in' statement.
+ """
return self
-# Python 2
def next(self):
+ """
+ Read next transducer from stream and return it. Needed for 'for ... in' statement.
+ """
if self.is_eof():
raise StopIteration
else:
return self.read();
-# Python 3
def __next__(self):
+ """
+ Read next transducer from stream and return it. Needed for 'for ... in' statement.
+ """
return self.next()
%}
}
-};
+}; // class HfstInputStream
- /* class MultiCharSymbolTrie;
- typedef std::vector<MultiCharSymbolTrie*> MultiCharSymbolTrieVector;
- typedef std::vector<bool> SymbolEndVector;
- class MultiCharSymbolTrie
- {
- public:
- MultiCharSymbolTrie(void);
- ~MultiCharSymbolTrie(void);
- void add(const char * p);
- const char * find(const char * p) const;
- }; */
+// *** HfstTokenizer *** //
class HfstTokenizer
- {
+ {
public:
HfstTokenizer();
void add_skip_symbol(const std::string &symbol);
@@ -1477,20 +1081,20 @@ namespace implementations {
class HfstBasicTransition;
typedef unsigned int HfstState;
- typedef std::vector<std::vector<hfst::implementations::HfstBasicTransition> > HfstBasicStates;
+ typedef std::vector<hfst::implementations::HfstBasicTransition> HfstBasicTransitions;
+
+
+// *** HfstBasicTransducer *** //
class HfstBasicTransducer {
public:
- typedef std::vector<HfstBasicTransition> HfstTransitions;
-
HfstBasicTransducer(void);
- //HfstBasicTransducer(FILE *file);
- //HfstBasicTransducer &assign(const HfstBasicTransducer &graph);
HfstBasicTransducer(const HfstBasicTransducer &graph);
HfstBasicTransducer(const hfst::HfstTransducer &transducer);
+ std::string name;
void add_symbol_to_alphabet(const std::string &symbol);
void remove_symbol_from_alphabet(const std::string &symbol);
void remove_symbols_from_alphabet(const StringSet &symbols);
@@ -1504,81 +1108,67 @@ class HfstBasicTransducer {
HfstState add_state(HfstState s);
HfstState get_max_state() const;
std::vector<HfstState> states() const;
- void add_transition(HfstState s, const hfst::implementations::HfstBasicTransition & transition,
- bool add_symbols_to_alphabet=true);
- void remove_transition(HfstState s, const hfst::implementations::HfstBasicTransition & transition,
- bool remove_symbols_from_alphabet=false);
+ void add_transition(HfstState s, const hfst::implementations::HfstBasicTransition & transition, bool add_symbols_to_alphabet=true);
+ void remove_transition(HfstState s, const hfst::implementations::HfstBasicTransition & transition, bool remove_symbols_from_alphabet=false);
bool is_final_state(HfstState s) const;
float get_final_weight(HfstState s) const throw(StateIsNotFinalException, StateIndexOutOfBoundsException);
void set_final_weight(HfstState s, const float & weight);
- //HfstBasicTransducer &sort_arcs(void);
- const std::vector<HfstBasicTransition> & transitions(HfstState s) const;
- //HfstBasicTransducer &disjunct(const StringPairVector &spv, float weight);
- //HfstBasicTransducer &harmonize(HfstBasicTransducer &another);
-
+%rename("_transitions") transitions(HfstState s);
+ hfst::implementations::HfstBasicTransitions & transitions(HfstState s);
bool is_infinitely_ambiguous();
bool is_lookup_infinitely_ambiguous(const StringVector & s);
int longest_path_size();
- //HfstBasicTransducer & substitute_symbol(const std::string &old_symbol, const std::string &new_symbol, bool input_side=true, bool output_side=true);
- //HfstBasicTransducer & substitute_symbol_pair(const StringPair &old_symbol_pair, const StringPair &new_symbol_pair);
- //HfstBasicTransducer & substitute_symbol_pair_with_set(const StringPair &old_symbol_pair, const hfst::StringPairSet &new_symbol_pair_set);
- //HfstBasicTransducer & substitute_symbol_pair_with_transducer(const StringPair &symbol_pair, HfstBasicTransducer &transducer);
- //HfstBasicTransducer & substitute_symbols(const hfst::HfstSymbolSubstitutions &substitutions); // alias for the previous function which is shadowed
- //HfstBasicTransducer & substitute_symbol_pairs(const hfst::HfstSymbolPairSubstitutions &substitutions); // alias for the previous function which is shadowed
-
- //HfstBasicTransducer & insert_freely(const StringPair &symbol_pair, float weight);
- //HfstBasicTransducer & insert_freely(const HfstBasicTransducer &tr);
-
- // void lookup_fd(const StringVector &lookup_path, HfstTwoLevelPaths &results, size_t infinite_cutoff, float * max_weight = NULL)
-
- hfst::implementations::HfstBasicStates states_and_transitions() const;
-
-
-
%extend {
- void substitute_symbol(const std::string &old_symbol, const std::string &new_symbol, bool input_side=true, bool output_side=true) { self->substitute_symbol(old_symbol, new_symbol, input_side, output_side); }
- void substitute_symbol_pair(const StringPair &old_symbol_pair, const StringPair &new_symbol_pair) { self->substitute_symbol_pair(old_symbol_pair, new_symbol_pair); }
- void substitute_symbol_pair_with_set(const StringPair &old_symbol_pair, const hfst::StringPairSet &new_symbol_pair_set) { self->substitute_symbol_pair_with_set(old_symbol_pair, new_symbol_pair_set); }
- void substitute_symbol_pair_with_transducer(const StringPair &symbol_pair, HfstBasicTransducer &transducer) { self->substitute_symbol_pair_with_transducer(symbol_pair, transducer); }
- void substitute_symbols(const hfst::HfstSymbolSubstitutions &substitutions) { self->substitute_symbols(substitutions); } // alias for the previous function which is shadowed
- void substitute_symbol_pairs(const hfst::HfstSymbolPairSubstitutions &substitutions) { self->substitute_symbol_pairs(substitutions); } // alias for the previous function which is shadowed
+ void _substitute_symbol(const std::string &old_symbol, const std::string &new_symbol, bool input_side=true, bool output_side=true) { self->substitute_symbol(old_symbol, new_symbol, input_side, output_side); }
+ void _substitute_symbol_pair(const StringPair &old_symbol_pair, const StringPair &new_symbol_pair) { self->substitute_symbol_pair(old_symbol_pair, new_symbol_pair); }
+ void _substitute_symbol_pair_with_set(const StringPair &old_symbol_pair, const hfst::StringPairSet &new_symbol_pair_set) { self->substitute_symbol_pair_with_set(old_symbol_pair, new_symbol_pair_set); }
+ void _substitute_symbol_pair_with_transducer(const StringPair &symbol_pair, HfstBasicTransducer &transducer) { self->substitute_symbol_pair_with_transducer(symbol_pair, transducer); }
+ void _substitute_symbols(const hfst::HfstSymbolSubstitutions &substitutions) { self->substitute_symbols(substitutions); } // alias for the previous function which is shadowed
+ void _substitute_symbol_pairs(const hfst::HfstSymbolPairSubstitutions &substitutions) { self->substitute_symbol_pairs(substitutions); } // alias for the previous function which is shadowed
void insert_freely(const StringPair &symbol_pair, float weight) { self->insert_freely(symbol_pair, weight); }
void insert_freely(const HfstBasicTransducer &tr) { self->insert_freely(tr); }
void sort_arcs() { self->sort_arcs(); }
void disjunct(const StringPairVector &spv, float weight) { self->disjunct(spv, weight); }
void harmonize(HfstBasicTransducer &another) { self->harmonize(another); }
- HfstTwoLevelPaths lookup_fd_(const StringVector &lookup_path, size_t * infinite_cutoff, float * max_weight)
+ HfstTwoLevelPaths _lookup(const StringVector &lookup_path, size_t * infinite_cutoff, float * max_weight, bool obey_flags)
{
hfst::HfstTwoLevelPaths results;
- $self->lookup_fd(lookup_path, results, infinite_cutoff, max_weight);
+ $self->lookup(lookup_path, results, infinite_cutoff, max_weight, obey_flags);
return results;
}
- void write_prolog(hfst::HfstFile & f, const std::string & name, bool write_weights=true) {
- $self->write_in_prolog_format(f.get_file(), name, write_weights);
- }
- static HfstBasicTransducer read_prolog(hfst::HfstFile & f) {
- unsigned int linecount = 0;
- return hfst::implementations::HfstBasicTransducer::read_in_prolog_format(f.get_file(), linecount);
- }
- void write_xfst(hfst::HfstFile & f, bool write_weights=true) {
- $self->write_in_xfst_format(f.get_file(), write_weights);
+
+ std::string get_prolog_string(bool write_weights)
+ {
+ std::ostringstream oss;
+ $self->write_in_prolog_format(oss, self->name, write_weights);
+ return oss.str();
}
- void write_att(hfst::HfstFile & f, bool write_weights=true) {
- $self->write_in_att_format(f.get_file(), write_weights);
+
+ std::string get_xfst_string(bool write_weights)
+ {
+ std::ostringstream oss;
+ $self->write_in_xfst_format(oss, write_weights);
+ return oss.str();
}
- static HfstBasicTransducer read_att(hfst::HfstFile & f, std::string epsilon="@_EPSILON_SYMBOL_@") throw(EndOfStreamException, NotValidAttFormatException) {
- unsigned int linecount = 0;
- return hfst::implementations::HfstBasicTransducer::read_in_att_format(f.get_file(), epsilon, linecount);
+
+ std::string get_att_string(bool write_weights)
+ {
+ std::ostringstream oss;
+ $self->write_in_att_format(oss, write_weights);
+ std::string retval = oss.str();
+ if (retval == "") // empty transducer must be represented as empty line in python, else read_att fails...
+ retval = std::string("\n");
+ return retval;
}
char * __str__()
{
- static char str[1024];
- $self->write_in_att_format(str, true); // write_weights=true
- return str;
+ std::ostringstream oss;
+ $self->write_in_att_format(oss, true);
+ return strdup(oss.str().c_str());
}
void add_transition(HfstState source, HfstState target, std::string input, std::string output, float weight=0) {
@@ -1587,22 +1177,111 @@ class HfstBasicTransducer {
}
%pythoncode %{
+ def transitions(self, s):
+ """
+ ...
+ """
+ tr = self._transitions(s)
+ retval = []
+ for i in range(0, len(tr)):
+ retval.append(tr[i])
+ return retval
+
+ def states_and_transitions(self):
+ """
+ ...
+ """
+ retval = []
+ for s in self.states():
+ retval.append(self.transitions(s))
+ return retval
+
def __iter__(self):
+ """
+ Return states and transitions of the transducer.
+ """
return self.states_and_transitions().__iter__()
def __enumerate__(self):
+ """
+ Return an enumeration of states and transitions of the transducer.
+ """
return enumerate(self.states_and_transitions())
- def lookup_fd(self, lookup_path, **kvargs):
+ def write_prolog(self, f, write_weights=True):
+ """
+ Write the transducer in prolog format with name *name* to file *f*,
+ *write_weights* defined whether weights are written.
+
+ Parameters
+ ----------
+ * `f` :
+ A python file where the transducer is written.
+ * `write_weights` :
+ Whether weights are written.
+ """
+ prologstr = self.get_prolog_string(write_weights)
+ f.write(prologstr)
+
+ def write_xfst(self, f, write_weights=True):
+ """
+ Write the transducer in xfst format to file *f*, *write_weights* defined whether
+ weights are written.
+
+ Parameters
+ ----------
+ * `f` :
+ A python file where transducer is written.
+ * `write_weights` :
+ Whether weights are written.
+ """
+ xfststr = self.get_xfst_string(write_weights)
+ f.write(prologstr)
+
+ def write_att(self, f, write_weights=True):
+ """
+ Write the transducer in AT&T format to file *f*, *write_weights* defined whether
+ weights are written.
+
+ Parameters
+ ----------
+ * `f` :
+ A python file where transducer is written.
+ * `write_weights` :
+ Whether weights are written.
+ """
+ attstr = self.get_att_string(write_weights)
+ f.write(attstr)
+
+ def lookup(self, lookup_path, **kvargs):
+ """
+ Lookup tokenized input *input* in the transducer.
+
+ Parameters
+ ----------
+ * `str` :
+ A list/tuple of strings to look up.
+ * `kvargs` :
+ infinite_cutoff=-1, max_weight=None, obey_flags=False
+ * `max_epsilon_loops` :
+ How many times epsilon input loops are followed. Defaults to -1, i.e. infinitely.
+ * `max_weight` :
+ What is the maximum weight of a result allowed. Defaults to None, i.e. infinity.
+ * `obey_flags` :
+ Whether flag diacritic constraints are obeyed. Defaults to False.
+ """
max_weight = None
- infinite_cutoff = None
+ max_epsilon_loops = None
+ obey_flags = False
output='dict' # 'dict' (default), 'text', 'raw'
for k,v in kvargs.items():
if k == 'max_weight' :
max_weight=v
- elif k == 'infinite_cutoff' :
+ elif k == 'max_epsilon_loops' :
infinite_cutoff=v
+ elif k == 'obey_flags' :
+ obey_flags=v
elif k == 'output':
if v == 'text':
output == 'text'
@@ -1616,17 +1295,70 @@ class HfstBasicTransducer {
else:
print('Warning: ignoring unknown argument %s.' % (k))
- retval = self.lookup_fd_(lookup_path, infinite_cutoff, max_weight)
+ retval = self._lookup(lookup_path, max_epsilon_loops, max_weight, obey_flags)
if output == 'text':
- return two_level_paths_to_string(retval)
+ return _two_level_paths_to_string(retval)
elif output == 'dict':
- return two_level_paths_to_dict(retval)
+ return _two_level_paths_to_dict(retval)
else:
return retval
def substitute(self, s, S=None, **kvargs):
-
+ """
+
+ Substitute symbols or transitions in the transducer.
+
+ Parameters
+ ----------
+ * `s` :
+ The symbol or transition to be substituted. Can also be a dictionary of
+ substitutions, if S == None.
+ * `S` :
+ The symbol, transition, a tuple of transitions or a transducer
+ (hfst.HfstBasicTransducer) that substitutes *s*.
+ * `kvargs` :
+ Arguments recognized are 'input' and 'output', their values can be False or
+ True, True being the default. These arguments are valid only if *s* and *S*
+ are strings, else they are ignored.
+ * `input` :
+ Whether substitution is performed on input side, defaults to True. Valid
+ only if *s* and *S* are strings.
+ * `output` :
+ Whether substitution is performed on output side, defaults to True. Valid
+ only if *s* and *S* are strings.
+
+ Possible combinations of arguments and their types are:
+
+ (1) substitute(str, str, input=bool, output=bool): substitute symbol with symbol
+ on input, output or both sides of each transition in the transducer. (2)
+ substitute(strpair, strpair): substitute transition with transition (3)
+ substitute(strpair, strpairtuple): substitute transition with several
+ transitions (4) substitute(strpair, transducer): substitute transition with a
+ transducer (5) substitute(dict): perform several symbol-to-symbol substitutions
+ (6) substitute(dict): perform several transition-to-transition substitutions
+
+ Examples:
+
+ (1) tr.substitute('a', 'A', input=True, output=False): substitute lowercase a:s
+ with uppercase ones (2) tr.substitute(('a','b'),('A','B')): substitute
+ transitions that map lowercase a into lowercase b with transitions that map
+ uppercase a into uppercase b (3) tr.substitute(('a','b'),
+ (('A','B'),('a','B'),('A','b'))): change either or both sides of a transition
+ [a:b] to uppercase (4) tr.substitute(('a','b'), hfst.regex('[a:b]+')) change
+ [a:b] transition into one or more consecutive [a:b] transitions (5)
+ tr.substitute({'a':'A', 'b':'B', 'c':'C'}) change lowercase a, b and c into
+ their uppercase variants (6) tr.substitute( {('a','a'):('A','A'),
+ ('b','b'):('B','B'), ('c','c'):('C','C')} ): change lowercase a, b and c into
+ their uppercase variants
+
+ In case (4), epsilon transitions are used to attach copies of transducer *S*
+ between the SOURCE and TARGET state of each transition that is substituted. The
+ transition itself is deleted, but its weight is copied to the epsilon transition
+ leading from SOURCE to the initial state of *S*. Each final state of *S* is made
+ non-final and an epsilon transition leading to TARGET is attached to it. The
+ final weight is copied to the epsilon transition.
+ """
if S == None:
if not isinstance(s, dict):
raise RuntimeError('First input argument must be a dictionary.')
@@ -1634,30 +1366,30 @@ class HfstBasicTransducer {
subst_type=""
for k, v in s.items():
- if is_string(k):
+ if _is_string(k):
if subst_type == "":
subst_type="string"
elif subst_type == "string pair":
raise RuntimeError('')
- if not is_string(v):
+ if not _is_string(v):
raise RuntimeError('')
- elif is_string_pair(k):
+ elif _is_string_pair(k):
if subst_type == "":
subst_type="string pair"
elif subst_type == "string":
raise RuntimeError('')
- if not is_string_pair(v):
+ if not _is_string_pair(v):
raise RuntimeError('')
else:
raise RuntimeError('')
if subst_type == "string":
- return self.substitute_symbols(s)
+ return self._substitute_symbols(s)
else:
- return self.substitute_symbol_pairs(s)
+ return self._substitute_symbol_pairs(s)
- if is_string(s):
- if is_string(S):
+ if _is_string(s):
+ if _is_string(S):
input=True
output=True
for k,v in kvargs.items():
@@ -1669,16 +1401,16 @@ class HfstBasicTransducer {
output=False
else:
raise RuntimeError('Free argument not recognized.')
- return self.substitute_symbol(s, S, input, output)
+ return self._substitute_symbol(s, S, input, output)
else:
raise RuntimeError('...')
- elif is_string_pair(s):
- if is_string_pair(S):
- return self.substitute_symbol_pair(s, S)
- elif is_string_pair_vector(S):
- return self.substitute_symbol_pair_with_set(s, S)
+ elif _is_string_pair(s):
+ if _is_string_pair(S):
+ return self._substitute_symbol_pair(s, S)
+ elif _is_string_pair_vector(S):
+ return self._substitute_symbol_pair_with_set(s, S)
elif isinstance(S, HfstBasicTransducer):
- return self.substitute_symbol_pair_with_transducer(s, S)
+ return self._substitute_symbol_pair_with_transducer(s, S)
else:
raise RuntimeError('...')
else:
@@ -1688,7 +1420,9 @@ class HfstBasicTransducer {
}
-};
+}; // class HfstBasicTransducer
+
+// *** HfstBasicTransition *** //
class HfstBasicTransition {
public:
@@ -1697,8 +1431,11 @@ class HfstBasicTransition {
~HfstBasicTransition();
HfstState get_target_state() const;
std::string get_input_symbol() const;
+ void set_input_symbol(const std::string & symbol);
std::string get_output_symbol() const;
+ void set_output_symbol(const std::string & symbol);
float get_weight() const;
+ void set_weight(float f);
%extend{
char *__str__() {
@@ -1708,22 +1445,12 @@ class HfstBasicTransition {
}
}
-};
+}; // class HfstBasicTransition
-}
+} // namespace implementations
-namespace pmatch {
- class PmatchCompiler
- {
- public:
- PmatchCompiler();
- PmatchCompiler(hfst::ImplementationType impl);
- void set_flatten(bool val) { flatten = val; }
- void set_verbose(bool val) { verbose = val; }
- void define(const std::string& name, const std::string& pmatch);
- std::map<std::string, HfstTransducer*> compile(const std::string& pmatch);
- };
-}
+
+// *** XreCompiler: offer only a limited set of functions ***
namespace xre {
class XreCompiler
@@ -1731,30 +1458,38 @@ class XreCompiler
public:
XreCompiler();
XreCompiler(hfst::ImplementationType impl);
- //XreCompiler(const struct XreConstructorArguments & args);
- void define(const std::string& name, const std::string& xre);
void define_list(const std::string& name, const std::set<std::string>& symbol_list);
- bool define_function(const std::string& name,
- unsigned int arguments,
- const std::string& xre);
+ bool define_function(const std::string& name, unsigned int arguments, const std::string& xre);
bool is_definition(const std::string& name);
bool is_function_definition(const std::string& name);
- void define(const std::string& name, const HfstTransducer & transducer);
void undefine(const std::string& name);
HfstTransducer* compile(const std::string& xre);
- HfstTransducer* compile_first(const std::string& xre, unsigned int & chars_read);
- bool contained_only_comments();
- bool get_positions_of_symbol_in_xre
- (const std::string & symbol, const std::string & xre, std::set<unsigned int> & positions);
- void set_expand_definitions(bool expand);
- void set_harmonization(bool harmonize);
- void set_flag_harmonization(bool harmonize_flags);
void set_verbosity(bool verbose);
- XreCompiler& setOutputToConsole(bool value);
bool getOutputToConsole();
+ void set_expand_definitions(bool expand); // TODO: should this be set automatically to True?
+
+ // *** Some wrappers *** //
+%extend{
+ void define_xre(const std::string& name, const std::string& xre)
+ {
+ self->set_expand_definitions(true);
+ self->define(name, xre);
+ }
+ void define_transducer(const std::string& name, const HfstTransducer & transducer)
+ {
+ self->set_expand_definitions(true);
+ self->define(name, transducer);
+ }
+ void setOutputToConsole(bool value)
+ {
+ (void)self->setOutputToConsole(value);
+ }
+}
+
};
}
+// *** The LexcCompiler functions are offered only because they are needed in some python functions... *** //
namespace lexc {
class LexcCompiler
@@ -1763,44 +1498,13 @@ namespace lexc {
LexcCompiler();
LexcCompiler(hfst::ImplementationType impl);
LexcCompiler(hfst::ImplementationType impl, bool withFlags, bool alignStrings);
- LexcCompiler& parse(FILE* infile);
- LexcCompiler& parse(const char* filename);
LexcCompiler& setVerbosity(unsigned int verbose);
- unsigned int getVerbosity();
- bool isQuiet();
- LexcCompiler& setTreatWarningsAsErrors(bool value);
- bool areWarningsTreatedAsErrors();
- LexcCompiler& setAllowMultipleSublexiconDefinitions(bool value);
- LexcCompiler& setWithFlags(bool value);
- LexcCompiler& setMinimizeFlags(bool value);
- LexcCompiler& setRenameFlags(bool value);
- LexcCompiler& addAlphabet(const std::string& alphabet);
- LexcCompiler& addNoFlag(const std::string& lexname);
- LexcCompiler& setCurrentLexiconName(const std::string& lexicon_name);
- LexcCompiler& addStringEntry(const std::string& entry,
- const std::string& continuation,
- const double weight);
- LexcCompiler& addStringPairEntry(const std::string& upper,
- const std::string& lower,
- const std::string& continuation,
- const double weight);
- LexcCompiler& addXreEntry(const std::string& xre,
- const std::string& continuation,
- const double weight);
- LexcCompiler& addXreDefinition(const std::string& name,
- const std::string& xre);
- LexcCompiler& setInitialLexiconName(const std::string& lexicon_name);
- hfst::HfstTransducer* compileLexical();
- // not implemented?: const std::map<std::string,hfst::HfstTransducer>& getStringTries() const;
- // not implemented?: const std::map<std::string,hfst::HfstTransducer>& getRegexpUnions() const;
- const LexcCompiler& printConnectedness(bool & warnings_printed);
void setOutputToConsole(bool);
- bool getOutputToConsole();
};
-// ugh, the global
-//extern LexcCompiler* lexc_;
+
}
+// *** The XfstCompiler functions are offered only because they are needed in some python functions... *** //
namespace xfst {
class XfstCompiler
@@ -1808,27 +1512,20 @@ namespace xfst {
public:
XfstCompiler();
XfstCompiler(hfst::ImplementationType impl);
- //XfstCompiler& compile_regex(const char * indata, unsigned int & chars_read);
- //int parse(FILE * infile);
- //int parse(const char * filename);
- //int parse_line(char line []);
- int parse_line(std::string line);
- //XfstCompiler& setReadline(bool readline);
- XfstCompiler& setReadInteractiveTextFromStdin(bool value);
XfstCompiler& setOutputToConsole(bool value);
- //bool getReadline();
- //bool getReadInteractiveTextFromStdin();
- //bool getOutputToConsole();
+ XfstCompiler& setReadInteractiveTextFromStdin(bool Value);
+ XfstCompiler& setReadline(bool value);
XfstCompiler& setVerbosity(bool verbosity);
- XfstCompiler& setPromptVerbosity(bool verbosity);
- bool quit_requested();
- std::string get(const char *);
- const XfstCompiler& prompt();
- char* get_prompt() const;
XfstCompiler& set(const char* name, const char* text);
+ char * get_prompt() const;
+ XfstCompiler& apply_up(const char* indata);
+ XfstCompiler& apply_down(const char* indata);
+ int parse_line(std::string line);
+ bool quit_requested() const;
};
}
+// internal functions
std::string hfst::get_hfst_regex_error_message();
hfst::HfstTransducer * hfst::hfst_regex(hfst::xre::XreCompiler & comp, const std::string & regex_string, const std::string & error_stream);
@@ -1841,49 +1538,52 @@ int hfst::hfst_compile_xfst(hfst::xfst::XfstCompiler & comp, std::string input,
std::string hfst::get_hfst_lexc_output();
hfst::HfstTransducer * hfst::hfst_compile_lexc(hfst::lexc::LexcCompiler & comp, const std::string & filename, const std::string & error_stream);
-void hfst::set_default_fst_type(hfst::ImplementationType t);
-hfst::ImplementationType hfst::get_default_fst_type();
-std::string hfst::fst_type_to_string(hfst::ImplementationType t);
-
-hfst::HfstTransducer * hfst::read_att(hfst::HfstFile & f, std::string epsilon="@_EPSILON_SYMBOL_@") throw(EndOfStreamException, NotValidAttFormatException);
-hfst::HfstTransducer * hfst::read_prolog(hfst::HfstFile & f) throw(EndOfStreamException);
+std::string hfst::get_hfst_sfst_output();
+hfst::HfstTransducer * hfst::hfst_compile_sfst(const std::string & filename, const std::string & error_stream, bool verbose, bool output_to_console);
std::string hfst::one_level_paths_to_string(const HfstOneLevelPaths &);
std::string hfst::two_level_paths_to_string(const HfstTwoLevelPaths &);
+bool parse_prolog_network_line(const std::string & line, hfst::implementations::HfstBasicTransducer * graph);
+bool parse_prolog_arc_line(const std::string & line, hfst::implementations::HfstBasicTransducer * graph);
+bool parse_prolog_symbol_line(const std::string & line, hfst::implementations::HfstBasicTransducer * graph);
+bool parse_prolog_final_line(const std::string & line, hfst::implementations::HfstBasicTransducer * graph);
-hfst_ol::PmatchContainer * create_pmatch_container(const std::string & filename);
-namespace rules {
- enum ReplaceType {REPL_UP, REPL_DOWN, REPL_RIGHT, REPL_LEFT, REPL_DOWN_KARTTUNEN};
- enum TwolType {twol_right, twol_left, twol_both};
-}
+// fuctions visible under module hfst
+
+void hfst::set_default_fst_type(hfst::ImplementationType t);
+hfst::ImplementationType hfst::get_default_fst_type();
+std::string hfst::fst_type_to_string(hfst::ImplementationType t);
+
+
+// *** hfst_rules (will be wrapped under module hfst.rules) *** //
namespace hfst_rules {
HfstTransducer two_level_if(const HfstTransducerPair & context, const StringPairSet & mappings, const StringPairSet & alphabet);
- HfstTransducer two_level_only_if(const HfstTransducerPair &context, const StringPairSet &mappings, const StringPairSet &alphabet);
- HfstTransducer two_level_if_and_only_if(const HfstTransducerPair &context, const StringPairSet &mappings, const StringPairSet &alphabet);
- HfstTransducer replace_down(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
- HfstTransducer replace_down_karttunen(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
- HfstTransducer replace_right(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
- HfstTransducer replace_left(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
- HfstTransducer replace_up(const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
- HfstTransducer replace_down(const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
- HfstTransducer left_replace_up(const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
- HfstTransducer left_replace_up(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
- HfstTransducer left_replace_down(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
- HfstTransducer left_replace_down_karttunen(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
- HfstTransducer left_replace_left(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
- HfstTransducer left_replace_right(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
- HfstTransducer restriction(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet);
- HfstTransducer coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet);
- HfstTransducer restriction_and_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet);
- HfstTransducer surface_restriction(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet);
- HfstTransducer surface_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet);
- HfstTransducer surface_restriction_and_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet);
- HfstTransducer deep_restriction(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet);
- HfstTransducer deep_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet);
+ HfstTransducer two_level_only_if(const HfstTransducerPair &context, const StringPairSet &mappings, const StringPairSet &alphabet);
+ HfstTransducer two_level_if_and_only_if(const HfstTransducerPair &context, const StringPairSet &mappings, const StringPairSet &alphabet);
+ HfstTransducer replace_down(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
+ HfstTransducer replace_down_karttunen(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
+ HfstTransducer replace_right(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
+ HfstTransducer replace_left(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
+ HfstTransducer replace_up(const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
+ HfstTransducer replace_down(const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
+ HfstTransducer left_replace_up(const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
+ HfstTransducer left_replace_up(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
+ HfstTransducer left_replace_down(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
+ HfstTransducer left_replace_down_karttunen(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
+ HfstTransducer left_replace_left(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
+ HfstTransducer left_replace_right(const HfstTransducerPair &context, const HfstTransducer &mapping, bool optional, const StringPairSet &alphabet);
+ HfstTransducer restriction(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet);
+ HfstTransducer coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet);
+ HfstTransducer restriction_and_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet);
+ HfstTransducer surface_restriction(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet);
+ HfstTransducer surface_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet);
+ HfstTransducer surface_restriction_and_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet);
+ HfstTransducer deep_restriction(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet);
+ HfstTransducer deep_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet);
HfstTransducer deep_restriction_and_coercion(const HfstTransducerPairVector &contexts, const HfstTransducer &mapping, const StringPairSet &alphabet);
} // namespace hfst_rules
@@ -1891,366 +1591,70 @@ namespace hfst_rules {
} // namespace hfst
+// *** PmatchContainer *** //
+
namespace hfst_ol {
class PmatchContainer
{
public:
- //PmatchContainer(std::istream & is);
PmatchContainer(void);
+ PmatchContainer(hfst::HfstTransducerVector transducers);
~PmatchContainer(void);
std::string match(const std::string & input, double time_cutoff = 0.0);
std::string get_profiling_info(void);
void set_verbose(bool b);
- void set_extract_tags_mode(bool b);
+ //void set_extract_tags_mode(bool b);
void set_profile(bool b);
-%extend {
-
-%pythoncode %{
-
-def __init__(self, filename):
- self.this = _libhfst.create_pmatch_container(filename)
-
-%}
-
-}
-
- };
-}
-
-
-%pythoncode %{
-
-from sys import stdout
+}; // class PmatchContainer
+} // namespace hfst_ol
-EPSILON='@_EPSILON_SYMBOL_@'
-UNKNOWN='@_UNKNOWN_SYMBOL_@'
-IDENTITY='@_IDENTITY_SYMBOL_@'
-
-OUTPUT_TO_CONSOLE=False
-def set_output_to_console(val):
- global OUTPUT_TO_CONSOLE
- OUTPUT_TO_CONSOLE=val
-def get_output_to_console():
- return OUTPUT_TO_CONSOLE
-
-def regex(re, **kvargs):
- type = _libhfst.get_default_fst_type()
- to_console=get_output_to_console()
- import sys
- err=None
-
- for k,v in kvargs.items():
- if k == 'output_to_console':
- to_console=v
- if k == 'error':
- err=v
- else:
- print('Warning: ignoring unknown argument %s.' % (k))
-
- comp = XreCompiler(type)
- comp.setOutputToConsole(to_console)
-
- if err == None:
- return _libhfst.hfst_regex(comp, re, "")
- elif err == sys.stdout:
- return _libhfst.hfst_regex(comp, re, "cout")
- elif err == sys.stderr:
- return _libhfst.hfst_regex(comp, re, "cerr")
- else:
- retval = _libhfst.hfst_regex(comp, re, "")
- err.write(_libhfst.get_hfst_regex_error_message())
- return retval
-
-def replace_symbols(symbol):
- if symbol == "@0@":
- return EPSILON
- symbol = symbol.replace("@_SPACE_@", " ")
- symbol = symbol.replace("@_TAB_@", "\t")
- symbol = symbol.replace("@_COLON_@", ":")
- return symbol
-
-def parse_att_line(line, fsm):
- # get rid of extra whitespace
- line = line.replace('\t',' ')
- line = " ".join(line.split())
- fields = line.split(' ')
- if len(fields) == 1:
- fsm.add_state(int(fields[0]))
- fsm.set_final_weight(int(fields[0]), 0)
- elif len(fields) == 2:
- fsm.add_state(int(fields[0]))
- fsm.set_final_weight(int(fields[0]), float(fields[1]))
- elif len(fields) == 4:
- fsm.add_transition(int(fields[0]), int(fields[1]), replace_symbols(fields[2]), replace_symbols(fields[3]), 0)
- elif len(fields) == 5:
- fsm.add_transition(int(fields[0]), int(fields[1]), replace_symbols(fields[2]), replace_symbols(fields[3]), float(fields[4]))
- else:
- return False
- return True
-
-def read_att_string(att):
- fsm = HfstBasicTransducer()
- lines = att.split('\n')
- for line in lines:
- if not parse_att_line(line, fsm):
- raise NotValidAttFormatException()
- return HfstTransducer(fsm, _libhfst.get_default_fst_type())
-
-def read_att_input():
- fsm = HfstBasicTransducer()
- while True:
- line = input().rstrip()
- if line == "":
- break
- if not parse_att_line(line, fsm):
- raise NotValidAttFormatException()
- return HfstTransducer(fsm, _libhfst.get_default_fst_type())
-
-def start_xfst(**kvargs):
- import sys
- idle = 'idlelib' in sys.modules
- if idle:
- print('It seems that you are running python in in IDLE. Note that all output from xfst will be buffered.')
- print('This means that all warnings, e.g. about time-consuming operations, will be printed only after the operation is carried out.')
- print('Consider running python from shell, for example command prompt, if you wish to see output with no delays.')
-
- type = _libhfst.get_default_fst_type()
- quit_on_fail = 'OFF'
- to_console=get_output_to_console()
- for k,v in kvargs.items():
- if k == 'type':
- type = v
- elif k == 'output_to_console':
- to_console=v
- elif k == 'quit_on_fail':
- if v == True:
- quit_on_fail='ON'
- else:
- print('Warning: ignoring unknown argument %s.' % (k))
-
- comp = XfstCompiler(type)
- comp.setReadInteractiveTextFromStdin(True)
-
- if to_console and idle:
- print('Cannot output to console when running libhfst from IDLE.')
- to_console=False
- comp.setOutputToConsole(to_console)
- comp.set('quit-on-fail', quit_on_fail)
-
- expression=""
- import sys
- while True:
- expression += input(comp.get_prompt()).rstrip()
- if expression[-1] == '\\':
- expression = expression[:-2] + '\n'
- continue
- retval = -1
- if idle:
- retval = _libhfst.hfst_compile_xfst_to_string_one(comp, expression)
- stdout.write(_libhfst.get_hfst_xfst_string_one())
- else:
- retval = comp.parse_line(expression + "\n")
- if retval != 0:
- print("expression '%s' could not be parsed" % expression)
- if comp.get("quit-on-fail") == "ON":
- return
- if comp.quit_requested():
- break
- expression = ""
-
-def compile_xfst_file(filename, **kvargs):
- verbosity=0
- quit_on_fail='ON'
- type = _libhfst.get_default_fst_type()
- output=None
- error=None
- to_console=get_output_to_console()
-
- for k,v in kvargs.items():
- if k == 'verbosity':
- verbosity=v
- elif k == 'quit_on_fail':
- if v == False:
- quit_on_fail='OFF'
- elif k == 'output':
- output=v
- elif k == 'error':
- error=v
- elif k == 'output_to_console':
- to_console=v
- else:
- print('Warning: ignoring unknown argument %s.' % (k))
-
- if verbosity > 1:
- print('Compiling with %s implementation...' % _libhfst.fst_type_to_string(type))
- xfstcomp = XfstCompiler(type)
- xfstcomp.setOutputToConsole(to_console)
- xfstcomp.setVerbosity(verbosity > 0)
- xfstcomp.set('quit-on-fail', quit_on_fail)
- if verbosity > 1:
- print('Opening xfst file %s...' % filename)
- f = open(filename, 'r', encoding='utf-8')
- data = f.read()
- f.close()
- if verbosity > 1:
- print('File closed...')
-
- retval=-1
- import sys
- from io import StringIO
-
- # check special case
- if isinstance(output, StringIO) and isinstance(error, StringIO) and output == error:
- retval =_libhfst.hfst_compile_xfst_to_string_one(xfstcomp, data)
- output.write(_libhfst.get_hfst_xfst_string_one())
- else:
- arg1 = ""
- arg2 = ""
- if output == None or output == sys.stdout:
- arg1 = "cout"
- if output == sys.stderr:
- arg1 == "cerr"
- if error == None or error == sys.stderr:
- arg2 = "cerr"
- if error == sys.stdout:
- arg2 == "cout"
-
- retval = _libhfst.hfst_compile_xfst(xfstcomp, data, arg1, arg2)
-
- if isinstance(output, StringIO):
- output.write(_libhfst.get_hfst_xfst_string_one())
- if isinstance(error, StringIO):
- error.write(_libhfst.get_hfst_xfst_string_two())
-
- if verbosity > 1:
- print('Parsed file with return value %i (0 indicating succesful parsing).' % retval)
- return retval
-
-def compile_lexc_file(filename, **kvargs):
- verbosity=0
- withflags=False
- alignstrings=False
- type = _libhfst.get_default_fst_type()
- output=None
- to_console=get_output_to_console()
-
- for k,v in kvargs.items():
- if k == 'verbosity':
- verbosity=v
- elif k == 'with_flags':
- if v == True:
- withflags = v
- elif k == 'align_strings':
- alignstrings = v
- elif k == 'output':
- output=v
- elif k == 'output_to_console':
- to_console=v
- else:
- print('Warning: ignoring unknown argument %s.' % (k))
-
- lexccomp = LexcCompiler(type, withflags, alignstrings)
- lexccomp.setVerbosity(verbosity)
- lexccomp.setOutputToConsole(to_console)
-
- retval=-1
- import sys
- if output == None:
- retval = _libhfst.hfst_compile_lexc(lexccomp, filename, "")
- elif output == sys.stdout:
- retval = _libhfst.hfst_compile_lexc(lexccomp, filename, "cout")
- elif output == sys.stderr:
- retval = _libhfst.hfst_compile_lexc(lexccomp, filename, "cerr")
- else:
- retval = _libhfst.hfst_compile_lexc(lexccomp, filename, "")
- output.write(_libhfst.get_hfst_lexc_output())
-
- return retval
-
-def is_weighted_word(arg):
- if isinstance(arg, tuple) and len(arg) == 2 and isinstance(arg[0], str) and isinstance(arg[1], (int, float)):
- return True
- return False
-
-def check_word(arg):
- if len(arg) == 0:
- raise RuntimeError('Empty word.')
- return arg
-
-def fsa(arg):
- deftok = HfstTokenizer()
- retval = HfstBasicTransducer()
- if isinstance(arg, str):
- retval.disjunct(deftok.tokenize(check_word(arg)), 0)
- elif is_weighted_word(arg):
- retval.disjunct(deftok.tokenize(check_word(arg[0])), arg[1])
- elif isinstance(arg, tuple) or isinstance(arg, list):
- for word in arg:
- if is_weighted_word(word):
- retval.disjunct(deftok.tokenize(check_word(word[0])), word[1])
- elif isinstance(word, str):
- retval.disjunct(deftok.tokenize(check_word(word)), 0)
- else:
- raise RuntimeError('Tuple/list element not a string or tuple of string and weight.')
- else:
- raise RuntimeError('Not a string or tuple/list of strings.')
- return HfstTransducer(retval, _libhfst.get_default_fst_type())
-
-def fst(arg):
- if isinstance(arg, dict):
- retval = regex('[0-0]') # empty transducer
- for input, output in arg.items():
- if not isinstance(input, str):
- raise RuntimeError('Key not a string.')
- left = fsa(input)
- right = 0
- if isinstance(output, str):
- right = fsa(output)
- elif isinstance(output, list) or isinstance(output, tuple):
- right = fsa(output)
- else:
- raise RuntimeError('Value not a string or tuple/list of strings.')
- left.cross_product(right)
- retval.disjunct(left)
- return retval
- return fsa(arg)
-
-def tokenized_fst(arg, weight=0):
- retval = HfstBasicTransducer()
- state = 0
- if isinstance(arg, list) or isinstance(arg, tuple):
- for token in arg:
- if isinstance(token, str):
- new_state = retval.add_state()
- retval.add_transition(state, new_state, token, token, 0)
- state = new_state
- elif isinstance(token, list) or isinstance(token, tuple):
- if len(token) == 2:
- new_state = retval.add_state()
- retval.add_transition(state, new_state, token[0], token[1], 0)
- state = new_state
- elif len(token) == 1:
- new_state = retval.add_state()
- retval.add_transition(state, new_state, token, token, 0)
- state = new_state
- else:
- raise RuntimeError('Symbol or symbol pair must be given.')
- retval.set_final_weight(state, weight)
- return HfstTransducer(retval, _libhfst.get_default_fst_type())
- else:
- raise RuntimeError('Argument must be a list or a tuple')
-
-def empty_fst():
- return regex('[0-0]')
-
-def epsilon_fst(weight=0):
- return regex('[0]::' + str(weight))
+%pythoncode %{
-def concatenate(transducers):
- retval = empty_fst()
- for tr in transducers:
- retval.concatenate(tr)
- return retval.minimize()
+class ImplementationType:
+ """
+ Back-end implementation.
+
+ Attributes:
+
+ SFST_TYPE: SFST type, unweighted
+ TROPICAL_OPENFST_TYPE: OpenFst type with tropical weights
+ LOG_OPENFST_TYPE: OpenFst type with logarithmic weights (limited support)
+ FOMA_TYPE: FOMA type, unweighted
+ XFSM_TYPE: XFST type, unweighted (limited support)
+ HFST_OL_TYPE: HFST optimized-lookup type, unweighted
+ HFST_OLW_TYPE: HFST optimized-lookup type, weighted
+ HFST2_TYPE: HFST version 2 legacy type
+ UNSPECIFIED_TYPE: type not specified
+ ERROR_TYPE: (something went wrong)
+
+ """
+ SFST_TYPE = _libhfst.SFST_TYPE
+ TROPICAL_OPENFST_TYPE = _libhfst.TROPICAL_OPENFST_TYPE
+ LOG_OPENFST_TYPE = _libhfst.LOG_OPENFST_TYPE
+ FOMA_TYPE = _libhfst.FOMA_TYPE
+ XFSM_TYPE = _libhfst.XFSM_TYPE
+ HFST_OL_TYPE = _libhfst.HFST_OL_TYPE
+ HFST_OLW_TYPE = _libhfst.HFST_OLW_TYPE
+ HFST2_TYPE = _libhfst.HFST2_TYPE
+ UNSPECIFIED_TYPE = _libhfst.UNSPECIFIED_TYPE
+ ERROR_TYPE = _libhfst.ERROR_TYPE
+
+class ReplaceType:
+ """
+ Replace type in Xerox-type rules.
+
+ Attributes:
+
+ REPL_UP: Match contexts on input level
+ REPL_DOWN: Match contexts on output level
+ REPL_RIGHT: Match left contexts on input level and right contexts on output level
+ REPL_LEFT: Match left contexts on output level and right contexts on input level
+
+ """
+ REPL_UP = _libhfst.REPL_UP
+ REPL_DOWN = _libhfst.REPL_DOWN
+ REPL_RIGHT = _libhfst.REPL_RIGHT
+ REPL_LEFT = _libhfst.REPL_LEFT
%}
diff --git a/python/pypi/COPYING b/python/pypi/COPYING
new file mode 100644
index 0000000..7961421
--- /dev/null
+++ b/python/pypi/COPYING
@@ -0,0 +1,674 @@
+ GNU GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users. We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors. You can apply it to
+your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights. Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+ For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received. You must make sure that they, too, receive
+or can get the source code. And you must show them these terms so they
+know their rights.
+
+ Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+ For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software. For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+ Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so. This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software. The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable. Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products. If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+ Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary. To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+ Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+ No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+ When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+ 4. Conveying Verbatim Copies.
+
+ You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+ You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+ 5. Conveying Modified Source Versions.
+
+ You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.
+
+ A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+ 6. Conveying Non-Source Forms.
+
+ You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.
+
+ d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.
+
+ A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+ A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+ "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+ If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+ The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+ Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+ 7. Additional Terms.
+
+ "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+ When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+ Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.
+
+ All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+ If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+ Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+ 8. Termination.
+
+ You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+ However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+ Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+ Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+ 9. Acceptance Not Required for Having Copies.
+
+ You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+ 10. Automatic Licensing of Downstream Recipients.
+
+ Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+ An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+ You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+ 11. Patents.
+
+ A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+ A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+ In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+ If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+ If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+ A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+ Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+ 12. No Surrender of Others' Freedom.
+
+ If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+ 13. Use with the GNU Affero General Public License.
+
+ Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+ 14. Revised Versions of this License.
+
+ The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+ If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+ Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+ 15. Disclaimer of Warranty.
+
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. Limitation of Liability.
+
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+ 17. Interpretation of Sections 15 and 16.
+
+ If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+ If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+ <program> Copyright (C) <year> <name of author>
+ This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ This is free software, and you are welcome to redistribute it
+ under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License. Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+ You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<http://www.gnu.org/licenses/>.
+
+ The GNU General Public License does not permit incorporating your program
+into proprietary programs. If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library. If this is what you want to do, use the GNU General
+Public License instead of this License. But first, please read
+<http://www.gnu.org/philosophy/why-not-lgpl.html>.
diff --git a/python/pypi/MANIFEST.in b/python/pypi/MANIFEST.in
new file mode 100644
index 0000000..bc4613a
--- /dev/null
+++ b/python/pypi/MANIFEST.in
@@ -0,0 +1,22 @@
+include hfst_extensions.cc
+include hfst_file_extensions.cc
+include hfst_lexc_extensions.cc
+include hfst_lookup_extensions.cc
+include hfst_pmatch_extensions.cc
+include hfst_prolog_extensions.cc
+include hfst_regex_extensions.cc
+include hfst_rules_extensions.cc
+include hfst_xfst_extensions.cc
+include docstrings.i
+include libhfst/src/*.h
+include libhfst/src/implementations/*.h
+include libhfst/src/implementations/optimized-lookup/*.h
+include libhfst/src/implementations/compose_intersect/*.h
+include libhfst/src/parsers/*.h
+include libhfst/src/parsers/*.hh
+include back-ends/foma/*.h
+include back-ends/openfst/src/include/fst/*.h
+include config.h
+include setup.cfg
+include COPYING
+include README.rst
diff --git a/python/pypi/README b/python/pypi/README
new file mode 100644
index 0000000..e69de29
diff --git a/python/pypi/README.rst b/python/pypi/README.rst
new file mode 100644
index 0000000..bef326f
--- /dev/null
+++ b/python/pypi/README.rst
@@ -0,0 +1,116 @@
+
+*******************
+Package description
+*******************
+
+Package ``hfst`` contains python bindings for `HFST <https://hfst.github.io>`_
+(Helsinki Finite-State Technology) C++ library. HFST toolkit is intended for
+processing natural language morphologies. The toolkit is demonstrated by
+wide-coverage implementations of a number of languages of varying
+morphological complexity.
+
+Requirements
+############
+
+The bindings have been tested with python3. Wheels are offered only for python
+version 3.4 and higher for Windows and Mac OS X. For Linux users, we recommend
+using the `Debian packages
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstPython#Option_1_Installing_the_debian_p>`_.
+
+We currently offer only 64-bit wheels for Windows. They also require a 64-bit
+python to work correctly. Wheels for Mac are compiled as universal binaries
+that work on both 32- and 64-bit environments. OS X must be 10.7 or higher.
+
+Compiling from source requires at least swig (tested with versions 2.0.4 and
+3.0.5), a C++ compiler (tested with gcc 4.6.3, clang x.y.z and Visual C++ 10.0
+for python 3.4 and 14.0 for python >= 3.5), and python3 with setuptools
+(tested with version 28.8.0). All these must be located on directories listed
+on system PATH. On Linux and Mac OS X, readline and getline libraries must be
+available and the C++ compiler must support flag 'std=c++0x'. A known issue
+on OS X is that compiling C code fails as flag 'std=c++0x' must be set
+globally in setup.py but is not accepted when the source is pure C.
+
+Installation
+############
+
+We recommend using ``pip`` tool for installation. For python version 3, it is
+usually named ``pip3`` (plain ``pip`` being used for python version 2).
+Starting from python 3.4, pip is included by default and can be called with
+``python3 -m pip``.
+
+Basic installation with ``pip3`` on command line:
+
+``pip3 install [--upgrade] hfst``
+
+or, starting from python version 3.4, directly via python:
+
+``python3 -m pip install [--upgrade] hfst``
+
+The commands above are run in a shell/terminal/command prompt, but they can
+also be run on python command line or via a graphical user interface
+(e.g. IDLE) with pip.main that takes arguments in a list:
+
+| ``import pip``
+| ``pip.main(['install','--upgrade','hfst'])``
+
+
+Alternative `installation instructions <https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstPython>`_
+are given on our KitWiki pages.
+
+Documentation
+#############
+
+See Doxygen-generated `package documentation <https://hfst.github.io/python>`_
+on our Github pages. In python, you can also use ``dir`` and ``help``
+commands, e.g.:
+
+``dir(hfst)``
+
+``help(hfst.HfstTransducer)``
+
+License
+#######
+
+HFST is licensed under Gnu GPL version 3.0.
+
+Troubleshooting
+###############
+
+*Pip starts to compile from source although there is a wheel available:*
+
+Try upgrading pip with ``pip3 install --upgrade pip`` or
+``python3 -m pip install --upgrade pip``. Another reason for this can be that
+the source package on PyPI is newer (i.e. has a higher version number) than
+the corresponding wheel for the given environment. Report this via our
+`issue tracker <https://github.com/hfst/hfst/issues/>`_ so a fresh wheel
+will be created.
+
+*Error message "command ... failed with error code ...":*
+
+Try rerunning pip in verbose mode with
+``pip3 install --verbose [--upgrade] hfst`` to get more information.
+
+*Error message "error: could not delete ... : permission denied":*
+
+You do not have sufficient rights to install packages. On Mac and Linux, try
+installing as super user with ``sudo pip3 install [--upgrade] hfst``.
+On Windows, reopen Command Prompt/Python command line/IDLE by right-clicking
+and choose "Run as administrator", then run pip again.
+
+*Using flag -std=c++0x causes an error in the C++/C compiler:*
+
+This is a known issue on OS X when compiling from source. The flag must be
+set globally in setup.py but is not accepted when the source is pure C, as
+some of our back-end files are. If there isn't a wheel available for your
+environment, see alternative
+`installation instructions <https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstPython>`_.
+
+Links
+#####
+
+`HFST project main page <https://hfst.github.io>`_: more information about
+the project
+
+`Github issue tracker <https://github.com/hfst/hfst/issues/>`_: for comments,
+feature requests and bug reports
+
diff --git a/python/pypi/config.h b/python/pypi/config.h
new file mode 100644
index 0000000..e69de29
diff --git a/python/pypi/copy-files.sh b/python/pypi/copy-files.sh
new file mode 100755
index 0000000..b2e3428
--- /dev/null
+++ b/python/pypi/copy-files.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+if ! [ -d "back-ends" ]; then mkdir back-ends; fi
+if ! [ -d "libhfst" ]; then mkdir libhfst; fi
+if ! [ -d "hfst" ]; then mkdir hfst; fi
+
+cp -R ../../back-ends/* back-ends/
+cp -R ../../libhfst/* libhfst/
+cp -R ../hfst/* hfst/
+
+for file in hfst_extensions.cc hfst_file_extensions.cc hfst_lexc_extensions.cc \
+hfst_lookup_extensions.cc hfst_pmatch_extensions.cc hfst_prolog_extensions.cc \
+hfst_regex_extensions.cc hfst_rules_extensions.cc hfst_xfst_extensions.cc \
+hfst_sfst_extensions.cc libhfst.i docstrings.i ;
+do
+ cp ../$file $file
+done
diff --git a/python/pypi/setup.cfg b/python/pypi/setup.cfg
new file mode 100644
index 0000000..79bc678
--- /dev/null
+++ b/python/pypi/setup.cfg
@@ -0,0 +1,5 @@
+[bdist_wheel]
+# This flag says that the code is written to work on both Python 2 and Python
+# 3. If at all possible, it is good practice to do this. If you cannot, you
+# will need to generate wheels for each Python version that you support.
+universal=1
diff --git a/python/pypi/setup.py b/python/pypi/setup.py
new file mode 100644
index 0000000..16745f6
--- /dev/null
+++ b/python/pypi/setup.py
@@ -0,0 +1,234 @@
+#!/usr/bin/python3
+
+"""
+Setup for creating PIP packages for HFST Python bindings.
+
+Before running setup, recursively copy directories 'libhfst/src'
+and 'back-ends' from HFST c++ source code under the directory where
+setup is run. Make sure that the following c++ and header files
+from 'libhfst/src/parsers' have been generated from flex/yacc
+files before copying:
+
+ lexc-parser.cc pmatch_parse.cc xfst-parser.cc xre_parse.cc
+ lexc-parser.hh pmatch_parse.hh xfst-parser.hh xre_parse.hh
+
+Compiling the extensions requires python, swig and a c++ compiler,
+all located on a directory listed on system PATH. On linux and mac
+osx, readline and getline must be available and the c++ compiler
+must support flag 'std=c++11'.
+
+The setup script has been tested on linux with gcc 4.6.3, swig 3.0.12 and
+python 3.4 and on windows with msvc 10.0 and swig 3.0.5 (with python 3.3.
+and 3.4) and with msvc 14.0 (with python 3.5 and 3.6).
+
+"""
+
+from setuptools import setup, Extension
+
+from sys import platform
+if platform == "darwin":
+ import os
+ os.environ["_PYTHON_HOST_PLATFORM"] = 'macosx-10.7-x86_64'
+
+def readme():
+ with open('README.rst') as f:
+ return f.read()
+
+# HFST C++ headers needed by swig when creating the python/c++ interface
+swig_include_dir = "libhfst/src/"
+
+ext_swig_opts = ["-c++", "-I" + swig_include_dir, "-Wall"]
+import sys
+# for python3.3 and python3.4 on windows, add SDK include directory
+if platform == "win32" and sys.version_info[0] == 3 and (sys.version_info[1] == 3 or sys.version_info[1] == 4):
+ ext_swig_opts.extend(["-IC:\\Program Files (x86)\\Microsoft SDKs\\Windows\\v7.0A\\Include"])
+
+# readline is needed for hfst.start_xfst(), on windows the shell where HFST
+# python bindings are run from has its own readline which will do
+ext_extra_link_args = []
+if platform == "linux" or platform == "linux2" or platform == "darwin":
+ ext_extra_link_args = ['-lreadline']
+if platform == "darwin":
+ ext_extra_link_args.extend(['-mmacosx-version-min=10.7'])
+
+# HFST headers needed when compiling the actual c++ extension
+ext_include_dirs = [".", "libhfst/src/", "back-ends/foma", "back-ends",
+ "parsers", "libhfst/src/parsers"]
+if platform == "win32":
+ ext_include_dirs.append("back-ends/openfstwin/src/include")
+else:
+ ext_include_dirs.append("back-ends/openfst/src/include")
+
+# this replaces ./configure
+ext_define_macros = [ ('HAVE_FOMA', None), ('HAVE_OPENFST', None),
+ ('HAVE_OPENFST_LOG', None) ]
+if platform == "linux" or platform == "linux2" or platform == "darwin":
+ ext_define_macros.append(('HAVE_READLINE', None))
+ ext_define_macros.append(('HAVE_GETLINE', None))
+if platform == "win32":
+ # MSC_VER_ should already be defined
+ for macro in ["HFSTEXPORT", "OPENFSTEXPORT", "WINDOWS", "WIN32", "_CRT_SECURE_NO_WARNINGS"]:
+ ext_define_macros.append((macro, None))
+
+# use c++0x standard, if possible
+ext_extra_compile_args = []
+if platform == "linux" or platform == "linux2" or platform == "darwin":
+ ext_extra_compile_args = ["-std=c++0x", "-Wno-sign-compare", "-Wno-strict-prototypes"]
+if platform == "darwin":
+ ext_extra_compile_args.extend(["-stdlib=libc++", "-mmacosx-version-min=10.7"])
+# define error handling mechanism on windows
+if platform == "win32":
+ ext_extra_compile_args = ["/EHsc"]
+
+ext_library_dirs = []
+ext_libraries = []
+
+# on windows, c++ source files have 'cpp' extension
+cpp = ".cc"
+if platform == "win32":
+ cpp = ".cpp"
+
+# on windows, openfst back-end is in directory 'openfstwin'
+openfstdir = "openfst"
+if platform == "win32":
+ openfstdir = "openfstwin"
+
+# all c++ extension source files
+libhfst_source_files = ["libhfst/src/parsers/XfstCompiler" + cpp,
+ "libhfst/src/HfstApply" + cpp,
+ "libhfst/src/HfstInputStream" + cpp,
+ "libhfst/src/HfstTransducer" + cpp,
+ "libhfst/src/HfstOutputStream" + cpp,
+ "libhfst/src/HfstRules" + cpp,
+ "libhfst/src/HfstXeroxRules" + cpp,
+ "libhfst/src/HfstDataTypes" + cpp,
+ "libhfst/src/HfstSymbolDefs" + cpp,
+ "libhfst/src/HfstTokenizer" + cpp,
+ "libhfst/src/HfstFlagDiacritics" + cpp,
+ "libhfst/src/HfstExceptionDefs" + cpp,
+ "libhfst/src/HarmonizeUnknownAndIdentitySymbols" + cpp,
+ "libhfst/src/HfstLookupFlagDiacritics" + cpp,
+ "libhfst/src/HfstEpsilonHandler" + cpp,
+ "libhfst/src/HfstStrings2FstTokenizer" + cpp,
+ "libhfst/src/HfstPrintDot" + cpp,
+ "libhfst/src/HfstPrintPCKimmo" + cpp,
+ "libhfst/src/hfst-string-conversions" + cpp,
+ "libhfst/src/string-utils" + cpp,
+ "libhfst/src/implementations/HfstBasicTransducer" + cpp,
+ "libhfst/src/implementations/HfstBasicTransition" + cpp,
+ "libhfst/src/implementations/ConvertTransducerFormat" + cpp,
+ "libhfst/src/implementations/HfstTropicalTransducerTransitionData" + cpp,
+ "libhfst/src/implementations/ConvertTropicalWeightTransducer" + cpp,
+ "libhfst/src/implementations/ConvertLogWeightTransducer" + cpp,
+ "libhfst/src/implementations/ConvertFomaTransducer" + cpp,
+ "libhfst/src/implementations/ConvertOlTransducer" + cpp,
+ "libhfst/src/implementations/TropicalWeightTransducer" + cpp,
+ "libhfst/src/implementations/LogWeightTransducer" + cpp,
+ "libhfst/src/implementations/FomaTransducer" + cpp,
+ "libhfst/src/implementations/HfstOlTransducer" + cpp,
+ "libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair" + cpp,
+ "libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon" + cpp,
+ "libhfst/src/implementations/compose_intersect/ComposeIntersectRule" + cpp,
+ "libhfst/src/implementations/compose_intersect/ComposeIntersectFst" + cpp,
+ "libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities" + cpp,
+ "libhfst/src/implementations/optimized-lookup/transducer" + cpp,
+ "libhfst/src/implementations/optimized-lookup/convert" + cpp,
+ "libhfst/src/implementations/optimized-lookup/ospell" + cpp,
+ "libhfst/src/implementations/optimized-lookup/pmatch" + cpp,
+ "libhfst/src/implementations/optimized-lookup/find_epsilon_loops" + cpp,
+ "libhfst/src/parsers/xre_lex" + cpp,
+ "libhfst/src/parsers/xre_parse" + cpp,
+ "libhfst/src/parsers/pmatch_parse" + cpp,
+ "libhfst/src/parsers/pmatch_lex" + cpp,
+ "libhfst/src/parsers/lexc-parser" + cpp,
+ "libhfst/src/parsers/lexc-lexer" + cpp,
+ "libhfst/src/parsers/xfst-parser" + cpp,
+ "libhfst/src/parsers/xfst-lexer" + cpp,
+ "libhfst/src/parsers/LexcCompiler" + cpp,
+ "libhfst/src/parsers/PmatchCompiler" + cpp,
+ "libhfst/src/parsers/XreCompiler" + cpp,
+ "libhfst/src/parsers/lexc-utils" + cpp,
+ "libhfst/src/parsers/pmatch_utils" + cpp,
+ "libhfst/src/parsers/xre_utils" + cpp,
+ "libhfst/src/parsers/xfst-utils" + cpp,
+ "libhfst/src/parsers/xfst_help_message" + cpp,
+ "libhfst/src/parsers/sfst-scanner" + cpp,
+ "libhfst/src/parsers/sfst-compiler" + cpp,
+ "libhfst/src/parsers/SfstCompiler" + cpp,
+ "libhfst/src/parsers/SfstAlphabet" + cpp,
+ "libhfst/src/parsers/SfstBasic" + cpp,
+ "libhfst/src/parsers/SfstUtf8" + cpp ]
+
+foma_source_files = [ "back-ends/foma/int_stack.c",
+ "back-ends/foma/define.c",
+ "back-ends/foma/determinize.c",
+ "back-ends/foma/apply.c",
+ "back-ends/foma/rewrite.c",
+ "back-ends/foma/topsort.c",
+ "back-ends/foma/flags.c",
+ "back-ends/foma/minimize.c",
+ "back-ends/foma/reverse.c",
+ "back-ends/foma/extract.c",
+ "back-ends/foma/sigma.c",
+ "back-ends/foma/structures.c",
+ "back-ends/foma/constructions.c",
+ "back-ends/foma/coaccessible.c",
+ "back-ends/foma/io.c",
+ "back-ends/foma/utf8.c",
+ "back-ends/foma/spelling.c",
+ "back-ends/foma/dynarray.c",
+ "back-ends/foma/mem.c",
+ "back-ends/foma/stringhash.c",
+ "back-ends/foma/trie.c",
+ "back-ends/foma/lex.yy.c",
+ "back-ends/foma/regex.c" ]
+
+openfst_source_files = [ "back-ends/" + openfstdir + "/src/lib/compat" + cpp,
+ "back-ends/" + openfstdir + "/src/lib/flags" + cpp,
+ "back-ends/" + openfstdir + "/src/lib/fst" + cpp,
+ "back-ends/" + openfstdir + "/src/lib/properties" + cpp,
+ "back-ends/" + openfstdir + "/src/lib/symbol-table" + cpp,
+ "back-ends/" + openfstdir + "/src/lib/symbol-table-ops" + cpp,
+ "back-ends/" + openfstdir + "/src/lib/util" + cpp ]
+
+libhfst_source_files = libhfst_source_files + openfst_source_files
+
+if platform == "linux" or platform == "linux2" or platform == "win32":
+ libhfst_source_files = libhfst_source_files + foma_source_files
+
+# clang doesn't accept "-std=c++11" flag when compiling C,
+# so foma back-end must be compiled separately
+# it seems that subprocess doesn't work, so you must compile them manually before running setup.py:
+# for file in back-ends/foma/*.c; do clang -fPIC -std=c99 -arch i386 -arch x86_64 -mmacosx-version-min=10.7 -DHAVE_FOMA -c $file ; done
+foma_object_files = []
+if platform == "darwin":
+ for file in foma_source_files:
+ foma_object_files.append(file.replace('back-ends/foma/','').replace('.c','.o'))
+
+# The HFST c++ extension
+libhfst_module = Extension('_libhfst',
+ language = "c++",
+ sources = ["libhfst.i"] + libhfst_source_files,
+ swig_opts = ext_swig_opts,
+ include_dirs = ext_include_dirs,
+ library_dirs = ext_library_dirs,
+ libraries = ext_libraries,
+ define_macros = ext_define_macros,
+ extra_link_args = ext_extra_link_args,
+ extra_compile_args = ext_extra_compile_args,
+ extra_objects = foma_object_files
+ )
+
+setup(name = 'hfst',
+ version = '3.12.1.0_beta',
+ author = 'HFST team',
+ author_email = 'hfst-bugs at helsinki.fi',
+ url = 'http://hfst.github.io/',
+ description = 'Python interface for HFST',
+ long_description = readme(),
+ license = 'GNU GPL3',
+ ext_modules = [libhfst_module],
+ py_modules = ["libhfst"],
+ packages = ["hfst", "hfst.exceptions", "hfst.sfst_rules", "hfst.xerox_rules"],
+ data_files = []
+ )
diff --git a/python/setup.py b/python/setup.py
index 734d9f2..3c2b627 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -10,8 +10,17 @@ from distutils.core import setup, Extension
libhfst_src_path = '../libhfst/src/'
absolute_libhfst_src_path = os.path.abspath(libhfst_src_path)
-# when making the debian package, replace extra_link_args
+extra_link_arguments = []
+# If you wish to link to the local HFST library, replace the above with:
+# extra_link_arguments = ["-Wl,-rpath=" + absolute_libhfst_src_path + "/.libs"]
+
+# When making the debian package, replace extra_link_args
# with ["-L/usr/lib/", "-Wl,-rpath=/usr/lib/"]
+
+# If you wish to link hfst c++ library statically, use:
+# library_dirs = []
+# libraries = []
+# extra_objects = absolute_libhfst_src_path + "/.libs/libhfst.a"
libhfst_module = Extension('_libhfst',
language = "c++",
sources = ["libhfst.i"],
@@ -20,18 +29,19 @@ libhfst_module = Extension('_libhfst',
include_dirs = [absolute_libhfst_src_path],
library_dirs = [absolute_libhfst_src_path + "/.libs"],
libraries = ["hfst"],
- extra_link_args = []
+ extra_link_args = extra_link_arguments
)
-# when making the windows package, replace data_files with
+# When making the windows package, replace data_files with
# ["libhfst-NN.dll", "libgcc_s_seh-1.dll"] or
# ["libhfst-NN.dll", "libgcc_s_dw2-1.dll"] or
setup(name = 'libhfst_swig',
- version = '3.10.0_beta',
+ version = '3.12.1_beta',
author = 'HFST team',
author_email = 'hfst-bugs at helsinki.fi',
- url = 'http://hfst.sourceforge.net',
- description = 'SWIG-bound new libhfst interface',
+ url = 'http://hfst.github.io/',
+ description = 'SWIG-bound hfst interface',
ext_modules = [libhfst_module],
py_modules = ["libhfst"],
+ packages = ["hfst", "hfst.exceptions", "hfst.sfst_rules", "hfst.xerox_rules"],
data_files = []
)
diff --git a/python/test/Makefile.am b/python/test/Makefile.am
index 9f0014d..c867790 100644
--- a/python/test/Makefile.am
+++ b/python/test/Makefile.am
@@ -1 +1,9 @@
-EXTRA_DIST = README test_hfst.py examples.py test_fail.xfst testfile.att test_pass.xfst test.lexc foobar.hfst
+EXTRA_DIST = README test.sh \
+ test_fail.xfst testfile.att testfile_fail.att test_pass.xfst test.lexc test.sfstpl \
+ cats_and_dogs.prolog cats_and_dogs_fail.prolog \
+ test_pmatch.py test_prolog.py test_exceptions.py test_dir_hfst.py \
+ test_dir_hfst_exceptions.py test_dir_hfst_sfst_rules.py test_tokenizer.py \
+ test_read_att_transducer.py test_xre.py test_hfst.py test_examples.py \
+ test_att_reader.py test_prolog_reader.py \
+ test_streams_1.py test_streams_2.py test_streams_3.py streets.txt \
+ test_xerox_rules.py
diff --git a/python/test/README b/python/test/README
index cc5a13e..c353edc 100644
--- a/python/test/README
+++ b/python/test/README
@@ -1,7 +1,17 @@
-This directory contains tests for HFST python bindings. The tests are
-still under construction, as are the python bindings.
+This folder contains tests for HFST python bindings. To perform the tests, run
-For the time being, run
+ ./test.sh [--python PYTHON] [--pythonpath PYTHONPATH]
+
+ * --python defines the python compiler used (defaults to 'python3')
+ * --pythonpath inserts PYTHONPATH to sys.path before running each test
+
+The option --pythonpath is useful if the hfst bindings are not yet installed
+or there is an older installation available which python should not use.
+The typical use is something like:
+
+ ./test.sh --pythonpath path/to/hfst-top-dir/python
+
+where path/to/hfst-top-dir/python is the parent folder of this folder, if you
+are compiling from source.
- ./test.sh
diff --git a/python/test/cat.prolog b/python/test/cat.prolog
new file mode 100644
index 0000000..edf2e2a
--- /dev/null
+++ b/python/test/cat.prolog
@@ -0,0 +1,5 @@
+network(CAT).
+arc(CAT, 0, 1, "c").
+arc(CAT, 1, 2, "a").
+arc(CAT, 2, 3, "t").
+final(CAT, 3).
diff --git a/python/test/cats_and_dogs.prolog b/python/test/cats_and_dogs.prolog
new file mode 100644
index 0000000..c095141
--- /dev/null
+++ b/python/test/cats_and_dogs.prolog
@@ -0,0 +1,20 @@
+network(CAT).
+arc(CAT, 0, 1, "c").
+arc(CAT, 1, 2, "a").
+arc(CAT, 2, 3, "t").
+final(CAT, 3).
+
+network(EMPTY).
+
+network(DOG).
+arc(DOG, 0, 1, "d":"c").
+arc(DOG, 1, 2, "o":"a").
+arc(DOG, 2, 3, "g":"t").
+final(DOG, 3, 0.5).
+
+network(CHAT).
+arc(CHAT, 0, 1, "c").
+arc(CHAT, 1, 2, "a":"h").
+arc(CHAT, 2, 3, "t":"a").
+arc(CHAT, 3, 4, "0":"t", -1.5).
+final(CHAT, 4).
diff --git a/python/test/cats_and_dogs_fail.prolog b/python/test/cats_and_dogs_fail.prolog
new file mode 100644
index 0000000..15db99a
--- /dev/null
+++ b/python/test/cats_and_dogs_fail.prolog
@@ -0,0 +1,31 @@
+network(CAT).
+arc(CAT, 0, 1, "c").
+arc(CAT, 1, 2, "a").
+arc(CAT, 2, 3, "t").
+final(CAT, 3).
+
+network(EMPTY).
+
+network(DOG).
+arc(DOG, 0, 1, "d":"c").
+arc(DOG, 1, 2, "o":"a").
+arc(DOG, 2, 3, "g":"t").
+final(DOG, 3, 0.5).
+
+network(CHAT).
+arc(CHAT, 0, 1, "c").
+arc(CHAT, 1, 2, "a":"h").
+arc(CHAT, 2, 3, "t":"a").
+arc(CHAT, 3, 4, "0":"t", -1.5).
+final(CHAT, 4).
+
+network(FAIL).
+arc(FAIL, 0, 1, "foo").
+arc(1, 2, "bar").
+final(FAIL, 2, 1).
+
+network(CAT).
+arc(CAT, 0, 1, "c").
+arc(CAT, 1, 2, "a").
+arc(CAT, 2, 3, "t").
+final(CAT, 3).
diff --git a/python/test/examples.py b/python/test/examples.py
deleted file mode 100644
index 4d79bfd..0000000
--- a/python/test/examples.py
+++ /dev/null
@@ -1,388 +0,0 @@
-import sys
-sys.path.insert(1, '/home/eaxelson/hfst-git/hfst/python/')
-
-
-# The examples given in doxygen documentation
-
-import libhfst
-
-# StreamIsClosedException
-try:
- tr = libhfst.regex('foo')
- outstr = libhfst.HfstOutputStream(filename='testfile')
- outstr.close()
- outstr.write(tr)
-except libhfst.StreamIsClosedException:
- print("Could not write transducer: stream to file was closed.")
-
-# TransducerIsCyclicException
-transducer = libhfst.regex('[a:b]*')
-try:
- results = transducer.extract_paths(output='text')
- print("The transducer has %i paths:" % len(results))
- print(results)
-except libhfst.TransducerIsCyclicException:
- print("The transducer is cyclic and has an infinite number of paths. Some of them:")
- results = transducer.extract_paths(output='text', max_cycles=5)
- print(results)
-
-# NotTransducerStreamException
-f = open('foofile', 'w')
-f.write('This is an ordinary text file.\n')
-f.close()
-try:
- instr = libhfst.HfstInputStream('foofile')
- tr = instr.read()
- print(tr)
- instr.close()
-except libhfst.NotTransducerStreamException:
- print("Could not print transducer: the file does not contain binary transducers.")
-
-f = open('testfile1.att', 'w')
-f.write('0 1 a b\n\
-1 2 c\n\
-2\n')
-f.close()
-f = libhfst.hfst_open('testfile1.att', 'r')
-try:
- tr = libhfst.read_att(f)
-except libhfst.NotValidAttFormatException:
- print('Could not read file: it is not in valid ATT format.')
-f.close()
-
-# StateIsNotFinalException
-tr = libhfst.HfstBasicTransducer()
-tr.add_state(1)
-# An exception is thrown as state number 1 is not final
-try:
- w = tr.get_final_weight(1)
-except libhfst.StateIsNotFinalException:
- print("State is not final.")
-
-# ContextTransducersAreNotAutomataException
-# try:
-# tr = libhfst.regex('a -> b || c:c __ c:d')
-# except libhfst.ContextTransducersAreNotAutomataException:
-# print("Context transducers must be automata.")
-
-# TransducersAreNotAutomataException
-tr1 = libhfst.regex('foo:bar')
-tr2 = libhfst.regex('bar:baz')
-try:
- tr1.cross_product(tr2)
-except libhfst.TransducersAreNotAutomataException:
- print('Transducers must be automata in cross product.')
-
-# StateIndexOutOfBoundsException
-tr = libhfst.HfstBasicTransducer()
-tr.add_state(1)
-try:
- w = tr.get_final_weight(2)
-except libhfst.StateIndexOutOfBoundsException:
- print('State number 2 does not exist')
-
-# TransducerTypeMismatchException:
-libhfst.set_default_fst_type(libhfst.TROPICAL_OPENFST_TYPE)
-tr1 = libhfst.regex('foo')
-tr2 = libhfst.regex('bar')
-tr2.convert(libhfst.FOMA_TYPE)
-try:
- tr1.disjunct(tr2)
-except libhfst.TransducerTypeMismatchException:
- print('The implementation types of transducers must be the same.')
-
-# fst
-# One unweighted identity path:
-if not libhfst.fst('foo').compare(libhfst.regex('{foo}')):
- raise RuntimeError('')
-# Weighted path: a tuple of string and number, e.g.
-if not libhfst.fst(('foo',1.4)).compare(libhfst.regex('{foo}::1.4')):
- raise RuntimeError('')
-if not libhfst.fst(('bar',-3)).compare(libhfst.regex('{bar}::-3')):
- raise RuntimeError('')
-if not libhfst.fst(('baz',0)).compare(libhfst.regex('{baz}')):
- raise RuntimeError('')
-# Several paths: a list or a tuple of paths and/or weighted paths, e.g.
-if not libhfst.fst(['foo', 'bar']).compare(libhfst.regex('{foo}|{bar}')):
- raise RuntimeError('')
-if not libhfst.fst(('foo', ('bar',5.0))).compare(libhfst.regex('{foo}|{bar}::5.0')):
- raise RuntimeError('')
-if not libhfst.fst(('foo', ('bar',5.0), 'baz', 'Foo', ('Bar',2.4))).compare(libhfst.regex('{foo}|{bar}::5.0|{baz}|{Foo}|{Bar}::2.4')):
- raise RuntimeError('')
-if not libhfst.fst([('foo',-1), ('bar',0), ('baz',3.5)]).compare(libhfst.regex('{foo}::-1|{bar}|{baz}::3.5')):
- raise RuntimeError('')
-# A dictionary
-if not libhfst.fst({'foo':'foo', 'bar':('foo',1.4), 'baz':(('foo',-1),'BAZ')}).compare(libhfst.regex('{foo}|{bar}:{foo}::1.4|{baz}:{foo}::-1|{baz}:{BAZ}')):
- raise RuntimeError('')
-
-# tokenized_fst
-tok = libhfst.HfstTokenizer()
-tok.add_multichar_symbol('foo')
-tok.add_multichar_symbol('bar')
-tr = libhfst.tokenized_fst(tok.tokenize('foobar', 'foobaz'))
-if not tr.compare(libhfst.regex('[foo:foo bar:b 0:a 0:z]')):
- raise RuntimeError('')
-
-# HfstBasicTransducer
-# Create an empty transducer
-# The transducer has initially one start state (number zero)
-# that is not final
-fsm = libhfst.HfstBasicTransducer()
-# Add two states to the transducer
-fsm.add_state(1)
-fsm.add_state(2)
-# Create a transition [foo:bar] leading to state 1 with weight 0.1
-tr = libhfst.HfstBasicTransition(1, 'foo', 'bar', 0.1)
-# and add it to state zero
-fsm.add_transition(0, tr)
-# Add a transition [baz:baz] with weight 0 from state 1 to state 2
-fsm.add_transition(1, libhfst.HfstBasicTransition(2, 'baz', 'baz', 0.0))
-# Set state 2 as final with weight 0.3
-fsm.set_final_weight(2, 0.3)
-# Go through all states
-for state, arcs in enumerate(fsm):
- for arc in arcs:
- print('%i ' % (state), end='')
- print(arc)
- if fsm.is_final_state(state):
- print('%i %f' % (state, fsm.get_final_weight(state)) )
-
-for state in fsm.states():
- for arc in fsm.transitions(state):
- print('%i ' % (state), end='')
- print(arc)
- if fsm.is_final_state(state):
- print('%i %f' % (state, fsm.get_final_weight(state)) )
-
-# HfstBasicTransducer.disjunct
-lexicon = libhfst.HfstBasicTransducer()
-tok = libhfst.HfstTokenizer()
-lexicon.disjunct(tok.tokenize('dog'), 0.3)
-lexicon.disjunct(tok.tokenize('cat'), 0.5)
-lexicon.disjunct(tok.tokenize('elephant'), 1.6)
-lexicon = libhfst.HfstTransducer(lexicon)
-if not lexicon.compare(libhfst.regex('{dog}::0.3|{cat}::0.5|{elephant}::1.6')):
- raise RuntimeError('')
-
-# HfstBasicTransducer.transitions
-for state in fsm.states():
- for arc in fsm.transitions(state):
- print('%i ' % (state), end='')
- print(arc)
- if fsm.is_final_state(state):
- print('%i %f' % (state, fsm.get_final_weight(state)) )
-
-# HfstBasicTransducer.substitute and HfstTransducer.substitute
-hfst = libhfst.regex('a:a')
-basic = libhfst.HfstBasicTransducer(hfst)
-hfst.substitute('a', 'A', input=True, output=False)
-basic.substitute('a', 'A', input=True, output=False)
-
-hfst = libhfst.regex('a a:b b')
-basic = libhfst.HfstBasicTransducer(hfst)
-hfst.substitute(('a','b'),('A','B'))
-basic.substitute(('a','b'),('A','B'))
-
-hfst = libhfst.regex('a a:b b')
-basic = libhfst.HfstBasicTransducer(hfst)
-hfst.substitute(('a','b'), (('A','B'),('a','B'),('A','b')))
-basic.substitute(('a','b'), (('A','B'),('a','B'),('A','b')))
-
-hfst = libhfst.regex('a a:b b')
-basic = libhfst.HfstBasicTransducer(hfst)
-hfst.substitute(('a','b'), libhfst.regex('[a:b]+'))
-basic.substitute(('a','b'), libhfst.HfstBasicTransducer(libhfst.regex('[a:b]+')))
-
-hfst = libhfst.regex('a b c d')
-basic = libhfst.HfstBasicTransducer(hfst)
-hfst.substitute({'a':'A', 'b':'B', 'c':'C'})
-basic.substitute({'a':'A', 'b':'B', 'c':'C'})
-
-hfst = libhfst.regex('a a:b b b:c c c:d d')
-basic = libhfst.HfstBasicTransducer(hfst)
-hfst.substitute( {('a','a'):('A','A'), ('b','b'):('B','B'), ('c','c'):('C','C')} )
-basic.substitute( {('a','a'):('A','A'), ('b','b'):('B','B'), ('c','c'):('C','C')} )
-
-# HfstBasicTransducer.enumerate
-for state, arcs in enumerate(fsm):
- for arc in arcs:
- print('%i ' % (state), end='')
- print(arc)
- if fsm.is_final_state(state):
- print('%i %f' % (state, fsm.get_final_weight(state)) )
-
-# HfstTransducer
-# argument handling
-transducer1 = libhfst.regex('foo:bar baz')
-transducer2 = libhfst.regex('FOO:BAR BAZ')
-transducer1.reverse()
-transducer1.disjunct(transducer2)
-if not transducer2.compare(libhfst.regex('FOO:BAR BAZ')):
- raise RuntimeError('')
-transducer1.reverse()
-transducer1.determinize()
-transducer1.reverse()
-transducer1.determinize()
-# implementation types
-tropical_transducer = libhfst.regex('foo')
-tropical_transducer.convert(libhfst.TROPICAL_OPENFST_TYPE)
-foma_transducer = libhfst.regex('foo')
-foma_transducer.convert(libhfst.FOMA_TYPE)
-# TODO: segfaults
-try:
- tropical_transducer.compare(foma_transducer)
-except libhfst.TransducerTypeMismatchException:
- print('Implementation types of transducers must be the same.')
-
-# read_att from file
-f = open('testfile2.att', 'w')
-f.write(
-"""0 1 foo bar 0.3
-1 0.5
---
-0 0.0
---
---
-0 0.0
-0 0 a <eps> 0.2
-""")
-f.close()
-
-transducers = []
-ifile = libhfst.hfst_open('testfile2.att', 'r')
-try:
- while (not ifile.is_eof()):
- t = libhfst.read_att(ifile, '<eps>')
- transducers.append(t)
- print("read one transducer")
-except libhfst.NotValidAttFormatException:
- print("Error reading transducer: not valid AT&T format.")
-ifile.close()
-print("Read %i transducers in total" % len(transducers))
-
-# read_att from string
-#att_str = """0 1 a b
-#1 2 c d
-#2
-#"""
-#print(att_str)
-#tr = libhfst.read_att(att_str, '@0@')
-#print(tr)
-#exit(0)
-
-# write_att
-tr1 = libhfst.regex('[foo:bar baz:0 " "]::0.3')
-tr2 = libhfst.empty_fst()
-tr3 = libhfst.epsilon_fst(0.5)
-tr4 = libhfst.regex('[foo]')
-tr5 = libhfst.empty_fst()
-
-f = libhfst.hfst_open('testfile3.att', 'w')
-for tr in [tr1, tr2, tr3, tr4]:
- tr.write_att(f)
- f.write('--\n')
-tr5.write_att(f)
-f.close()
-
-# extract_paths
-tr = libhfst.regex('a:b+ (a:c+)')
-print(tr)
-print(tr.extract_paths(max_cycles=1, output='text'))
-print(tr.extract_paths(max_number=4, output='text'))
-print(tr.extract_paths(max_cycles=1, max_number=4, output='text'))
-
-# HfstOutputStream
-res = ['foo:bar','0','0 - 0','"?":?','a* b+']
-ostr = libhfst.HfstOutputStream(filename='testfile1.hfst')
-for re in res:
- ostr.write(libhfst.regex(re))
- ostr.flush()
-ostr.close()
-
-# HfstInputStream
-istr = libhfst.HfstInputStream('testfile1.hfst')
-transducers = []
-while not (istr.is_eof()):
- transducers.append(istr.read())
-istr.close()
-if not len(transducers) == len(res):
- raise RuntimeError('')
-i=0
-for tr in transducers:
- if not tr.compare(libhfst.regex(res[i])):
- raise RuntimeError('')
- i+=1
-
-# push_weights
-
-# QuickStart (1/3)
-
-tr1 = libhfst.regex('foo:bar')
-tr2 = libhfst.regex('bar:baz')
-tr1.compose(tr2)
-print(tr1)
-
-# QuickStart (2/3)
-
-# Create as HFST basic transducer [a:b] with transition weight 0.3 and final weight 0.5.
-t = libhfst.HfstBasicTransducer()
-t.add_state(1)
-t.add_transition(0, 1, 'a', 'b', 0.3)
-t.set_final_weight(1, 0.5)
-
-# Convert to tropical OpenFst format (the default) and push weights toward final state.
-T = libhfst.HfstTransducer(t)
-T.push_weights(libhfst.TO_FINAL_STATE)
-
-# Convert back to HFST basic transducer.
-tc = libhfst.HfstBasicTransducer(T)
-try:
- # Rounding might affect the precision.
- if (0.79 < tc.get_final_weight(1)) and (tc.get_final_weight(1) < 0.81):
- print("TEST PASSED")
- else:
- raise RuntimeError('')
-# If the state does not exist or is not final
-except libhfst.HfstException:
- print("TEST FAILED: An exception thrown.")
- raise RuntimeError('')
-
-# QuickStart (3/3)
-
-libhfst.set_default_fst_type(libhfst.FOMA_TYPE)
-
-# Create a simple lexicon transducer [[foo bar foo] | [foo bar baz]].
-tok = libhfst.HfstTokenizer()
-tok.add_multichar_symbol('foo')
-tok.add_multichar_symbol('bar')
-tok.add_multichar_symbol('baz')
-
-words = libhfst.tokenized_fst(tok.tokenize('foobarfoo'))
-t = libhfst.tokenized_fst(tok.tokenize('foobarbaz'))
-words.disjunct(t)
-
-# Create a rule transducer that optionally replaces 'bar' with 'baz' between 'foo' and 'foo'.
-rule = libhfst.regex('bar (->) baz || foo _ foo')
-
-# Apply the rule transducer to the lexicon.
-words.compose(rule)
-words.minimize()
-
-# Extract all string pairs from the result and print them to standard output.
-results = 0
-try:
- # Extract paths and remove tokenization
- results = words.extract_paths(output='dict')
-except libhfst.TransducerIsCyclicException:
- # This should not happen because transducer is not cyclic.
- print("TEST FAILED")
- exit(1)
-
-for input,outputs in results.items():
- print('%s:' % input)
- for output in outputs:
- print(' %s\t%f' % (output[0], output[1]))
-
-
-# Foma issue
-
diff --git a/python/test/foo_att_prolog b/python/test/foo_att_prolog
deleted file mode 100644
index b14256a..0000000
--- a/python/test/foo_att_prolog
+++ /dev/null
@@ -1,3 +0,0 @@
--- in ATT format --
--- in prolog format --
-network(FOOBAR_TRANSDUCER).
diff --git a/python/test/foo_basic b/python/test/foo_basic
index 10d921b..9e7bdde 100644
--- a/python/test/foo_basic
+++ b/python/test/foo_basic
@@ -1,4 +1,4 @@
-0 1 foo bar 0.500000
-0 0 baz baz 0.000000
-0 0 baz BAZ 0.100000
-1 0.300000
+0 1 foo bar 0.5
+0 0 baz baz 0
+0 0 baz BAZ 0.1
+1 0.3
diff --git a/python/test/foobar.hfst b/python/test/foobar.hfst
deleted file mode 100644
index 4a6eacd..0000000
Binary files a/python/test/foobar.hfst and /dev/null differ
diff --git a/python/test/foobar2.hfst b/python/test/foobar2.hfst
deleted file mode 100644
index 580b1c4..0000000
Binary files a/python/test/foobar2.hfst and /dev/null differ
diff --git a/python/test/streets.txt b/python/test/streets.txt
new file mode 100644
index 0000000..5ecc4b1
--- /dev/null
+++ b/python/test/streets.txt
@@ -0,0 +1,6 @@
+set need-separators off
+define CapWord UppercaseAlpha Alpha* ;
+define StreetWordFr [{avenue} | {boulevard} | {rue}] ;
+define DeFr [ [{de} | {du} | {des} | {de la}] Whitespace ] | [{d'} | {l'}] ;
+define StreetFr StreetWordFr (Whitespace DeFr) CapWord+ ;
+regex StreetFr EndTag(FrenchStreetName) ;
diff --git a/python/test/test.bat b/python/test/test.bat
new file mode 100755
index 0000000..4d40c8d
--- /dev/null
+++ b/python/test/test.bat
@@ -0,0 +1,17 @@
+ at ECHO OFF
+
+for %%i in (test_dir_hfst.py test_dir_hfst_exceptions.py ^
+ test_dir_hfst_sfst_rules.py ^
+ test_tokenizer.py test_exceptions.py test_xre.py ^
+ test_read_att_transducer.py test_prolog.py ^
+ test_att_reader.py test_prolog_reader.py ^
+ test_hfst.py test_examples.py test_pmatch.py test_xerox_rules.py) DO echo testing %%i & %1 >NUL 2>NUL %%i && echo passed
+
+for %%f in (sfst openfst foma) DO echo testing streams with format %%f & %1 test_streams_1.py %%f | %1 test_streams_2.py %%f | %1 test_streams_3.py %%f && echo passed
+
+del foo
+del foo_att_prolog
+del testfile3.att
+del testfile_.att
+del tmp
+
diff --git a/python/test/test.py b/python/test/test.py
deleted file mode 100644
index 496a6c0..0000000
--- a/python/test/test.py
+++ /dev/null
@@ -1,6 +0,0 @@
-import libhfst
-ab = libhfst.regex('a:b::2.8')
-out = libhfst.HfstOutputStream(hfst_format=False)
-out.write(ab)
-out.flush()
-out.close()
diff --git a/python/test/test.sfstpl b/python/test/test.sfstpl
new file mode 100644
index 0000000..3f5b68d
--- /dev/null
+++ b/python/test/test.sfstpl
@@ -0,0 +1 @@
+{cat}:{dog}
diff --git a/python/test/test.sh b/python/test/test.sh
index 86b44de..43dcb47 100755
--- a/python/test/test.sh
+++ b/python/test/test.sh
@@ -1,25 +1,100 @@
-for file in test_hfst.py examples.py test.py; do
- if python3 $file > /dev/null 2> /dev/null; then
- echo $file" passed"
+
+PYTHON="python3"
+PYTHONPATH=""
+
+if [ "$1" = "--help" -o "$1" = "-h" ]; then
+ echo ""
+ echo "Run all tests in this folder."
+ echo ""
+ echo "Usage: test.sh [--python PYTHON] [--pythonpath PATH]"
+ echo ""
+ echo "PYTHON: the python to be used for testing, defaults to 'python3'"
+ echo "PATH: insert PATH to sys.path before running each test"
+ echo ""
+ exit 0
+fi
+
+if [ "$1" = "--python" ]; then
+ PYTHON=$2
+fi
+
+if [ "$1" = "--pythonpath" ]; then
+ PYTHONPATH=$2
+fi
+
+if [ "$3" = "--python" ]; then
+ PYTHON=$4
+fi
+
+if [ "$3" = "--pythonpath" ]; then
+ PYTHONPATH=$4
+fi
+
+for file in test_dir_hfst.py test_dir_hfst_exceptions.py test_dir_hfst_sfst_rules.py \
+ test_tokenizer.py test_exceptions.py test_xre.py \
+ test_read_att_transducer.py test_prolog.py \
+ test_att_reader.py test_prolog_reader.py \
+ test_pmatch.py test_xerox_rules.py;
+do
+ if ! [ "$PYTHONPATH" = "" ]; then
+ echo 'import sys' > tmp
+ echo 'sys.path.insert(0, "'$PYTHONPATH'")' >> tmp
+ cat $file >> tmp
+ else
+ cat $file > tmp
+ fi
+ if ( $PYTHON tmp 2> /dev/null > /dev/null ); then
+ echo $file" passed"
else
- echo $file" failed"
+ echo $file" failed"
+ exit 1
fi
done
-for n in 2 3 5 7 9; do
- if python3 test$n.py > /dev/null 2> /dev/null; then
- echo "test"$n".py passed"
+# give pythonpath as argument, from __future__ statement must come first
+for file in test_hfst.py test_examples.py;
+do
+ if ( $PYTHON $file $PYTHONPATH 2> /dev/null > /dev/null ); then
+ echo $file" passed"
else
- echo "test"$n".py failed"
+ echo $file" failed"
+ exit 1
fi
done
-#if (cat foobar.hfst foobar2.hfst | python3 test4.py > /dev/null 2> /dev/null); then
-# echo "test4.py passed"
-#else
-# echo "test4.py failed"
-#fi
+if ! [ "$PYTHONPATH" = "" ]; then
+ echo 'import sys' > tmp1
+ echo 'sys.path.insert(0, "'$PYTHONPATH'")' >> tmp1
+ cat test_streams_1.py >> tmp1
+ echo 'import sys' > tmp2
+ echo 'sys.path.insert(0, "'$PYTHONPATH'")' >> tmp2
+ cat test_streams_2.py >> tmp2
+ echo 'import sys' > tmp3
+ echo 'sys.path.insert(0, "'$PYTHONPATH'")' >> tmp3
+ cat test_streams_3.py >> tmp3
+else
+ cat test_streams_1.py > tmp1
+ cat test_streams_2.py > tmp2
+ cat test_streams_3.py > tmp3
+fi
-echo "skipping test4.py"
-echo "skipping test8.py"
+for format in sfst openfst foma;
+do
+ if ( $PYTHON tmp1 $format | $PYTHON tmp2 $format | $PYTHON tmp3 $format ); then
+ echo "test_streams[1|2|3].py with "$format" format passed"
+ elif [ "$?" = "77" ]; then
+ echo "test_streams[1|2|3].py with "$format" format skipped"
+ else
+ echo "test_streams[1|2|3].py with "$format" format failed"
+ exit 1
+ fi
+done
+rm foo
+rm foo_att_prolog
+rm testfile3.att
+rm testfile_.att
+rm tmp
+rm tmp1
+rm tmp2
+rm tmp3
diff --git a/python/test/test_att_reader.py b/python/test/test_att_reader.py
new file mode 100644
index 0000000..18b3b3b
--- /dev/null
+++ b/python/test/test_att_reader.py
@@ -0,0 +1,25 @@
+import hfst
+
+transducers = []
+
+with open('testfile.att', 'r') as f:
+ r = hfst.AttReader(f, "<eps>")
+ for tr in r:
+ transducers.append(tr)
+
+assert(f.closed)
+assert(len(transducers)) == 4
+
+transducers = []
+
+with open('testfile_fail.att', 'r') as f:
+ try:
+ r = hfst.AttReader(f, "<eps>")
+ for tr in r:
+ transducers.append(tr)
+ except hfst.exceptions.NotValidAttFormatException as e:
+ assert("1 baz baz 0.3" in e.what())
+ assert("line: 11" in e.what())
+
+assert(f.closed)
+assert(len(transducers)) == 4
diff --git a/python/test/test_dir_hfst.py b/python/test/test_dir_hfst.py
new file mode 100644
index 0000000..144fcb3
--- /dev/null
+++ b/python/test/test_dir_hfst.py
@@ -0,0 +1,35 @@
+import hfst
+
+diritems = \
+['EPSILON',
+ 'HfstBasicTransducer', 'HfstBasicTransition', 'HfstInputStream', 'HfstOutputStream',
+ 'HfstTokenizer', 'HfstTransducer', 'IDENTITY', 'LexcCompiler', 'PmatchContainer',
+ 'UNKNOWN',
+ 'XfstCompiler', 'XreCompiler', 'compile_lexc_file',
+ 'compile_pmatch_expression', 'compile_pmatch_file', 'compile_xfst_file', 'concatenate', 'disjunct',
+ 'empty_fst', 'epsilon_fst', 'exceptions', 'fsa', 'fst', 'fst_type_to_string', 'get_default_fst_type',
+ 'get_output_to_console', 'intersect', 'is_diacritic',
+ 'read_att_input', 'read_att_string', 'regex', 'sfst_rules', 'xerox_rules', 'set_default_fst_type',
+ 'set_output_to_console', 'start_xfst', 'tokenized_fst']
+
+dirhfst = dir(hfst)
+
+for item in diritems:
+ if not item in dirhfst:
+ print('error: dir(hfst) does not contain', item)
+ assert(False)
+
+assert hfst.EPSILON == '@_EPSILON_SYMBOL_@'
+assert hfst.UNKNOWN == '@_UNKNOWN_SYMBOL_@'
+assert hfst.IDENTITY == '@_IDENTITY_SYMBOL_@'
+
+assert hfst.ImplementationType.SFST_TYPE == 0
+assert hfst.ImplementationType.TROPICAL_OPENFST_TYPE == 1
+assert hfst.ImplementationType.LOG_OPENFST_TYPE == 2
+assert hfst.ImplementationType.FOMA_TYPE == 3
+assert hfst.ImplementationType.XFSM_TYPE == 4
+assert hfst.ImplementationType.HFST_OL_TYPE == 5
+assert hfst.ImplementationType.HFST_OLW_TYPE == 6
+assert hfst.ImplementationType.HFST2_TYPE == 7
+assert hfst.ImplementationType.UNSPECIFIED_TYPE == 8
+assert hfst.ImplementationType.ERROR_TYPE == 9
diff --git a/python/test/test_dir_hfst_exceptions.py b/python/test/test_dir_hfst_exceptions.py
new file mode 100644
index 0000000..7d2e647
--- /dev/null
+++ b/python/test/test_dir_hfst_exceptions.py
@@ -0,0 +1,18 @@
+import hfst.exceptions
+
+diritems = \
+['ContextTransducersAreNotAutomataException', 'EmptySetOfContextsException', 'EmptyStringException', \
+ 'EndOfStreamException', 'FlagDiacriticsAreNotIdentitiesException', 'FunctionNotImplementedException', \
+ 'HfstException', 'HfstFatalException', 'HfstTransducerTypeMismatchException', \
+ 'ImplementationTypeNotAvailableException', 'IncorrectUtf8CodingException', 'MetadataException', \
+ 'MissingOpenFstInputSymbolTableException', 'NotTransducerStreamException', 'NotValidAttFormatException', \
+ 'NotValidLexcFormatException', 'NotValidPrologFormatException', 'SpecifiedTypeRequiredException', \
+ 'StateIndexOutOfBoundsException', 'StateIsNotFinalException', 'StreamCannotBeWrittenException', \
+ 'StreamIsClosedException', 'StreamNotReadableException', 'SymbolNotFoundException', \
+ 'TransducerHasWrongTypeException', 'TransducerHeaderException', 'TransducerIsCyclicException', \
+ 'TransducerTypeMismatchException', 'TransducersAreNotAutomataException']
+
+dirhfstexceptions = dir(hfst.exceptions)
+
+for item in diritems:
+ assert item in dirhfstexceptions
diff --git a/python/test/test_dir_hfst_sfst_rules.py b/python/test/test_dir_hfst_sfst_rules.py
new file mode 100644
index 0000000..d8d6e41
--- /dev/null
+++ b/python/test/test_dir_hfst_sfst_rules.py
@@ -0,0 +1,13 @@
+import hfst.sfst_rules
+
+diritems = \
+['coercion', 'deep_coercion', 'deep_restriction', 'deep_restriction_and_coercion', \
+ 'left_replace_down', 'left_replace_down_karttunen', 'left_replace_left', 'left_replace_right', \
+ 'left_replace_up', 'replace_down', 'replace_down_karttunen', 'replace_left', 'replace_right', \
+ 'replace_up', 'restriction', 'restriction_and_coercion', 'surface_coercion', 'surface_restriction', \
+ 'surface_restriction_and_coercion', 'two_level_if', 'two_level_if_and_only_if', 'two_level_only_if']
+
+dirhfstsfstrules = dir(hfst.sfst_rules)
+
+for item in diritems:
+ assert item in dirhfstsfstrules
diff --git a/python/test/test_examples.py b/python/test/test_examples.py
new file mode 100644
index 0000000..2aaad0a
--- /dev/null
+++ b/python/test/test_examples.py
@@ -0,0 +1,405 @@
+# The examples given in doxygen documentation
+from __future__ import print_function
+import sys
+if len(sys.argv) > 1:
+ sys.path.insert(0, sys.argv[1])
+import hfst
+
+types = []
+if hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.SFST_TYPE):
+ types.append(hfst.ImplementationType.SFST_TYPE)
+if hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.TROPICAL_OPENFST_TYPE):
+ types.append(hfst.ImplementationType.TROPICAL_OPENFST_TYPE)
+if hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.FOMA_TYPE):
+ types.append(hfst.ImplementationType.FOMA_TYPE)
+
+for type in types:
+ if hfst.HfstTransducer.is_implementation_type_available(type):
+
+ hfst.set_default_fst_type(type)
+
+ # StreamIsClosedException
+ try:
+ tr = hfst.regex('foo')
+ outstr = hfst.HfstOutputStream(filename='testfile')
+ outstr.close()
+ outstr.write(tr)
+ except hfst.exceptions.StreamIsClosedException:
+ print("Could not write transducer: stream to file was closed.")
+
+ # TransducerIsCyclicException
+ transducer = hfst.regex('[a:b]*')
+ try:
+ results = transducer.extract_paths(output='text')
+ print("The transducer has %i paths:" % len(results))
+ print(results)
+ except hfst.exceptions.TransducerIsCyclicException:
+ print("The transducer is cyclic and has an infinite number of paths. Some of them:")
+ results = transducer.extract_paths(output='text', max_cycles=5)
+ print(results)
+
+ # NotTransducerStreamException
+ f = open('foofile', 'w')
+ f.write('This is an ordinary text file.\n')
+ f.close()
+ try:
+ instr = hfst.HfstInputStream('foofile')
+ tr = instr.read()
+ print(tr)
+ instr.close()
+ except hfst.exceptions.NotTransducerStreamException:
+ print("Could not print transducer: the file does not contain binary transducers.")
+
+ f = open('testfile1.att', 'w')
+ f.write('0 1 a b\n1 2 c\n2\n')
+ f.close()
+ f = open('testfile1.att', 'r')
+ try:
+ tr = hfst.read_att_transducer(f)
+ except hfst.exceptions.NotValidAttFormatException:
+ print('Could not read file: it is not in valid ATT format.')
+ f.close()
+
+ # StateIsNotFinalException
+ tr = hfst.HfstBasicTransducer()
+ tr.add_state(1)
+ # An exception is thrown as state number 1 is not final
+ try:
+ w = tr.get_final_weight(1)
+ except hfst.exceptions.StateIsNotFinalException:
+ print("State is not final.")
+
+ # ContextTransducersAreNotAutomataException
+ # try:
+ # tr = hfst.regex('a -> b || c:c __ c:d')
+ # except hfst.exceptions.ContextTransducersAreNotAutomataException:
+ # print("Context transducers must be automata.")
+
+ # TransducersAreNotAutomataException
+ tr1 = hfst.regex('foo:bar')
+ tr2 = hfst.regex('bar:baz')
+ try:
+ tr1.cross_product(tr2)
+ except hfst.exceptions.TransducersAreNotAutomataException:
+ print('Transducers must be automata in cross product.')
+
+ # StateIndexOutOfBoundsException
+ tr = hfst.HfstBasicTransducer()
+ tr.add_state(1)
+ try:
+ w = tr.get_final_weight(2)
+ except hfst.exceptions.StateIndexOutOfBoundsException:
+ print('State number 2 does not exist')
+
+ # TransducerTypeMismatchException:
+ if hfst.ImplementationType.FOMA_TYPE in types:
+ hfst.set_default_fst_type(hfst.ImplementationType.TROPICAL_OPENFST_TYPE)
+ tr1 = hfst.regex('foo')
+ tr2 = hfst.regex('bar')
+ tr2.convert(hfst.ImplementationType.FOMA_TYPE)
+ try:
+ tr1.disjunct(tr2)
+ except hfst.exceptions.TransducerTypeMismatchException:
+ print('The implementation types of transducers must be the same.')
+
+ hfst.set_default_fst_type(type)
+
+ # fst
+ # One unweighted identity path:
+ if not hfst.fst('foo').compare(hfst.regex('{foo}')):
+ raise RuntimeError('')
+ # Weighted path: a tuple of string and number, e.g.
+ if not hfst.fst(('foo',1.4)).compare(hfst.regex('{foo}::1.4')):
+ raise RuntimeError('')
+ if not hfst.fst(('bar',-3)).compare(hfst.regex('{bar}::-3')):
+ raise RuntimeError('')
+ if not hfst.fst(('baz',0)).compare(hfst.regex('{baz}')):
+ raise RuntimeError('')
+ # Several paths: a list or a tuple of paths and/or weighted paths, e.g.
+ if not hfst.fst(['foo', 'bar']).compare(hfst.regex('{foo}|{bar}')):
+ raise RuntimeError('')
+ if not hfst.fst(('foo', ('bar',5.0))).compare(hfst.regex('{foo}|{bar}::5.0')):
+ raise RuntimeError('')
+ if not hfst.fst(('foo', ('bar',5.0), 'baz', 'Foo', ('Bar',2.4))).compare(hfst.regex('{foo}|{bar}::5.0|{baz}|{Foo}|{Bar}::2.4')):
+ raise RuntimeError('')
+ if not hfst.fst([('foo',-1), ('bar',0), ('baz',3.5)]).compare(hfst.regex('{foo}::-1|{bar}|{baz}::3.5')):
+ raise RuntimeError('')
+ # A dictionary
+ if not hfst.fst({'foo':'foo', 'bar':('foo',1.4), 'baz':(('foo',-1),'BAZ')}).compare(hfst.regex('{foo}|{bar}:{foo}::1.4|{baz}:{foo}::-1|{baz}:{BAZ}')):
+ raise RuntimeError('')
+
+ # tokenized_fst
+ tok = hfst.HfstTokenizer()
+ tok.add_multichar_symbol('foo')
+ tok.add_multichar_symbol('bar')
+ tr = hfst.tokenized_fst(tok.tokenize('foobar', 'foobaz'))
+ if not tr.compare(hfst.regex('[foo:foo bar:b 0:a 0:z]')):
+ raise RuntimeError('')
+
+ # HfstBasicTransducer
+ # Create an empty transducer
+ # The transducer has initially one start state (number zero)
+ # that is not final
+ fsm = hfst.HfstBasicTransducer()
+ # Add two states to the transducer
+ fsm.add_state(1)
+ fsm.add_state(2)
+ # Create a transition [foo:bar] leading to state 1 with weight 0.1
+ tr = hfst.HfstBasicTransition(1, 'foo', 'bar', 0.1)
+ # and add it to state zero
+ fsm.add_transition(0, tr)
+ # Add a transition [baz:baz] with weight 0 from state 1 to state 2
+ fsm.add_transition(1, hfst.HfstBasicTransition(2, 'baz', 'baz', 0.0))
+ # Set state 2 as final with weight 0.3
+ fsm.set_final_weight(2, 0.3)
+ # Go through all states
+ for state, arcs in enumerate(fsm):
+ for arc in arcs:
+ print('%i ' % (state), end='')
+ print(arc)
+ if fsm.is_final_state(state):
+ print('%i %f' % (state, fsm.get_final_weight(state)) )
+
+ for state in fsm.states():
+ for arc in fsm.transitions(state):
+ print('%i ' % (state), end='')
+ print(arc)
+ if fsm.is_final_state(state):
+ print('%i %f' % (state, fsm.get_final_weight(state)) )
+
+ # HfstBasicTransducer.disjunct
+ lexicon = hfst.HfstBasicTransducer()
+ tok = hfst.HfstTokenizer()
+ lexicon.disjunct(tok.tokenize('dog'), 0.3)
+ lexicon.disjunct(tok.tokenize('cat'), 0.5)
+ lexicon.disjunct(tok.tokenize('elephant'), 1.6)
+ lexicon = hfst.HfstTransducer(lexicon)
+ if not lexicon.compare(hfst.regex('{dog}::0.3|{cat}::0.5|{elephant}::1.6')):
+ raise RuntimeError('')
+
+ # HfstBasicTransducer.transitions
+ for state in fsm.states():
+ for arc in fsm.transitions(state):
+ print('%i ' % (state), end='')
+ print(arc)
+ if fsm.is_final_state(state):
+ print('%i %f' % (state, fsm.get_final_weight(state)) )
+
+ # HfstBasicTransducer.substitute and HfstTransducer.substitute
+ HFST = hfst.regex('a:a')
+ basic = hfst.HfstBasicTransducer(HFST)
+ HFST.substitute('a', 'A', input=True, output=False)
+ basic.substitute('a', 'A', input=True, output=False)
+
+ HFST = hfst.regex('a a:b b')
+ basic = hfst.HfstBasicTransducer(HFST)
+ HFST.substitute(('a','b'),('A','B'))
+ basic.substitute(('a','b'),('A','B'))
+
+ HFST = hfst.regex('a a:b b')
+ basic = hfst.HfstBasicTransducer(HFST)
+ HFST.substitute(('a','b'), (('A','B'),('a','B'),('A','b')))
+ basic.substitute(('a','b'), (('A','B'),('a','B'),('A','b')))
+
+ HFST = hfst.regex('a a:b b')
+ basic = hfst.HfstBasicTransducer(HFST)
+ HFST.substitute(('a','b'), hfst.regex('[a:b]+'))
+ basic.substitute(('a','b'), hfst.HfstBasicTransducer(hfst.regex('[a:b]+')))
+
+ HFST = hfst.regex('a b c d')
+ basic = hfst.HfstBasicTransducer(HFST)
+ HFST.substitute({'a':'A', 'b':'B', 'c':'C'})
+ basic.substitute({'a':'A', 'b':'B', 'c':'C'})
+
+ HFST = hfst.regex('a a:b b b:c c c:d d')
+ basic = hfst.HfstBasicTransducer(HFST)
+ HFST.substitute( {('a','a'):('A','A'), ('b','b'):('B','B'), ('c','c'):('C','C')} )
+ basic.substitute( {('a','a'):('A','A'), ('b','b'):('B','B'), ('c','c'):('C','C')} )
+
+ # HfstBasicTransducer.enumerate
+ for state, arcs in enumerate(fsm):
+ for arc in arcs:
+ print('%i ' % (state), end='')
+ print(arc)
+ if fsm.is_final_state(state):
+ print('%i %f' % (state, fsm.get_final_weight(state)) )
+
+ # HfstTransducer
+ # argument handling
+ transducer1 = hfst.regex('foo:bar baz')
+ transducer2 = hfst.regex('FOO:BAR BAZ')
+ transducer1.reverse()
+ transducer1.disjunct(transducer2)
+ if not transducer2.compare(hfst.regex('FOO:BAR BAZ')):
+ raise RuntimeError('')
+ transducer1.reverse()
+ transducer1.determinize()
+ transducer1.reverse()
+ transducer1.determinize()
+ # implementation types
+ if hfst.ImplementationType.FOMA_TYPE in types:
+ tropical_transducer = hfst.regex('foo')
+ tropical_transducer.convert(hfst.ImplementationType.TROPICAL_OPENFST_TYPE)
+ foma_transducer = hfst.regex('foo')
+ foma_transducer.convert(hfst.ImplementationType.FOMA_TYPE)
+ # TODO: segfaults
+ try:
+ tropical_transducer.compare(foma_transducer)
+ except hfst.exceptions.TransducerTypeMismatchException:
+ print('Implementation types of transducers must be the same.')
+
+ # read_att from file
+ f = open('testfile2.att', 'w')
+ f.write(
+"""0 1 foo bar 0.3
+1 0.5
+--
+0 0.0
+--
+--
+0 0.0
+0 0 a <eps> 0.2
+""")
+ f.close()
+
+ transducers = []
+ ifile = open('testfile2.att', 'r')
+ try:
+ while (True):
+ t = hfst.read_att_transducer(ifile, '<eps>')
+ transducers.append(t)
+ print("read one transducer")
+ except hfst.exceptions.NotValidAttFormatException:
+ print("Error reading transducer: not valid AT&T format.")
+ except hfst.exceptions.EndOfStreamException:
+ pass
+ ifile.close()
+ print("Read %i transducers in total" % len(transducers))
+
+# read_att from string
+#att_str = """0 1 a b
+#1 2 c d
+#2
+#"""
+#print(att_str)
+#tr = hfst.read_att(att_str, '@0@')
+#print(tr)
+#exit(0)
+
+ # write_att
+ tr1 = hfst.regex('[foo:bar baz:0 " "]::0.3')
+ tr2 = hfst.empty_fst()
+ tr3 = hfst.epsilon_fst(0.5)
+ tr4 = hfst.regex('[foo]')
+ tr5 = hfst.empty_fst()
+
+ f = open('testfile3.att', 'w')
+ for tr in [tr1, tr2, tr3, tr4]:
+ tr.write_att(f)
+ f.write('--\n')
+ tr5.write_att(f)
+ f.close()
+
+ # extract_paths
+ tr = hfst.regex('a:b+ (a:c+)')
+ print(tr)
+ print(tr.extract_paths(max_cycles=1, output='text'))
+ print(tr.extract_paths(max_number=4, output='text'))
+ print(tr.extract_paths(max_cycles=1, max_number=4, output='text'))
+
+ # HfstOutputStream
+ res = ['foo:bar','0','0 - 0','"?":?','a* b+']
+ ostr = hfst.HfstOutputStream(filename='testfile1.hfst')
+ for re in res:
+ ostr.write(hfst.regex(re))
+ ostr.flush()
+ ostr.close()
+
+ # HfstInputStream
+ istr = hfst.HfstInputStream('testfile1.hfst')
+ transducers = []
+ while not (istr.is_eof()):
+ transducers.append(istr.read())
+ istr.close()
+ if not len(transducers) == len(res):
+ raise RuntimeError('')
+ i=0
+ for tr in transducers:
+ if not tr.compare(hfst.regex(res[i])):
+ raise RuntimeError('')
+ i+=1
+
+# push_weights
+
+# QuickStart (1/3)
+
+ tr1 = hfst.regex('foo:bar')
+ tr2 = hfst.regex('bar:baz')
+ tr1.compose(tr2)
+ print(tr1)
+
+# QuickStart (2/3)
+
+# Create as HFST basic transducer [a:b] with transition weight 0.3 and final weight 0.5.
+ t = hfst.HfstBasicTransducer()
+ t.add_state(1)
+ t.add_transition(0, 1, 'a', 'b', 0.3)
+ t.set_final_weight(1, 0.5)
+
+# Convert to tropical OpenFst format (the default, if not set) and push weights toward final state.
+ T = hfst.HfstTransducer(t)
+ T.push_weights_to_end()
+
+# Convert back to HFST basic transducer.
+ tc = hfst.HfstBasicTransducer(T)
+ try:
+ # Rounding might affect the precision.
+ if (0.79 < tc.get_final_weight(1)) and (tc.get_final_weight(1) < 0.81):
+ print("TEST PASSED")
+ else:
+ if hfst.get_default_fst_type() == hfst.ImplementationType.TROPICAL_OPENFST_TYPE:
+ raise RuntimeError('')
+ else:
+ pass # ok to fail if weights are not in use
+ # If the state does not exist or is not final
+ except hfst.exceptions.HfstException:
+ print("TEST FAILED: An exception thrown.")
+ raise RuntimeError('')
+
+# QuickStart (3/3)
+
+# Create a simple lexicon transducer [[foo bar foo] | [foo bar baz]].
+ tok = hfst.HfstTokenizer()
+ tok.add_multichar_symbol('foo')
+ tok.add_multichar_symbol('bar')
+ tok.add_multichar_symbol('baz')
+
+ words = hfst.tokenized_fst(tok.tokenize('foobarfoo'))
+ t = hfst.tokenized_fst(tok.tokenize('foobarbaz'))
+ words.disjunct(t)
+
+# Create a rule transducer that optionally replaces 'bar' with 'baz' between 'foo' and 'foo'.
+ rule = hfst.regex('bar (->) baz || foo _ foo')
+
+# Apply the rule transducer to the lexicon.
+ words.compose(rule)
+ words.minimize()
+
+# Extract all string pairs from the result and print them to standard output.
+ results = 0
+ try:
+ # Extract paths and remove tokenization
+ results = words.extract_paths(output='dict')
+ except hfst.exceptions.TransducerIsCyclicException:
+ # This should not happen because transducer is not cyclic.
+ print("TEST FAILED")
+ exit(1)
+
+ for input,outputs in results.items():
+ print('%s:' % input)
+ for output in outputs:
+ print(' %s\t%f' % (output[0], output[1]))
+
+
+# Foma issue
+
diff --git a/python/test/test_exceptions.py b/python/test/test_exceptions.py
new file mode 100644
index 0000000..09bd71f
--- /dev/null
+++ b/python/test/test_exceptions.py
@@ -0,0 +1,49 @@
+import hfst.exceptions
+
+# HfstException and its subclasses
+
+e = hfst.exceptions.HfstException()
+e = hfst.exceptions.HfstException('foo','bar', 10)
+e = hfst.exceptions.HfstTransducerTypeMismatchException('foo','bar', 10)
+e = hfst.exceptions.ImplementationTypeNotAvailableException('foo','bar', 10, 1)
+e = hfst.exceptions.FunctionNotImplementedException('foo','bar', 10)
+e = hfst.exceptions.StreamNotReadableException('foo','bar', 10)
+e = hfst.exceptions.StreamCannotBeWrittenException('foo','bar', 10)
+e = hfst.exceptions.StreamIsClosedException('foo','bar', 10)
+e = hfst.exceptions.EndOfStreamException('foo','bar', 10)
+e = hfst.exceptions.TransducerIsCyclicException('foo','bar', 10)
+e = hfst.exceptions.NotTransducerStreamException('foo','bar', 10)
+e = hfst.exceptions.NotValidAttFormatException('foo','bar', 10)
+e = hfst.exceptions.NotValidPrologFormatException('foo','bar', 10)
+e = hfst.exceptions.NotValidLexcFormatException('foo','bar', 10)
+e = hfst.exceptions.StateIsNotFinalException('foo','bar', 10)
+e = hfst.exceptions.ContextTransducersAreNotAutomataException('foo','bar', 10)
+e = hfst.exceptions.TransducersAreNotAutomataException('foo','bar', 10)
+e = hfst.exceptions.StateIndexOutOfBoundsException('foo','bar', 10)
+e = hfst.exceptions.TransducerHeaderException('foo','bar', 10)
+e = hfst.exceptions.MissingOpenFstInputSymbolTableException('foo','bar', 10)
+e = hfst.exceptions.TransducerTypeMismatchException('foo','bar', 10)
+e = hfst.exceptions.EmptySetOfContextsException('foo','bar', 10)
+e = hfst.exceptions.SpecifiedTypeRequiredException('foo','bar', 10)
+e = hfst.exceptions.HfstFatalException('foo','bar', 10)
+e = hfst.exceptions.TransducerHasWrongTypeException('foo','bar', 10)
+e = hfst.exceptions.IncorrectUtf8CodingException('foo','bar', 10)
+e = hfst.exceptions.EmptyStringException('foo','bar', 10)
+e = hfst.exceptions.SymbolNotFoundException('foo','bar', 10)
+e = hfst.exceptions.MetadataException('foo','bar', 10)
+e = hfst.exceptions.FlagDiacriticsAreNotIdentitiesException('foo','bar', 10)
+
+import hfst
+
+# Test that importing exceptions via a package works
+if hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.FOMA_TYPE) and hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.TROPICAL_OPENFST_TYPE):
+ try:
+ foo = hfst.regex('foo')
+ bar = hfst.regex('bar')
+ foo.convert(hfst.ImplementationType.FOMA_TYPE)
+ bar.convert(hfst.ImplementationType.TROPICAL_OPENFST_TYPE)
+ foo.concatenate(bar)
+ assert False
+ except hfst.exceptions.TransducerTypeMismatchException as e:
+ pass
+
diff --git a/python/test/test_hfst.py b/python/test/test_hfst.py
index 7c2c81c..4804aa0 100644
--- a/python/test/test_hfst.py
+++ b/python/test/test_hfst.py
@@ -1,7 +1,9 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function
import sys
-sys.path.insert(1, '/home/eaxelson/hfst-git/hfst/python/')
-
-import libhfst
+if len(sys.argv) > 1:
+ sys.path.insert(0, sys.argv[1])
+import hfst
import os.path
from inspect import currentframe
@@ -9,20 +11,42 @@ def get_linenumber():
cf = currentframe()
return cf.f_back.f_lineno
-for type in (libhfst.TROPICAL_OPENFST_TYPE, libhfst.FOMA_TYPE):
+from sys import version
+if int(version[0]) > 2:
+ def unicode(s, c):
+ return s
+
+types = []
+if hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.TROPICAL_OPENFST_TYPE):
+ types.append(hfst.ImplementationType.TROPICAL_OPENFST_TYPE)
+if hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.FOMA_TYPE):
+ types.append(hfst.ImplementationType.FOMA_TYPE)
- print('\n--- Testing implementation type %s ---\n' % libhfst.fst_type_to_string(type))
+for type in types:
- libhfst.set_default_fst_type(type)
+ # print('\n--- Testing implementation type %s ---\n' % hfst.fst_type_to_string(type))
+
+ hfst.set_default_fst_type(type)
tr1 = None
tr2 = None
tr3 = None
+ type_ = hfst.ImplementationType.TROPICAL_OPENFST_TYPE
+ ostr = hfst.HfstOutputStream(filename='foobar.hfst', type=type_)
+
+ tr_ = hfst.regex('{foo}:{bar}::0.5')
+ tr_.convert(type_)
+
+ ostr.write(tr_)
+ ostr.write(tr_)
+ ostr.flush()
+ ostr.close()
+
if not os.path.isfile('foobar.hfst'):
raise RuntimeError('Missing file: foobar.hfst')
- istr = libhfst.HfstInputStream('foobar.hfst')
+ istr = hfst.HfstInputStream('foobar.hfst')
numtr = 0
try:
tr1 = istr.read()
@@ -31,7 +55,7 @@ for type in (libhfst.TROPICAL_OPENFST_TYPE, libhfst.FOMA_TYPE):
numtr += 1
tr3 = istr.read()
numtr += 1
- except libhfst.EndOfStreamException:
+ except hfst.exceptions.EndOfStreamException:
pass
except:
raise RuntimeError(get_linenumber())
@@ -40,10 +64,10 @@ for type in (libhfst.TROPICAL_OPENFST_TYPE, libhfst.FOMA_TYPE):
if numtr != 2:
raise RuntimeError(get_linenumber())
- tr1.convert(libhfst.get_default_fst_type())
- tr2.convert(libhfst.get_default_fst_type())
+ tr1.convert(hfst.get_default_fst_type())
+ tr2.convert(hfst.get_default_fst_type())
- ostr = libhfst.HfstOutputStream(filename='foobar2.hfst')
+ ostr = hfst.HfstOutputStream(filename='foobar2.hfst')
ostr.write(tr1)
ostr.write(tr2)
ostr.flush()
@@ -53,7 +77,7 @@ for type in (libhfst.TROPICAL_OPENFST_TYPE, libhfst.FOMA_TYPE):
TR2 = None
TR3 = None
- istr = libhfst.HfstInputStream('foobar2.hfst')
+ istr = hfst.HfstInputStream('foobar2.hfst')
numtr = 0
try:
TR1 = istr.read()
@@ -62,7 +86,7 @@ for type in (libhfst.TROPICAL_OPENFST_TYPE, libhfst.FOMA_TYPE):
numtr += 1
TR3 = istr.read()
numtr += 1
- except libhfst.EndOfStreamException:
+ except hfst.exceptions.EndOfStreamException:
pass
except:
raise RuntimeError(get_linenumber())
@@ -77,134 +101,142 @@ for type in (libhfst.TROPICAL_OPENFST_TYPE, libhfst.FOMA_TYPE):
raise RuntimeError(get_linenumber())
# Copy constructor
- transducer = libhfst.HfstTransducer(TR1)
+ transducer = hfst.HfstTransducer(TR1)
if not (TR1.compare(transducer)):
raise RuntimeError(get_linenumber())
if not (transducer.compare(TR1)):
raise RuntimeError(get_linenumber())
# Read lexc
- tr = libhfst.compile_lexc_file('test.lexc')
+ tr = hfst.compile_lexc_file('test.lexc')
tr.insert_freely(tr1)
tr.minimize()
tr.insert_freely(('A','B'))
tr.minimize()
+ # Read sfst
+ tr = hfst.compile_sfst_file('test.sfstpl')
+ assert(not (tr == None))
+
# Substitute
- tr = libhfst.regex('a a:b b;')
+ tr = hfst.regex('a a:b b;')
tr.substitute('a', 'A', input=True, output=False)
- eq = libhfst.regex('A:a A:b b;')
+ eq = hfst.regex('A:a A:b b;')
if not (tr.compare(eq)):
raise RuntimeError(get_linenumber())
- tr = libhfst.regex('a a:b b;')
+ tr = hfst.regex('a a:b b;')
tr.substitute('a', 'A', input=False, output=True)
- eq = libhfst.regex('a:A a:b b;')
+ eq = hfst.regex('a:A a:b b;')
if not (tr.compare(eq)):
raise RuntimeError(get_linenumber())
- tr = libhfst.regex('a a:b b;')
+ tr = hfst.regex('a a:b b;')
tr.substitute('a','A')
- eq = libhfst.regex('A A:b b;')
+ eq = hfst.regex('A A:b b;')
if not (tr.compare(eq)):
raise RuntimeError(get_linenumber())
- tr = libhfst.regex('a a:b b;')
+ tr = hfst.regex('a a:b b;')
tr.substitute(('a','b'),('A','B'))
- eq = libhfst.regex('a A:B b;')
+ eq = hfst.regex('a A:B b;')
if not (tr.compare(eq)):
raise RuntimeError(get_linenumber())
- tr = libhfst.regex('a a:b b;')
+ tr = hfst.regex('a a:b b;')
tr.substitute(('a','b'),(('A','B'),('B','C'),('C','D')))
- eq = libhfst.regex('a [A:B|B:C|C:D] b;')
+ eq = hfst.regex('a [A:B|B:C|C:D] b;')
if not (tr.compare(eq)):
raise RuntimeError(get_linenumber())
- tr = libhfst.regex('a a:b b;')
+ tr = hfst.regex('a a:b b;')
tr.substitute(('a','b'),(('A','B'),('B','C'),('C','D')))
- eq = libhfst.regex('a [A:B|B:C|C:D] b;')
+ eq = hfst.regex('a [A:B|B:C|C:D] b;')
if not (tr.compare(eq)):
raise RuntimeError(get_linenumber())
- tr = libhfst.regex('a a:b b;')
+ tr = hfst.regex('a a:b b;')
tr.substitute({'a':'A', 'b':'B', 'c':'C'})
- eq = libhfst.regex('A A:B B;')
+ eq = hfst.regex('A A:B B;')
if not (tr.compare(eq)):
raise RuntimeError(get_linenumber())
- tr = libhfst.regex('a a:b b;')
+ tr = hfst.regex('a a:b b;')
tr.substitute({('a','a'):('A','a'), ('a','b'):('a','B'), ('c','c'):('C','c')})
- eq = libhfst.regex('A:a a:B b;')
+ eq = hfst.regex('A:a a:B b;')
if not (tr.compare(eq)):
raise RuntimeError(get_linenumber())
- tr = libhfst.regex('a a:b b;')
- sub = libhfst.regex('[c:d]+;')
+ tr = hfst.regex('a a:b b;')
+ sub = hfst.regex('[c:d]+;')
tr.substitute(('a','b'),sub)
- eq = libhfst.regex('a [c:d]+ b;')
+ eq = hfst.regex('a [c:d]+ b;')
if not (tr.compare(eq)):
raise RuntimeError(get_linenumber())
# push weights
- tr = libhfst.regex('[a::1 a:b::0.3 b::0]::0.7;')
- tr.push_weights(libhfst.TO_INITIAL_STATE)
- tr.push_weights(libhfst.TO_FINAL_STATE)
+ tr = hfst.regex('[a::1 a:b::0.3 b::0]::0.7;')
+ tr.push_weights_to_start()
+ tr.push_weights_to_end()
# set final weights
- tr = libhfst.regex('(a a:b (b));')
+ tr = hfst.regex('(a a:b (b));')
tr.set_final_weights(0.1)
tr.set_final_weights(0.4, True)
# reading and writing in text format
- f = open('testfile.att', 'w')
+ f = open('testfile_.att', 'w')
f.write('0 1 foo bar 0.5\n\
0 1 fo ba 0.2\n\
0 1 f b 0\n\
1 2 baz baz\n\
2 0.1\n\
---\n')
+--\n\n')
f.close()
numtr = 0
- f = libhfst.hfst_open('testfile.att', 'r')
- while not f.is_eof():
- TR = libhfst.read_att(f)
- numtr += 1
+ f = open('testfile_.att', 'r')
try:
- libhfst.read_att(f)
- except libhfst.EndOfStreamException:
+ while True:
+ TR = hfst.read_att_transducer(f)
+ numtr += 1
+ except hfst.exceptions.EndOfStreamException:
pass
f.close()
if numtr != 2:
raise RuntimeError(get_linenumber())
- f = libhfst.hfst_open('foo_att_prolog', 'w')
+ f = open('foo_att_prolog', 'w')
f.write('-- in ATT format --\n')
TR.write_att(f)
f.write('-- in prolog format --\n')
- TR.write_prolog(f, 'FOOBAR_TRANSDUCER')
+ TR.write_prolog(f)
f.close()
- fsm = libhfst.read_att_string(' 0\t 1 a b\n\
+ fsm = hfst.read_att_string(' 0\t 1 a b\n\
1 2 c d 0.5\n\
2 \n\
2 3 \t\te f\n\
3 0.3 ')
- #if not fsm.compare(libhfst.regex('a:b c:d::0.5 (e:f::0.3)')):
+ #if not fsm.compare(hfst.regex('a:b c:d::0.5 (e:f::0.3)')):
# raise RuntimeError('read_att_string failed')
# Lookup and path extraction
- tr = libhfst.regex('foo:bar::0.5 | foo:baz')
+ tr = hfst.regex('foo:bar::0.5 | foo:baz')
- print('tr.lookup')
+ f = open('foo', 'w')
try:
- print(tr.lookup('foo', max_number=5, output='text'))
- except libhfst.FunctionNotImplementedException:
- print('converting...')
- TR = libhfst.HfstTransducer(tr)
- TR.convert(libhfst.HFST_OLW_TYPE)
- print(TR.lookup('foo', max_number=5, output='text'))
+ print(tr.lookup('foo', max_number=5, output='text'), file=f)
+ except hfst.exceptions.FunctionNotImplementedException:
+ TR = hfst.HfstTransducer(tr)
+ TR.convert(hfst.ImplementationType.HFST_OLW_TYPE)
+ print(TR.lookup('foo', max_number=5, output='text'), file=f)
+
+ tr_ = tr.copy()
+ tr.lookup_optimize()
+ tr.lookup('foo', max_number=5, output='text', file=f)
+ tr.remove_optimization()
+ assert(tr.compare(tr_))
# def lookup_fd(self, lookup_path, **kvargs):
# max_weight = None
@@ -212,18 +244,19 @@ for type in (libhfst.TROPICAL_OPENFST_TYPE, libhfst.FOMA_TYPE):
# output='dict' # 'dict' (default), 'text', 'raw'
- fsm = libhfst.HfstBasicTransducer(tr)
- print(fsm.lookup_fd((('foo'))))
+ fsm = hfst.HfstBasicTransducer(tr)
+ print(fsm.lookup((('foo'))), file=f)
- print('tr.extract_paths')
- print(tr.extract_paths(obey_flags='True', filter_flags='False', max_number=3, output='dict'))
+ print(tr.extract_paths(obey_flags='True', filter_flags='False', max_number=3, output='dict'), file=f)
def test_fst(input, result):
- tr1_ = libhfst.fst(input)
- tr2_ = libhfst.regex(result)
+ tr1_ = hfst.fst(input)
+ tr2_ = hfst.regex(result)
if not tr1_.compare(tr2_):
raise RuntimeError('test_fst failed with input: ' + input)
+ f.close()
+
# Create automaton:
# unweighted
test_fst('foobar', '[f o o b a r]')
@@ -237,7 +270,7 @@ for type in (libhfst.TROPICAL_OPENFST_TYPE, libhfst.FOMA_TYPE):
test_fst('*** FOO ***', '{*** FOO ***}')
try:
- foo = libhfst.fst('')
+ foo = hfst.fst('')
raise RuntimeError(get_linenumber())
except RuntimeError as e:
if not e.__str__() == 'Empty word.':
@@ -260,13 +293,13 @@ for type in (libhfst.TROPICAL_OPENFST_TYPE, libhfst.FOMA_TYPE):
test_fst({'*** FOO ***':'+++ BAR +++'}, '{*** FOO ***}:{+++ BAR +++}')
try:
- foo = libhfst.fst({'':'foo'})
+ foo = hfst.fst({'':'foo'})
raise RuntimeError(get_linenumber())
except RuntimeError as e:
if not e.__str__() == 'Empty word.':
raise RuntimeError(get_linenumber())
try:
- foo = libhfst.fst({'foo':''})
+ foo = hfst.fst({'foo':''})
raise RuntimeError(get_linenumber())
except RuntimeError as e:
if not e.__str__() == 'Empty word.':
@@ -279,13 +312,13 @@ for type in (libhfst.TROPICAL_OPENFST_TYPE, libhfst.FOMA_TYPE):
tokenized = tok.tokenize_one_level(pathin)
else:
tokenized = tok.tokenize(pathin, pathout)
- if not libhfst.tokenized_fst(tokenized, weight).compare(libhfst.regex(exp)):
+ if not hfst.tokenized_fst(tokenized, weight).compare(hfst.regex(exp)):
if pathout == None:
raise RuntimeError('test_tokenized failed with input: ' + pathin)
else:
raise RuntimeError('test_tokenized failed with input: ' + pathin + ", " + pathout)
- tok = libhfst.HfstTokenizer()
+ tok = hfst.HfstTokenizer()
test_tokenized(tok, 'foobar', None, '[f o o b a r]')
test_tokenized(tok, 'foobar', 'foobar', '[f o o b a r]')
@@ -313,123 +346,150 @@ for type in (libhfst.TROPICAL_OPENFST_TYPE, libhfst.FOMA_TYPE):
test_tokenized(tok, 'fööbär', 'föbar', '[fööbär:b 0:a 0:r]')
test_tokenized(tok, 'föfööfö', 'föföföföö', '[föö]')
- tok = libhfst.HfstTokenizer()
+ tok = hfst.HfstTokenizer()
tok.add_skip_symbol('?')
tok.add_skip_symbol(' ')
test_tokenized(tok, 'How is this tokenized?', None, '[H o w i s t h i s t o k e n i z e d]')
tok.add_skip_symbol(' is ')
test_tokenized(tok, 'How is this tokenized?', None, '[H o w t h i s t o k e n i z e d]')
- tok = libhfst.HfstTokenizer()
- tok.add_multichar_symbol(libhfst.EPSILON) # TODO: should this be included by default???
+ tok = hfst.HfstTokenizer()
+ tok.add_multichar_symbol(hfst.EPSILON) # TODO: should this be included by default???
test_tokenized(tok, '@_EPSILON_SYMBOL_ at foo', None, '[f o o]')
- if not libhfst.tokenized_fst([(libhfst.EPSILON,'b'),('f','a'),('o','a'),('o','r')]).compare(libhfst.regex('[0:b f:a o:a o:r]')):
+ if not hfst.tokenized_fst([(hfst.EPSILON,'b'),('f','a'),('o','a'),('o','r')]).compare(hfst.regex('[0:b f:a o:a o:r]')):
raise RuntimeError(get_linenumber())
# Is this ok???
- if not libhfst.regex('"' + libhfst.EPSILON + '"').compare(libhfst.regex('[0]')):
+ if not hfst.regex('"' + hfst.EPSILON + '"').compare(hfst.regex('[0]')):
raise RuntimeError(get_linenumber())
- if not libhfst.regex('"' + libhfst.IDENTITY + '"').compare(libhfst.regex('[?]')):
+ if not hfst.regex('"' + hfst.IDENTITY + '"').compare(hfst.regex('[?]')):
raise RuntimeError(get_linenumber())
- if not libhfst.regex('"' + libhfst.UNKNOWN + '":"' + libhfst.UNKNOWN + '"').compare(libhfst.regex('[?:?]')):
+ if not hfst.regex('"' + hfst.UNKNOWN + '":"' + hfst.UNKNOWN + '"').compare(hfst.regex('[?:?]')):
raise RuntimeError(get_linenumber())
# other python functions
- if not libhfst.empty_fst().compare(libhfst.regex('[0-0]')):
+ if not hfst.empty_fst().compare(hfst.regex('[0-0]')):
raise RuntimeError(get_linenumber())
- if not libhfst.epsilon_fst().compare(libhfst.regex('[0]')):
+ if not hfst.epsilon_fst().compare(hfst.regex('[0]')):
raise RuntimeError(get_linenumber())
- if not libhfst.epsilon_fst(-1.5).compare(libhfst.regex('[0]::-1.5')):
+ if not hfst.epsilon_fst(-1.5).compare(hfst.regex('[0]::-1.5')):
raise RuntimeError(get_linenumber())
# Non-ascii characters and unknowns/identities
- tr1 = libhfst.regex('Ä:é å ?;')
- tr2 = libhfst.regex('? Ö;')
+ tr1 = hfst.regex('Ä:é å ?;')
+ tr2 = hfst.regex('? Ö;')
tr1.concatenate(tr2)
- result = libhfst.regex('Ä:é å [Ä|é|å|Ö|?] [Ä|é|å|Ö|?] Ö;')
+ result = hfst.regex('Ä:é å [Ä|é|å|Ö|?] [Ä|é|å|Ö|?] Ö;')
if not tr1.compare(result):
raise RuntimeError(get_linenumber())
- tr1 = libhfst.regex('ñ ?:á;')
- tr2 = libhfst.regex('Ê:?;')
+ tr1 = hfst.regex('ñ ?:á;')
+ tr2 = hfst.regex('Ê:?;')
tr1.concatenate(tr2)
- result = libhfst.regex('ñ [ñ:á|á|Ê:á|?:á] [Ê:ñ|Ê|Ê:á|Ê:?];')
+ result = hfst.regex('ñ [ñ:á|á|Ê:á|?:á] [Ê:ñ|Ê|Ê:á|Ê:?];')
if not tr1.compare(result):
raise RuntimeError(get_linenumber())
# Other functions (TODO: more extensixe checks)
- tr = libhfst.regex('[foo]|[foo bar]|[f o o bar baz]')
+ tr = hfst.regex('[foo]|[foo bar]|[f o o bar baz]')
if not tr.longest_path_size() == 5:
raise RuntimeError(get_linenumber())
result = tr.extract_longest_paths()
if not len(result) == 1:
raise RuntimeError(get_linenumber())
- result = tr.extract_shortest_paths()
+ result = tr.extract_shortest_paths()
if not len(result) == 1:
raise RuntimeError(get_linenumber())
# XfstCompiler
- if libhfst.compile_xfst_file('test_pass.xfst') != 0:
- raise RuntimeError(get_linenumber())
- if libhfst.compile_xfst_file('test_fail.xfst') == 0:
- raise RuntimeError(get_linenumber())
- if libhfst.compile_xfst_file('test_fail.xfst', quit_on_fail=False, verbosity=0) != 0:
- raise RuntimeError(get_linenumber())
+ if int(version[0]) > 2:
+ import io
+ msg = io.StringIO()
+ if hfst.compile_xfst_file('test_pass.xfst', verbosity=0, output=msg, error=msg) != 0:
+ raise RuntimeError(get_linenumber())
+ if hfst.compile_xfst_file('test_fail.xfst', verbosity=0, output=msg, error=msg) == 0:
+ raise RuntimeError(get_linenumber())
+ if hfst.compile_xfst_file('test_fail.xfst', quit_on_fail=False, verbosity=0, output=msg, error=msg) != 0:
+ raise RuntimeError(get_linenumber())
# regex compiler
import io
msg = io.StringIO()
- msg.write('This is the error message:\n')
- tr = libhfst.regex('foo\\', error=msg)
+ msg.write(unicode('This is the error message:\n', 'utf-8'))
+ tr = hfst.regex('foo\\', error=msg)
if (tr == None):
- msg.write('This was the error message.\n')
- print(msg.getvalue())
+ msg.write(unicode('This was the error message.\n', 'utf-8'))
+ # print(msg.getvalue())
import sys
msg = sys.stdout
- tr = libhfst.regex('foo\\', error=msg)
+ tr = hfst.regex('foo\\', error=msg)
# lexc compiler
msg = io.StringIO()
- tr = libhfst.compile_lexc_file('test.lexc', output=msg, verbosity=2)
- print('This is the output from lexc:')
- print(msg.getvalue())
+ tr = hfst.compile_lexc_file('test.lexc', output=msg, verbosity=2)
+ # print('This is the output from lexc:')
+ # print(msg.getvalue())
+
+ # default constructor
+ tr = hfst.HfstTransducer()
+ assert(tr.compare(hfst.empty_fst()))
+
+ defs = {'foo':hfst.regex('Foo'), 'bar':hfst.regex('Bar')}
+ tr = hfst.regex('foo bar', definitions=defs)
+ assert(tr.compare(hfst.regex('Foo Bar')))
+ tr = hfst.regex('foo bar')
+ assert(tr.compare(hfst.regex('foo bar')))
-print('\n--- Testing HfstBasicTransducer ---\n')
+# print('\n--- Testing HfstBasicTransducer ---\n')
# Create basic transducer, write it to file, read it, and test equivalence
-fsm = libhfst.HfstBasicTransducer()
+fsm = hfst.HfstBasicTransducer()
fsm.add_state(0)
fsm.add_state(1)
fsm.set_final_weight(1, 0.3)
-tr = libhfst.HfstBasicTransition(1, 'foo', 'bar', 0.5)
+tr = hfst.HfstBasicTransition(1, 'foo', 'bar', 0.5)
fsm.add_transition(0, tr)
fsm.add_transition(0, 0, 'baz', 'baz')
fsm.add_transition(0, 0, 'baz', 'BAZ', 0.1)
-f = libhfst.hfst_open('foo_basic', 'w')
+f = open('foo_basic', 'w')
fsm.write_att(f)
f.close()
-f = libhfst.hfst_open('foo_basic', 'r')
-fsm2 = libhfst.HfstBasicTransducer.read_att(f, libhfst.EPSILON)
+f = open('foo_basic', 'r')
+fsm2 = hfst.HfstBasicTransducer(hfst.read_att_transducer(f, hfst.EPSILON))
f.close()
-FSM = libhfst.HfstTransducer(fsm, libhfst.FOMA_TYPE)
-FSM2 = libhfst.HfstTransducer(fsm2, libhfst.FOMA_TYPE)
-
-if not (FSM.compare(FSM2)):
- raise RuntimeError(get_linenumber())
-
-for type in (libhfst.TROPICAL_OPENFST_TYPE, libhfst.FOMA_TYPE):
- FSM.convert(type)
- Fsm = libhfst.HfstBasicTransducer(FSM)
- FSM2.convert(type)
- Fsm2 = libhfst.HfstBasicTransducer(FSM2)
+# Modify weights of a basic transducer
+fsm = hfst.HfstBasicTransducer()
+fsm.add_state(0)
+fsm.add_state(1)
+fsm.set_final_weight(1, 0.3)
+fsm.add_transition(0, 0, 'baz', 'baz')
+arcs = fsm.transitions(0)
+arcs[0].set_weight(0.5)
+arcs = fsm.transitions(0)
+assert(arcs[0].get_weight() == 0.5)
+
+# comparison can fail because of rounding
+#for type in types:
+# FSM = hfst.HfstTransducer(fsm, type)
+# FSM2 = hfst.HfstTransducer(fsm2, type)
+#
+# if not (FSM.compare(FSM2)):
+# raise RuntimeError(get_linenumber())
+#
+# this test does not assert anything
+#for type in types:
+# FSM.convert(type)
+# Fsm = hfst.HfstBasicTransducer(FSM)
+# FSM2.convert(type)
+# Fsm2 = hfst.HfstBasicTransducer(FSM2)
# Print basic transducer
-fsm = libhfst.HfstBasicTransducer()
+fsm = hfst.HfstBasicTransducer()
for state in [0,1,2]:
fsm.add_state(state)
fsm.add_transition(0,1,'foo','bar',1)
@@ -438,30 +498,65 @@ fsm.add_transition(1,2,'baz','baz',0)
fsm.set_final_weight(2,0.5)
# Different ways to print the transducer
+f = open('foo', 'w')
for state in fsm.states():
for arc in fsm.transitions(state):
- print('%i ' % (state), end='')
- print(arc)
+ print('%i ' % (state), end='', file=f)
+ print(arc, file=f)
if fsm.is_final_state(state):
- print('%i %f' % (state, fsm.get_final_weight(state)) )
+ print('%i %f' % (state, fsm.get_final_weight(state)), file=f )
for state, arcs in enumerate(fsm):
for arc in arcs:
- print('%i ' % (state), end='')
- print(arc)
+ print('%i ' % (state), end='', file=f)
+ print(arc, file=f)
if fsm.is_final_state(state):
- print('%i %f' % (state, fsm.get_final_weight(state)) )
+ print('%i %f' % (state, fsm.get_final_weight(state)), file=f)
index=0
for state in fsm.states_and_transitions():
for transition in state:
- print('%u\t%u\t%s\t%s\t%.2f' % (index, transition.get_target_state(), transition.get_input_symbol(), transition.get_output_symbol(), transition.get_weight()))
+ print('%u\t%u\t%s\t%s\t%.2f' % (index, transition.get_target_state(), transition.get_input_symbol(), transition.get_output_symbol(), transition.get_weight()), file=f)
if fsm.is_final_state(index):
- print('%s\t%.2f' % (index, fsm.get_final_weight(index)))
+ print('%s\t%.2f' % (index, fsm.get_final_weight(index)), file=f)
index = index + 1
-print(fsm)
+print(fsm, file=f)
+f.close()
-tr = libhfst.HfstBasicTransducer(libhfst.regex('foo'))
+tr = hfst.HfstBasicTransducer(hfst.regex('foo'))
tr.substitute({'foo':'bar'})
tr.substitute({('foo','foo'):('bar','bar')})
+
+tr = hfst.fst({'foo':'bar'})
+fst = hfst.HfstBasicTransducer(tr)
+fsa = hfst.fst_to_fsa(fst, '^')
+fst = hfst.fsa_to_fst(fsa, '^')
+TR = hfst.HfstTransducer(fst)
+assert(TR.compare(tr))
+
+tr = hfst.regex('{foo}:{bar}|{FOO}:{BAR}')
+fsm = hfst.HfstBasicTransducer(tr)
+net = fsm.states_and_transitions()
+for state in net:
+ for arc in state:
+ arc.set_input_symbol(arc.get_input_symbol() + '>')
+ arc.set_output_symbol('<' + arc.get_output_symbol())
+ arc.set_weight(arc.get_weight() - 0.5)
+
+for state, arcs in enumerate(fsm):
+ for arc in arcs:
+ arc.set_input_symbol('<' + arc.get_input_symbol())
+ arc.set_output_symbol(arc.get_output_symbol() + '>')
+ arc.set_weight(arc.get_weight() - 1.5)
+
+for state in fsm:
+ for arc in state:
+ arc.set_input_symbol('' + arc.get_input_symbol() + '')
+ arc.set_output_symbol('' + arc.get_output_symbol() + '')
+ arc.set_weight(arc.get_weight() - 0.5)
+
+tr = hfst.regex('[["<f>" "<o>" "<o>"]:["<b>" "<a>" "<r>"]|["<F>" "<O>" "<O>"]:["<B>" "<A>" "<R>"]]::-7.5')
+assert(not (tr == None))
+TR = hfst.HfstTransducer(fsm)
+assert(TR.compare(tr))
diff --git a/python/test/test_pmatch.py b/python/test/test_pmatch.py
new file mode 100644
index 0000000..869f108
--- /dev/null
+++ b/python/test/test_pmatch.py
@@ -0,0 +1,44 @@
+import hfst
+
+import os.path
+assert os.path.isfile('streets.txt')
+
+# pmatch transducers are always in ol format, so this has actually no effect...
+for type in [hfst.ImplementationType.SFST_TYPE, hfst.ImplementationType.TROPICAL_OPENFST_TYPE, hfst.ImplementationType.FOMA_TYPE]:
+ if hfst.HfstTransducer.is_implementation_type_available(type):
+ print(hfst.fst_type_to_string(type))
+ hfst.set_default_fst_type(type)
+
+ # (1) compile the file directly
+ defs = hfst.compile_pmatch_file('streets.txt')
+ cont = hfst.PmatchContainer(defs)
+ assert cont.match("Je marche seul dans l'avenue des Ternes.") == "Je marche seul dans l'<FrenchStreetName>avenue des Ternes</FrenchStreetName>."
+
+ # (2) compile the contents of file
+ with open('streets.txt', 'r') as myfile:
+ data=myfile.read()
+ myfile.close()
+ defs = hfst.compile_pmatch_expression(data)
+ cont = hfst.PmatchContainer(defs)
+ assert cont.match("Je marche seul dans l'avenue des Ternes.") == "Je marche seul dans l'<FrenchStreetName>avenue des Ternes</FrenchStreetName>."
+
+ # (3) try to compile a nonexistent file
+ nonexistent_file = 'foofoofoofoofoofoofoofoofoofoofoofoo'
+ assert not os.path.isfile(nonexistent_file)
+ try:
+ hfst.compile_pmatch_file(nonexistent_file)
+ assert False
+ except IOError as e:
+ pass
+
+ # (4) try to compile meaningless and invalid expressions
+ # skip these tests, it seems that PmatchCompiler should be reseted after
+ # compilation that fails...
+ continue
+ for expr in ["foobarbaz; ????", "", "efine CapWord UppercaseAlpha Alpha* :"]:
+ print(expr)
+ try:
+ defs = hfst.compile_pmatch_expression(expr)
+ assert False
+ except hfst.exceptions.HfstException as e:
+ pass
diff --git a/python/test/test_prolog.py b/python/test/test_prolog.py
new file mode 100644
index 0000000..a3d6584
--- /dev/null
+++ b/python/test/test_prolog.py
@@ -0,0 +1,65 @@
+import hfst
+import sys
+
+for type in [hfst.ImplementationType.SFST_TYPE, hfst.ImplementationType.TROPICAL_OPENFST_TYPE, hfst.ImplementationType.FOMA_TYPE]:
+ if hfst.HfstTransducer.is_implementation_type_available(type):
+
+ f = open('cats_and_dogs.prolog', 'r')
+ F = open('tmp', 'w')
+
+ tr = hfst.read_prolog_transducer(f)
+ re = hfst.regex('{cat}')
+ assert(tr.compare(re))
+ tr.write_prolog(F, True)
+ F.write('\n')
+
+ tr = hfst.read_prolog_transducer(f)
+ re = hfst.regex('0 - 0')
+ assert(tr.compare(re))
+ tr.write_prolog(F, True)
+ F.write('\n')
+
+ tr = hfst.read_prolog_transducer(f)
+ re = hfst.regex('{dog}:{cat}::0.5')
+ assert(tr.compare(re))
+ tr.write_prolog(F, True)
+ F.write('\n')
+
+ tr = hfst.read_prolog_transducer(f)
+ re = hfst.regex('[c a:h t:a 0:t]::-1.5')
+ assert(tr.compare(re))
+ tr.write_prolog(F, True)
+
+ try:
+ tr = hfst.read_prolog_transducer(f)
+ assert(False)
+ except hfst.exceptions.EndOfStreamException as e:
+ pass
+
+ f.close()
+ F.close()
+
+ f = open('tmp', 'r')
+
+ tr = hfst.read_prolog_transducer(f)
+ re = hfst.regex('{cat}')
+ assert(tr.compare(re))
+
+ tr = hfst.read_prolog_transducer(f)
+ re = hfst.regex('0 - 0')
+ assert(tr.compare(re))
+
+ tr = hfst.read_prolog_transducer(f)
+ re = hfst.regex('{dog}:{cat}::0.5')
+ assert(tr.compare(re))
+
+ tr = hfst.read_prolog_transducer(f)
+ re = hfst.regex('[c a:h t:a 0:t]::-1.5')
+
+ try:
+ tr = hfst.read_prolog_transducer(f)
+ assert(False)
+ except hfst.exceptions.EndOfStreamException as e:
+ pass
+
+ f.close()
diff --git a/python/test/test_prolog_reader.py b/python/test/test_prolog_reader.py
new file mode 100644
index 0000000..617e2d2
--- /dev/null
+++ b/python/test/test_prolog_reader.py
@@ -0,0 +1,25 @@
+import hfst
+
+transducers = []
+
+with open('cats_and_dogs.prolog', 'r') as f:
+ r = hfst.PrologReader(f)
+ for tr in r:
+ transducers.append(tr)
+
+assert(f.closed)
+assert(len(transducers)) == 4
+
+transducers = []
+
+with open('cats_and_dogs_fail.prolog', 'r') as f:
+ try:
+ r = hfst.PrologReader(f)
+ for tr in r:
+ transducers.append(tr)
+ except hfst.exceptions.NotValidPrologFormatException as e:
+ assert 'arc(1, 2, "bar").' in e.what()
+ assert 'line: 24' in e.what()
+
+assert(f.closed)
+assert(len(transducers)) == 4
diff --git a/python/test/test_read_att_transducer.py b/python/test/test_read_att_transducer.py
new file mode 100644
index 0000000..a5c380e
--- /dev/null
+++ b/python/test/test_read_att_transducer.py
@@ -0,0 +1,18 @@
+import hfst
+
+for type in [hfst.ImplementationType.SFST_TYPE, hfst.ImplementationType.TROPICAL_OPENFST_TYPE, hfst.ImplementationType.FOMA_TYPE]:
+ if hfst.HfstTransducer.is_implementation_type_available(type):
+
+ transducers = []
+ ifile = open('testfile.att', 'r')
+ try:
+ while (True):
+ t = hfst.read_att_transducer(ifile, '<eps>')
+ transducers.append(t)
+ except hfst.exceptions.NotValidAttFormatException as e:
+ print("Error reading transducer: not valid AT&T format.")
+ except hfst.exceptions.EndOfStreamException as e:
+ pass
+ ifile.close()
+ assert(len(transducers) == 4)
+
diff --git a/python/test/test_streams_1.py b/python/test/test_streams_1.py
index 6a29789..085961d 100644
--- a/python/test/test_streams_1.py
+++ b/python/test/test_streams_1.py
@@ -1,10 +1,27 @@
-import libhfst
+# -*- coding: utf-8 -*-
+import hfst
-tr1 = libhfst.regex('föö:bär')
-tr2 = libhfst.regex('0')
-tr3 = libhfst.regex('0-0')
+import sys
+if sys.argv[1] == 'sfst':
+ if not hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.SFST_TYPE):
+ sys.exit(77)
+ hfst.set_default_fst_type(hfst.ImplementationType.SFST_TYPE)
+elif sys.argv[1] == 'foma':
+ if not hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.FOMA_TYPE):
+ sys.exit(77)
+ hfst.set_default_fst_type(hfst.ImplementationType.FOMA_TYPE)
+elif sys.argv[1] == 'openfst':
+ if not hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.TROPICAL_OPENFST_TYPE):
+ sys.exit(77)
+ hfst.set_default_fst_type(hfst.ImplementationType.TROPICAL_OPENFST_TYPE)
+else:
+ raise RuntimeError('implementation format not recognized')
-ostr = libhfst.HfstOutputStream()
+tr1 = hfst.regex('föö:bär')
+tr2 = hfst.regex('0')
+tr3 = hfst.regex('0-0')
+
+ostr = hfst.HfstOutputStream()
ostr.write(tr1)
ostr.write(tr2)
ostr.write(tr3)
diff --git a/python/test/test_streams_2.py b/python/test/test_streams_2.py
index 61743af..3d03998 100644
--- a/python/test/test_streams_2.py
+++ b/python/test/test_streams_2.py
@@ -1,7 +1,24 @@
-import libhfst
+# -*- coding: utf-8 -*-
+import hfst
+import sys
+
+if sys.argv[1] == 'sfst':
+ if not hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.SFST_TYPE):
+ sys.exit(77)
+ hfst.set_default_fst_type(hfst.ImplementationType.SFST_TYPE)
+elif sys.argv[1] == 'foma':
+ if not hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.FOMA_TYPE):
+ sys.exit(77)
+ hfst.set_default_fst_type(hfst.ImplementationType.FOMA_TYPE)
+elif sys.argv[1] == 'openfst':
+ if not hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.TROPICAL_OPENFST_TYPE):
+ sys.exit(77)
+ hfst.set_default_fst_type(hfst.ImplementationType.TROPICAL_OPENFST_TYPE)
+else:
+ raise RuntimeError('implementation format not recognized')
transducers = []
-istr = libhfst.HfstInputStream()
+istr = hfst.HfstInputStream()
while not istr.is_eof():
transducers.append(istr.read())
istr.close()
@@ -11,12 +28,12 @@ if not len(transducers) == 3:
i = 0
for re in ['föö:bär','0','0-0']:
- if not transducers[i].compare(libhfst.regex(re)):
+ if not transducers[i].compare(hfst.regex(re)):
raise RuntimeError('Transducers are not equivalent.')
i += 1
if len(transducers) > 0:
- f = libhfst.hfst_stdout()
+ f = sys.stdout
i=0
transducers[i].write_att(f)
i += 1
diff --git a/python/test/test_streams_3.py b/python/test/test_streams_3.py
index ccd6221..e36949d 100644
--- a/python/test/test_streams_3.py
+++ b/python/test/test_streams_3.py
@@ -1,5 +1,35 @@
-import libhfst
+# -*- coding: utf-8 -*-
+import hfst
+import sys
-f = libhfst.hfst_stdin()
-while not f.is_eof():
- print(libhfst.read_att(f))
+if sys.argv[1] == 'sfst':
+ if not hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.SFST_TYPE):
+ sys.exit(77)
+ hfst.set_default_fst_type(hfst.ImplementationType.SFST_TYPE)
+elif sys.argv[1] == 'foma':
+ if not hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.FOMA_TYPE):
+ sys.exit(77)
+ hfst.set_default_fst_type(hfst.ImplementationType.FOMA_TYPE)
+elif sys.argv[1] == 'openfst':
+ if not hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.TROPICAL_OPENFST_TYPE):
+ sys.exit(77)
+ hfst.set_default_fst_type(hfst.ImplementationType.TROPICAL_OPENFST_TYPE)
+else:
+ raise RuntimeError('implementation format not recognized')
+
+transducers = []
+
+try:
+ while (True):
+ transducers.append(hfst.read_att_transducer(sys.stdin))
+except hfst.exceptions.EndOfStreamException:
+ pass
+
+if not len(transducers) == 3:
+ raise RuntimeError('Wrong number of transducers read.')
+
+i = 0
+for re in ['föö:bär','0','0-0']:
+ if not transducers[i].compare(hfst.regex(re)):
+ raise RuntimeError('Transducers are not equivalent.')
+ i += 1
diff --git a/python/test/test_tokenizer.py b/python/test/test_tokenizer.py
new file mode 100644
index 0000000..3cfc299
--- /dev/null
+++ b/python/test/test_tokenizer.py
@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+import hfst
+from sys import version
+
+for type in [hfst.ImplementationType.SFST_TYPE, hfst.ImplementationType.TROPICAL_OPENFST_TYPE, hfst.ImplementationType.FOMA_TYPE]:
+ if hfst.HfstTransducer.is_implementation_type_available(type):
+
+ tok = hfst.HfstTokenizer()
+ tok.add_multichar_symbol('foo')
+ tok.add_skip_symbol('b')
+ assert tok.tokenize('foobar') == (('foo', 'foo'), ('a', 'a'), ('r', 'r'))
+ assert tok.tokenize_one_level('foobar') == ('foo', 'a', 'r')
+ assert tok.tokenize('foobar','barfoo') == (('foo', 'a'), ('a', 'r'), ('r', 'foo'))
+
+ tok = hfst.HfstTokenizer()
+
+ tok = hfst.HfstTokenizer()
+ tok.add_skip_symbol('foo')
+ assert tok.tokenize_one_level('foofofoobar') == ('f', 'o', 'b', 'a', 'r')
+
+ tok = hfst.HfstTokenizer()
+ tok.add_multichar_symbol('foo')
+ tok.add_skip_symbol('fo')
+ assert tok.tokenize_one_level('foofo') == ('foo',)
+
+ tok = hfst.HfstTokenizer()
+ tok.add_multichar_symbol('fo')
+ tok.add_skip_symbol('foo')
+ assert tok.tokenize_one_level('foofo') == ('fo',)
+
+ tok = hfst.HfstTokenizer()
+ tok.add_multichar_symbol('fo')
+ tok.add_multichar_symbol('foo')
+ tok.add_multichar_symbol('of')
+ assert tok.tokenize_one_level('fofoofooof') == ('fo', 'foo', 'foo', 'of')
+
+ tok = hfst.HfstTokenizer()
+ t = tok.tokenize('foobar')
+ assert t == (('f', 'f'), ('o', 'o'), ('o', 'o'), ('b', 'b'), ('a', 'a'), ('r', 'r'))
+
+ tok = hfst.HfstTokenizer()
+ t = tok.tokenize_one_level('foobar')
+ assert t == ('f', 'o', 'o', 'b', 'a', 'r')
+
+ tok = hfst.HfstTokenizer()
+ tok.add_multichar_symbol('foo')
+ tok.add_skip_symbol('b')
+ assert tok.tokenize('foobar','Foobar') == \
+ (('foo', 'F'), ('a', 'o'), ('r', 'o'), ('@_EPSILON_SYMBOL_@', 'a'), ('@_EPSILON_SYMBOL_@', 'r'))
+
+ tok = hfst.HfstTokenizer()
+ assert tok.tokenize_space_separated('f o o b a r') == \
+ (('f', 'f'), ('o', 'o'), ('o', 'o'), ('b', 'b'), ('a', 'a'), ('r', 'r'))
+ assert tok.tokenize_space_separated('foo b a r') == \
+ (('foo', 'foo'), ('b', 'b'), ('a', 'a'), ('r', 'r'))
+ assert tok.tokenize_space_separated('f o o bar') == \
+ (('f', 'f'), ('o', 'o'), ('o', 'o'), ('bar', 'bar'))
+ assert tok.tokenize_space_separated('foo bar') == \
+ (('foo', 'foo'), ('bar', 'bar'))
+ assert tok.tokenize_space_separated('foobar') == \
+ (('foobar', 'foobar'),)
+
+ if int(version[0]) > 2:
+ try:
+ hfst.HfstTokenizer.check_utf8_correctness('föö')
+ except hfst.exceptions.IncorrectUtf8CodingException as e:
+ assert(False)
diff --git a/python/test/test_xerox_rules.py b/python/test/test_xerox_rules.py
new file mode 100644
index 0000000..011f19e
--- /dev/null
+++ b/python/test/test_xerox_rules.py
@@ -0,0 +1,49 @@
+import hfst
+
+types = []
+if hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.SFST_TYPE):
+ types.append(hfst.ImplementationType.SFST_TYPE)
+if hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.TROPICAL_OPENFST_TYPE):
+ types.append(hfst.ImplementationType.TROPICAL_OPENFST_TYPE)
+if hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.FOMA_TYPE):
+ types.append(hfst.ImplementationType.FOMA_TYPE)
+
+from hfst.xerox_rules import *
+from hfst import regex
+
+for type in types:
+ if hfst.HfstTransducer.is_implementation_type_available(type):
+
+ hfst.set_default_fst_type(type)
+
+ rule = Rule() # just testing the default constructor
+
+ mapping = ( (regex('a'),regex('b')), )
+ rule = Rule(mapping)
+ assert(replace(rule, False).compare(regex('a -> b')))
+ assert(replace(rule, True).compare(regex('a (->) b')))
+
+ mapping = ( (regex('a'),regex('b')), (regex('b'),regex('a')) )
+ rule = Rule(mapping)
+ assert(replace(rule, False).compare(regex('a -> b, b -> a')))
+ assert(replace(rule, True).compare(regex('a (->) b, b (->) a')))
+
+ for repl_type in [(ReplaceType.REPL_UP, '||'), (ReplaceType.REPL_DOWN, '\\/'), (ReplaceType.REPL_LEFT, '\\\\'), (ReplaceType.REPL_RIGHT,'//')]:
+
+ mapping1 = ( (regex('a'),regex('b')), )
+ context1 = ( (regex('c'),regex('c')), )
+
+ rule1 = Rule(mapping1, context1, repl_type[0])
+ assert(replace(rule1, False).compare(regex('a -> b ' + repl_type[1] + ' c _ c')))
+ assert(replace(rule1, True).compare(regex('a (->) b ' + repl_type[1] + ' c _ c')))
+
+ mapping2 = ( (regex('a'),regex('b')), (regex('b'),regex('a')) )
+ context2 = ( (regex('c'),regex('c')), (regex('d'),regex('d')) )
+
+ rule2 = Rule(mapping2, context2, repl_type[0])
+ assert(replace(rule2, False).compare(regex('a -> b, b -> a ' + repl_type[1] + ' c _ c, d _ d')))
+ assert(replace(rule2, True).compare(regex('a (->) b, b (->) a ' + repl_type[1] + ' c _ c, d _ d')))
+
+ rules = (rule1, rule2)
+ assert(replace(rules, False).compare(regex('a -> b ' + repl_type[1] + ' c _ c ,, a -> b, b -> a ' + repl_type[1] + ' c _ c, d _ d')))
+ assert(replace(rules, True).compare(regex('a (->) b ' + repl_type[1] + ' c _ c ,, a (->) b, b (->) a ' + repl_type[1] + ' c _ c, d _ d')))
diff --git a/python/test/test_xre.py b/python/test/test_xre.py
new file mode 100644
index 0000000..9892fac
--- /dev/null
+++ b/python/test/test_xre.py
@@ -0,0 +1,22 @@
+import hfst
+
+for type in [hfst.ImplementationType.SFST_TYPE, hfst.ImplementationType.TROPICAL_OPENFST_TYPE, hfst.ImplementationType.FOMA_TYPE]:
+ if hfst.HfstTransducer.is_implementation_type_available(type):
+
+ comp = hfst.XreCompiler(hfst.get_default_fst_type())
+ comp.set_expand_definitions(True)
+ comp.define_xre('FooStar', '[foo]*')
+ tr = hfst.regex('[foo]+')
+ comp.define_transducer('FooPlus', tr)
+ comp.define_xre('Bar', 'bar')
+ comp.undefine('Bar')
+
+ TR = comp.compile('FooStar a FooPlus Bar')
+ TR1 = hfst.regex('[foo* a foo+ Bar]')
+ assert TR1.compare(TR)
+
+ tr = hfst.regex('foo:bar')
+ comp.define_transducer('FooBar', tr)
+ TR = comp.compile('FooBar.l')
+ TR1 = hfst.regex('bar')
+ assert TR1.compare(TR)
diff --git a/python/test/testfile.att b/python/test/testfile.att
index e1f2bdc..75bb957 100644
--- a/python/test/testfile.att
+++ b/python/test/testfile.att
@@ -1,6 +1,8 @@
-0 1 foo bar 0.5
-0 1 fo ba 0.2
-0 1 f b 0
-1 2 baz baz
-2 0.1
+0 1 foo bar 0.3
+1 0.5
--
+0 0.0
+--
+--
+0 0.0
+0 0 a <eps> 0.2
diff --git a/python/test/testfile1.hfst b/python/test/testfile1.hfst
deleted file mode 100644
index e63db94..0000000
Binary files a/python/test/testfile1.hfst and /dev/null differ
diff --git a/python/test/testfile3.att b/python/test/testfile3.att
deleted file mode 100644
index 4c7facd..0000000
--- a/python/test/testfile3.att
+++ /dev/null
@@ -1,11 +0,0 @@
-0 1 foo bar 0.299805
-1 2 baz @0@ 0.000000
-2 3 @_SPACE_@ @_SPACE_@ 0.000000
-3 0.000000
---
---
-0 0.500000
---
-0 1 foo foo 0.000000
-1 0.000000
---
diff --git a/python/test/testfile_fail.att b/python/test/testfile_fail.att
new file mode 100644
index 0000000..48525fa
--- /dev/null
+++ b/python/test/testfile_fail.att
@@ -0,0 +1,15 @@
+0 1 foo bar 0.3
+1 0.5
+--
+0 0.0
+--
+--
+0 0.0
+0 0 a <eps> 0.2
+--
+0 1 foo bar 0.0
+1 baz baz 0.3
+1 0.0
+--
+0 1 foo bar 0.3
+1 0.5
diff --git a/scripts/README b/scripts/README
index fb7a669..c5658b6 100644
--- a/scripts/README
+++ b/scripts/README
@@ -10,7 +10,7 @@ make-hfst-FOO.bat Individual scripts for creating hfst command line
make-htwolcpre[1|2|3].bat Individual scripts for creating htwolcpre*.exe tools that the script hfst-twolc.bat can call
make-foma.bat A script for making foma dll (not used at the moment)
make-implementations.bat A script for compiling hfst implementations object files (not used at the moment)
-make-libhfst.bat A script for making hfst dll (not used at the moment)
+make-libhfst.bat A script for making hfst dll (not used at the moment)
make-openfstwin.bat A script for making openfstwin dll (not used at the moment)
make-parsers.bat A script for compiling hfst parsers object files (not used at the moment)
test-hfst-tools.bat A script that performs simple tests for hfst tools
@@ -19,11 +19,11 @@ generate-python-bindings.bat A script for making python hfst bindings for Wind
windows_tests/ A directory that contains files for testing hfst tools on Windows
inttypes.h A header file needed for Windows compilation
stdint.h A header file needed for Windows compilation
-README_eight_tools_win.txt README file for Windows package that contains eight command line tools
-README_xfst_win.txt README file for Windows package that contains hfst-xfst.exe command line tool
+README_eight_tools_win.txt README file for Windows package that contains eight command line tools
+README_xfst_win.txt README file for Windows package that contains hfst-xfst.exe command line tool
copy-for-windows.sh TARGET SOURCE Copy winopenfst and foma back-ends as well as libhfst/src (SOURCE is the top directory) under TARGET directory
libhfst_win.i Tentative interface file for new HFST Python bindings for Windows
-test_libhfst_win.py Tentative test script for HFST Python bindings for Windows
+test_libhfst_win.py Tentative test script for HFST Python bindings for Windows
Mac-related files
@@ -31,14 +31,14 @@ Mac-related files
generate-static-binaries.sh For packaging hfst for Mac
package-static-binaries.sh For packaging hfst for Mac
-README_eight_tools_mac.txt README file for Mac package that contains eight command line tools
-README_xfst_mac.txt README file for MAc package that contains hfst-xfst command line tool
+README_eight_tools_mac.txt README file for Mac package that contains eight command line tools
+README_xfst_mac.txt README file for MAc package that contains hfst-xfst command line tool
Foma wrapper files
------------------
-Makefile.am Make foma wrapper (not so much needed now because hfst-xfst replaces it)
+Makefile.am Make foma wrapper (not so much needed now because hfst-xfst replaces it)
hfst-foma-wrapper.sh.in Needed for foma wrapper?
hfst-foma.in Needed for foma wrapper?
diff --git a/scripts/copy-for-windows.sh b/scripts/copy-for-windows.sh
index fff261c..6dd04cb 100755
--- a/scripts/copy-for-windows.sh
+++ b/scripts/copy-for-windows.sh
@@ -1,8 +1,22 @@
#!/bin/sh
-# A script for copying winopenfst (top directory given as the second argument)
-# and foma backends and libhfst/src in a directory given as first argument
-# for native windows compilation.
+if [ "$1" = "--help" -o "$1" = "-h" ]; then
+ echo ""
+ echo "Copy winopenfst and foma back-ends, libhfst/src and python bindings to a directory."
+ echo "The directory can be used for creating python bindings on Windows."
+ echo ""
+ echo "Usage: copy-for-windows.sh [DESTDIR]"
+ echo ""
+ echo "DESTDIR: name of the directory where files are copied"
+ echo ""
+ exit 0
+fi
+
+# A script for copying winopenfst and foma backends, libhfst/src and
+# python bindings in a directory given as first argument.
+
+# In the destination directory, run:
+# C:\PythonXY\python.exe setup.py [ARGS]
if [ -d "$1" ]; then
echo "Directory $1 exists"
@@ -19,6 +33,45 @@ fi
# openfstdir=$2
#fi
+# copy python stuff
+for file in docstrings.i libhfst.i \
+hfst_extensions.cc hfst_file_extensions.cc hfst_lexc_extensions.cc hfst_lookup_extensions.cc \
+hfst_pmatch_extensions.cc hfst_prolog_extensions.cc hfst_regex_extensions.cc \
+hfst_rules_extensions.cc hfst_xfst_extensions.cc hfst_sfst_extensions.cc ;
+do
+ cp python/$file $1/
+done
+
+for file in setup.py MANIFEST.in README README.rst setup.cfg config.h COPYING ;
+do
+ cp python/pypi/$file $1/
+done
+
+mkdir $1/hfst
+mkdir $1/hfst/exceptions
+mkdir $1/hfst/sfst_rules
+mkdir $1/hfst/xerox_rules
+
+cp python/hfst/__init__.py $1/hfst/
+cp python/hfst/exceptions/__init__.py $1/hfst/exceptions/
+cp python/hfst/sfst_rules/__init__.py $1/hfst/sfst_rules/
+cp python/hfst/xerox_rules/__init__.py $1/hfst/xerox_rules/
+
+mkdir $1/test
+for file in README test.sh test.bat \
+ test_fail.xfst testfile.att testfile_fail.att test_pass.xfst test.lexc \
+ cats_and_dogs.prolog cats_and_dogs_fail.prolog \
+ test_pmatch.py test_prolog.py test_exceptions.py test_dir_hfst.py \
+ test_dir_hfst_exceptions.py test_dir_hfst_sfst_rules.py test_tokenizer.py \
+ test_read_att_transducer.py test_xre.py test_hfst.py test_examples.py \
+ test_att_reader.py test_prolog_reader.py \
+ test_streams_1.py test_streams_2.py test_streams_3.py streets.txt \
+ test_xerox_rules.py ;
+do
+ cp python/test/$file $1/test/
+done
+
+
mkdir $1/back-ends
# foma back-end
@@ -28,16 +81,16 @@ for file in \
apply.c coaccessible.c constructions.c \
define.c determinize.c dynarray.c \
extract.c flags.c int_stack.c io.c \
-lex.cmatrix.c lex.yy.c mem.c minimize.c \
+lex.cmatrix.c lex.yy.c lexcread.c mem.c minimize.c \
regex.c reverse.c rewrite.c sigma.c \
spelling.c stringhash.c structures.c \
topsort.c trie.c utf8.c foma.h fomalib.h \
-fomalibconf.h regex.h;
-do
+fomalibconf.h regex.h lexc.h;
+do
cp back-ends/foma/$file $1/back-ends/foma/;
done
# there is a file with the same name in openfstwin
-mv $1/back-ends/foma/flags.c $1/back-ends/foma/_flags.c
+#mv $1/back-ends/foma/flags.c $1/back-ends/foma/_flags.c
# openfstwin back-end
mkdir $1/back-ends/openfstwin
@@ -47,9 +100,9 @@ mkdir $1/back-ends/openfstwin/src/include/fst
mkdir $1/back-ends/openfstwin/src/lib
for file in \
-compat flags fst properties symbol-table symbol-table-ops util;
+compat flags fst properties symbol-table symbol-table-ops util;
do
- cp $openfstdir/src/lib/$file.cc $1/back-ends/openfstwin/src/lib/$file.cpp
+ cp $openfstdir/src/lib/$file.cc $1/back-ends/openfstwin/src/lib/$file.cpp
done
# file symbol-table-ops.cc not in version 1.2.6
@@ -66,6 +119,13 @@ mkdir $1/libhfst/src/implementations
mkdir $1/libhfst/src/implementations/compose_intersect
mkdir $1/libhfst/src/implementations/optimized-lookup
mkdir $1/libhfst/src/parsers
+mkdir $1/libhfst/src/parsers/alphabet_src
+mkdir $1/libhfst/src/parsers/commandline_src
+mkdir $1/libhfst/src/parsers/io_src
+mkdir $1/libhfst/src/parsers/rule_src
+mkdir $1/libhfst/src/parsers/string_src
+mkdir $1/libhfst/src/parsers/variable_src
+
# libhfst/src without subdirectories
for file in \
@@ -75,7 +135,8 @@ HfstExtractStrings.h HfstFlagDiacritics.h \
HfstInputStream.h HfstLookupFlagDiacritics.h HfstOutputStream.h \
HfstSymbolDefs.h HfstTokenizer.h HfstTransducer.h HfstXeroxRules.h \
HfstStrings2FstTokenizer.h hfst.h hfst.hpp.in hfst_apply_schemas.h hfstdll.h \
-hfst-string-conversions.h HfstPrintDot.h HfstPrintPCKimmo.h;
+hfst-string-conversions.h HfstPrintDot.h HfstPrintPCKimmo.h \
+string-utils.h;
do
cp libhfst/src/$file $1/libhfst/src/
done
@@ -86,7 +147,8 @@ HfstEpsilonHandler HfstExceptionDefs HfstFlagDiacritics \
HfstInputStream HfstLookupFlagDiacritics HfstOutputStream HfstRules \
HfstSymbolDefs HfstTokenizer HfstTransducer HfstXeroxRules \
hfst-string-conversions \
-HfstStrings2FstTokenizer HfstXeroxRulesTest HfstPrintDot HfstPrintPCKimmo;
+HfstStrings2FstTokenizer HfstXeroxRulesTest HfstPrintDot HfstPrintPCKimmo \
+string-utils;
do
cp libhfst/src/$file.cc $1/libhfst/src/$file.cpp
done
@@ -94,7 +156,7 @@ done
# libhfst/src/implementations without subdirectories
for file in \
ConvertTransducerFormat.h FomaTransducer.h \
-HfstOlTransducer.h HfstTransition.h HfstTransitionGraph.h \
+HfstOlTransducer.h HfstBasicTransition.h HfstBasicTransducer.h \
HfstTropicalTransducerTransitionData.h LogWeightTransducer.h \
TropicalWeightTransducer.h;
do
@@ -104,7 +166,7 @@ done
for file in \
ConvertFomaTransducer ConvertLogWeightTransducer ConvertOlTransducer \
ConvertTransducerFormat ConvertTropicalWeightTransducer FomaTransducer \
-HfstOlTransducer HfstTransitionGraph HfstTropicalTransducerTransitionData \
+HfstOlTransducer HfstBasicTransducer HfstBasicTransition HfstTropicalTransducerTransitionData \
LogWeightTransducer TropicalWeightTransducer;
do
cp libhfst/src/implementations/$file.cc $1/libhfst/src/implementations/$file.cpp
@@ -140,7 +202,8 @@ done
# libhfst/src/parsers
for file in \
LexcCompiler PmatchCompiler XreCompiler XfstCompiler xfst_help_message \
-lexc-utils pmatch_utils xre_utils xfst-utils;
+TwolcCompiler \
+lexc-utils pmatch_utils xre_utils xfst-utils SfstCompiler SfstAlphabet SfstBasic SfstUtf8;
do
cp libhfst/src/parsers/$file.cc \
$1/libhfst/src/parsers/$file.cpp
@@ -148,28 +211,90 @@ do
$1/libhfst/src/parsers/$file.h
done
for file in \
-lexc-lexer pmatch_lex xre_lex xfst-lexer;
+lexc-lexer pmatch_lex xre_lex xfst-lexer sfst-scanner htwolcpre1-lexer htwolcpre2-lexer htwolcpre3-lexer;
do
cp libhfst/src/parsers/$file.cc \
$1/libhfst/src/parsers/$file.cpp
done
for file in \
-lexc-parser pmatch_parse xre_parse xfst-parser;
+lexc-parser pmatch_parse xre_parse xfst-parser sfst-compiler htwolcpre1-parser htwolcpre2-parser htwolcpre3-parser;
do
cp libhfst/src/parsers/$file.cc \
$1/libhfst/src/parsers/$file.cpp
cp libhfst/src/parsers/$file.hh \
$1/libhfst/src/parsers/$file.hh
done
-sed -i 's/#include <unistd.h>/#include <io.h>/' $1/libhfst/src/parsers/xfst-lexer.cpp
-sed -i 's/hxfstwrap( )/hxfstwrap(void)/' $1/libhfst/src/parsers/xfst-lexer.cpp
+
+for file in xfst-lexer htwolcpre1-lexer htwolcpre2-lexer htwolcpre3-lexer;
+do
+ sed -i 's/#include <unistd.h>/#include <io.h>/' $1/libhfst/src/parsers/$file.cpp
+ sed -i 's/hxfstwrap( )/hxfstwrap(void)/' $1/libhfst/src/parsers/$file.cpp
+done
+
+for file in HfstTwolcDefs.h common_globals.h grammar_defs.h;
+do
+ cp libhfst/src/parsers/$file $1/libhfst/src/parsers/$file
+done
+
+PARSERDIR=libhfst/src/parsers
+
+cp $PARSERDIR/alphabet_src/Alphabet.cc $1/$PARSERDIR/alphabet_src/Alphabet.cpp
+cp $PARSERDIR/alphabet_src/Alphabet.h $1/$PARSERDIR/alphabet_src/Alphabet.h
+cp $PARSERDIR/commandline_src/CommandLine.cc $1/$PARSERDIR/commandline_src/CommandLine.cpp
+cp $PARSERDIR/commandline_src/CommandLine.h $1/$PARSERDIR/commandline_src/CommandLine.h
+cp $PARSERDIR/io_src/InputReader.cc $1/$PARSERDIR/io_src/InputReader.cpp
+cp $PARSERDIR/io_src/InputReader.h $1/$PARSERDIR/io_src/InputReader.h
+cp $PARSERDIR/io_src/input_defs.h $1/$PARSERDIR/io_src/input_defs.h
+
+for file in ConflictResolvingLeftArrowRule ConflictResolvingRightArrowRule \
+ LeftArrowRule LeftArrowRuleContainer LeftRestrictionArrowRule OtherSymbolTransducer \
+ RightArrowRule RightArrowRuleContainer Rule RuleContainer TwolCGrammar;
+do
+ cp $PARSERDIR/rule_src/$file.h $1/$PARSERDIR/rule_src/$file.h
+ cp $PARSERDIR/rule_src/$file.cc $1/$PARSERDIR/rule_src/$file.cpp
+done
+
+cp $PARSERDIR/string_src/string_manipulation.cc $1/$PARSERDIR/string_src/string_manipulation.cpp
+cp $PARSERDIR/string_src/string_manipulation.h $1/$PARSERDIR/string_src/string_manipulation.h
+
+for file in ConstContainerIterator.h MatchedConstContainerIterator.h MixedConstContainerIterator.h \
+ RuleSymbolVector.h RuleVariables.h RuleVariablesConstIterator.h VariableBlock.h VariableBlockContainer.h \
+ VariableContainer.h VariableContainerBase.h VariableDefs.h VariableValueIterator.h VariableValues.h;
+do
+ cp $PARSERDIR/variable_src/$file $1/$PARSERDIR/variable_src/
+done
+
+for file in RuleSymbolVector RuleVariables RuleVariablesConstIterator VariableValues;
+do
+ cp $PARSERDIR/variable_src/$file.cc $1/$PARSERDIR/variable_src/$file.cpp
+done
cp scripts/windows/make-python-bindings.bat $1/libhfst/src/
-cp python/test/test_hfst.py $1/libhfst/src/
-cp python/test/examples.py $1/libhfst/src/
-cp python/test/foobar.hfst $1/libhfst/src/
-cp python/libhfst.i $1/libhfst/src/
+cp scripts/windows/make-python-package.bat $1/make-python-package.bat
+cp scripts/readmes/README_python_win.txt $1/README_python_win.txt
+unix2dos $1/README_python_win.txt
+
+for file in libhfst.i docstrings.i hfst_extensions.cc \
+hfst_file_extensions.cc hfst_lexc_extensions.cc hfst_lookup_extensions.cc \
+hfst_pmatch_extensions.cc hfst_prolog_extensions.cc hfst_regex_extensions.cc \
+hfst_rules_extensions.cc hfst_xfst_extensions.cc
+do
+ cp python/$file $1/libhfst/src/
+done
+
+mkdir $1/libhfst/src/hfst
+mkdir $1/libhfst/src/hfst/exceptions
+mkdir $1/libhfst/src/hfst/sfst_rules
+mkdir $1/libhfst/src/hfst/xerox_rules
+
+cp python/hfst/__init__.py $1/libhfst/src/hfst/
+cp python/hfst/exceptions/__init__.py $1/libhfst/src/hfst/exceptions/
+cp python/hfst/sfst_rules/__init__.py $1/libhfst/src/hfst/sfst_rules/
+cp python/hfst/xerox_rules/__init__.py $1/libhfst/src/hfst/xerox_rules/
+
+# todo: python/test/
+
cp scripts/windows/make-hfst-xfst.bat $1/libhfst/src/
cp scripts/windows/make-hfst-proc.bat $1/libhfst/src/
cp scripts/windows/make-hfst-lexc.bat $1/libhfst/src/
@@ -215,12 +340,6 @@ mkdir $1/tools/src/hfst-proc
mkdir $1/tools/src/inc
mkdir $1/tools/src/hfst-twolc
mkdir $1/tools/src/hfst-twolc/src
-mkdir $1/tools/src/hfst-twolc/src/alphabet_src
-mkdir $1/tools/src/hfst-twolc/src/commandline_src
-mkdir $1/tools/src/hfst-twolc/src/io_src
-mkdir $1/tools/src/hfst-twolc/src/rule_src
-mkdir $1/tools/src/hfst-twolc/src/string_src
-mkdir $1/tools/src/hfst-twolc/src/variable_src
# tools/src/parsers
for file in hfst-xfst init_help;
@@ -232,6 +351,9 @@ do
cp tools/src/parsers/$file $1/tools/src/parsers/
done
+# twolc
+cp tools/src/hfst-twolc/src/htwolc-main.cc $1/tools/src/hfst-twolc/src/hfst-twolc.cpp
+
# compare, strings2fst and txt2fst are needed for testing hfst-xfst
for file in \
hfst-program-options hfst-commandline hfst-tool-metadata \
@@ -270,61 +392,3 @@ globals-common.h globals-unary.h;
do
cp tools/src/inc/$file $1/tools/src/inc/
done
-
-
-# Copy twolc
-
-TWOLC_DIR=tools/src/hfst-twolc/src
-
-for file in \
-HfstTwolcDefs.h common_globals.h grammar_defs.h hfst-twolc.bat \
-htwolcpre1.hh htwolcpre2.hh htwolcpre3.hh;
-do
- cp $TWOLC_DIR/$file $1/$TWOLC_DIR/
-done
-
-for file in \
-hfst-twolc-system htwolcpre1 htwolcpre2 htwolcpre3 scanner1 scanner2 scanner3;
-do
- cp $TWOLC_DIR/$file.cc $1/$TWOLC_DIR/$file.cpp
-done
-
-for file in scanner1.cpp scanner2.cpp scanner3.cpp;
-do
- sed -i 's/#include <unistd.h>/#include <io.h>/' $1/$TWOLC_DIR/$file
- sed -i 's/yywrap( )/yywrap(void)/' $1/$TWOLC_DIR/$file
-done
-
-cp $TWOLC_DIR/alphabet_src/Alphabet.cc $1/$TWOLC_DIR/alphabet_src/Alphabet.cpp
-cp $TWOLC_DIR/alphabet_src/Alphabet.h $1/$TWOLC_DIR/alphabet_src/Alphabet.h
-cp $TWOLC_DIR/commandline_src/CommandLine.cc $1/$TWOLC_DIR/commandline_src/CommandLine.cpp
-cp $TWOLC_DIR/commandline_src/CommandLine.h $1/$TWOLC_DIR/commandline_src/CommandLine.h
-cp $TWOLC_DIR/io_src/InputReader.cc $1/$TWOLC_DIR/io_src/InputReader.cpp
-cp $TWOLC_DIR/io_src/InputReader.h $1/$TWOLC_DIR/io_src/InputReader.h
-cp $TWOLC_DIR/io_src/input_defs.h $1/$TWOLC_DIR/io_src/input_defs.h
-
-for file in ConflictResolvingLeftArrowRule ConflictResolvingRightArrowRule \
-LeftArrowRule LeftArrowRuleContainer LeftRestrictionArrowRule OtherSymbolTransducer \
-RightArrowRule RightArrowRuleContainer Rule RuleContainer TwolCGrammar;
-do
- cp $TWOLC_DIR/rule_src/$file.h $1/$TWOLC_DIR/rule_src/$file.h
- cp $TWOLC_DIR/rule_src/$file.cc $1/$TWOLC_DIR/rule_src/$file.cpp
-done
-
-cp $TWOLC_DIR/string_src/string_manipulation.cc $1/$TWOLC_DIR/string_src/string_manipulation.cpp
-cp $TWOLC_DIR/string_src/string_manipulation.h $1/$TWOLC_DIR/string_src/string_manipulation.h
-
-for file in ConstContainerIterator.h MatchedConstContainerIterator.h MixedConstContainerIterator.h \
-RuleSymbolVector.h RuleVariables.h RuleVariablesConstIterator.h VariableBlock.h VariableBlockContainer.h \
-VariableContainer.h VariableContainerBase.h VariableDefs.h VariableValueIterator.h VariableValues.h;
-do
- cp $TWOLC_DIR/variable_src/$file $1/$TWOLC_DIR/variable_src/
-done
-
-for file in RuleSymbolVector RuleVariables RuleVariablesConstIterator VariableValues;
-do
- cp $TWOLC_DIR/variable_src/$file.cc $1/$TWOLC_DIR/variable_src/$file.cpp
-done
-
-# For convenience
-cp $TWOLC_DIR/hfst-twolc.bat $1/libhfst/src/
diff --git a/scripts/fetch-stats.sh b/scripts/fetch-stats.sh
new file mode 100755
index 0000000..c3b7c03
--- /dev/null
+++ b/scripts/fetch-stats.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+
+if [ -f "results" -o -d "results" ]; then
+ echo "file/directory 'results' exists"
+ exit 1
+fi
+touch results
+
+if [ -f "foo" -o -d "foo" ]; then
+ echo "file/directory 'foo' exists"
+ exit 1
+fi
+
+for year in 2016;
+do
+ for month in `seq --equal-width 39 52`;
+ do
+ url="http://apertium.projectjj.com/apt/stats/weekly/"$year"-"$month".html"
+ if curl --output foo --silent --fail $url; then
+ cat foo | perl -pe 's/<tr>/\n/g;' | grep '>hfst<' | perl -pe 's/[^0-9]*([0-9]+)[^0-9]*([0-9]+)[^0-9]*([0-9]+)[^0-9]*([0-9]+)[^0-9]*/\1 \2 \3 \4/;' >> results
+ echo "" >> results
+ fi
+ done
+done
+
+col=1
+for field in "Total" "Linux" "Win" "Mac";
+do
+ cut -d' ' -f$col results | perl -pe 's/\n/ + /g;' | perl -pe 's/(.*) \+ /\1/;' > foo
+ echo "" >> foo
+ echo $field": "
+ cat foo | bc
+ col=$((col+1))
+done
diff --git a/scripts/generate-cc-files.sh b/scripts/generate-cc-files.sh
index bcf0be0..a91fec2 100755
--- a/scripts/generate-cc-files.sh
+++ b/scripts/generate-cc-files.sh
@@ -22,12 +22,12 @@ test -e lexc-parser.h && mv lexc-parser.h lexc-parser.hh
test -e xfst-parser.h && mv xfst-parser.h xfst-parser.hh
cd ../../..
cd tools/src
-rm -f hfst-compiler.h
-rm -f hfst-compiler.cc
-rm -f hfst-scanner.cc
-make hfst-scanner.cc
-make hfst-compiler.cc
-test -e hfst-compiler.h && mv hfst-compiler.h hfst-compiler.hh
+rm -f sfst-compiler.h
+rm -f sfst-compiler.cc
+rm -f sfst-scanner.cc
+make sfst-scanner.cc
+make sfst-compiler.cc
+test -e sfst-compiler.h && mv sfst-compiler.h sfst-compiler.hh
cd hfst-twolc/src/
rm -f htwolcpre1.h htwolcpre2.h htwolcpre3.h
rm -f htwolcpre1.cc htwolcpre2.cc htwolcpre3.cc
diff --git a/scripts/generate-static-binaries.sh b/scripts/generate-static-binaries.sh
index 7a1d97d..16cf27e 100755
--- a/scripts/generate-static-binaries.sh
+++ b/scripts/generate-static-binaries.sh
@@ -13,7 +13,7 @@ DYLIB_EXTENSION= # so/dylib
if (uname -a | grep 'Linux' > /dev/null); then
ENVIRONMENT='Linux';
SHOW_LINKS='ldd';
-# CONFIGURE_ARGS are ok as such
+# CONFIGURE_ARGS are ok as such
DYLIB_EXTENSION='so';
echo "generating for Linux environment"
elif (uname -a | grep 'Darwin' > /dev/null); then
@@ -43,7 +43,7 @@ compile_statically ()
}
-autoreconf -i &&
+autoreconf -i &&
./configure $CONFIGURE_ARGS && scripts/generate-cc-files.sh
# these are needed for compiling individual tools
diff --git a/scripts/hfst-foma-wrapper.sh.in b/scripts/hfst-foma-wrapper.sh.in
index becf635..9165c25 100644
--- a/scripts/hfst-foma-wrapper.sh.in
+++ b/scripts/hfst-foma-wrapper.sh.in
@@ -1,4 +1,7 @@
#!/bin/bash
+
+## Legacy wrapper for foma
+
#foma_exec=@FOMACLI@
foma_exec=@prefix@/bin/hfst_foma
if [ ! -x $foma_exec ] ; then
@@ -58,7 +61,7 @@ if [ -z $@ ] ; then
if [ "x$verbose" == "x-v" ] ; then
echo "running $foma_exec $fomaopts"
fi
- if ! $foma_exec $fomaopts; then
+ if ! $foma_exec $fomaopts; then
echo "$foma_exec failed"
exit 1
fi
diff --git a/scripts/hfst-foma.in b/scripts/hfst-foma.in
index 8637215..dc974f8 100755
--- a/scripts/hfst-foma.in
+++ b/scripts/hfst-foma.in
@@ -1,4 +1,7 @@
#!/bin/sh
+
+## Legacy wrapper for foma
+
print_usage() {
echo 'usage: $0 [-utf8|-latin1] [-e "command"] [-f scriptfile] [-h] [-l startscript] [-s binaryfile] [-q] [-v] [-F hfst_format] [-V]'
}
diff --git a/scripts/make-libhfst-dist.sh b/scripts/make-libhfst-dist.sh
new file mode 100755
index 0000000..2ad4638
--- /dev/null
+++ b/scripts/make-libhfst-dist.sh
@@ -0,0 +1,43 @@
+#!/bin/sh
+
+# A script for creating a minimal LGPL package for HFST which contains
+# only openfst back-ends, HFST library and python bindings.
+# First argument is the the normal HFST
+# distribution tarball and second one the directory where alternative files
+# for LGPL distribution can be found (folder lgpl-release in hfst repository).
+
+if [ "$1" = "--help" -o "$1" = "-h" ]; then
+ echo ""
+ echo "A script for creating a minimal LGPL package for HFST which contains"
+ echo "only openfst back-ends, HFSTlibrary and python bindings."
+ echo ""
+ echo "Usage: make-libhfst-dist.sh DIST LGPL_DIR"
+ echo ""
+ echo "DIST: the normal HFST distribution tarball"
+ echo "LGPL_DIR: the folder where alternative files for LGPL distribution can be found"
+ echo ""
+ exit 0
+fi
+
+if [ ! -e "$1" ]; then
+ echo "File "$1" does not exist"
+ exit 1
+fi
+
+if [ ! -d "$2" ]; then
+ echo "Directory "$2" does not exist"
+ exit 1
+fi
+
+OUTPUT=`tar -xzvf $1`
+DIR=`echo $OUTPUT | perl -pe 's/ /\n/g;' | head -1 | perl -pe 's/\///;'`
+cd $DIR
+cp -R $2/* .
+autoreconf -i && ./configure && make dist
+tar -xzf $1
+NEWDIRNAME="lib"$DIR
+mv $DIR $NEWDIRNAME
+TARBALL="lib"$DIR".tar.gz"
+tar -czvf $TARBALL "lib"$DIR
+cp $TARBALL ../
+echo "Created lgpl package "$TARBALL
diff --git a/scripts/make-python-package.sh b/scripts/make-python-package.sh
new file mode 100644
index 0000000..f029fea
--- /dev/null
+++ b/scripts/make-python-package.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+mkdir $1
+mkdir $1/hfst
+mkdir $1/hfst/exceptions
+mkdir $1/hfst/sfst_rules
+mkdir $1/hfst/xerox_rules
+
+cp README_python_mac.txt $1/README
+cp _libhfst.so $1/
+cp libhfst.py $1/
+cp hfst/__init__.py $1/hfst/__init__.py
+cp hfst/exceptions/__init__.py $1/hfst/exceptions/__init__.py
+cp hfst/sfst_rules/__init__.py $1/hfst/sfst_rules/__init__.py
+cp hfst/xerox_rules/__init__.py $1/hfst/xerox_rules/__init__.py
+
+zip -r $1".zip" $1
diff --git a/scripts/readmes/README_eight_tools_mac.txt b/scripts/readmes/README_eight_tools_mac.txt
index 78ec5c9..65e0689 100644
--- a/scripts/readmes/README_eight_tools_mac.txt
+++ b/scripts/readmes/README_eight_tools_mac.txt
@@ -11,7 +11,7 @@ This package contains the following command line tools for [32|64]-bit Mac OS X:
- hfst-pmatch2fst
-For more info about the tools, see their KitWiki pages:
+For more info about the tools, see their KitWiki pages:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstXfst>
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstLexc>
@@ -25,6 +25,6 @@ For more info about the tools, see their KitWiki pages:
or run hfst-TOOLNAME --help.
-For more info about the HFST project, see:
+For more info about the HFST project, see:
<http://hfst.sourceforge.net/>
diff --git a/scripts/readmes/README_eight_tools_win.txt b/scripts/readmes/README_eight_tools_win.txt
index b3612af..e91c799 100644
--- a/scripts/readmes/README_eight_tools_win.txt
+++ b/scripts/readmes/README_eight_tools_win.txt
@@ -11,7 +11,7 @@ This package contains the following command line tools for 64-bit Windows:
- hfst-pmatch2fst
-For more info about the tools, see their KitWiki pages:
+For more info about the tools, see their KitWiki pages:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstXfst>
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstLexc>
@@ -25,7 +25,7 @@ For more info about the tools, see their KitWiki pages:
or run hfst-TOOLNAME --help in command prompt.
-For more info about the HFST project, see:
+For more info about the HFST project, see:
<http://hfst.sourceforge.net/>
@@ -34,7 +34,7 @@ Usage
The tools hfst-xfst, hfst-lookup, hfst-optimized-lookup and hfst-pmatch work
by default in an interactive mode. They can be started either by
-double-clicking the icon (opens up a new console window) or executing
+double-clicking the icon (opens up a new console window) or executing
'hfst-[xfst|lookup|optimized-lookup|pmatch]' in command prompt (runs the
program in the same window). They read input from the console, i.e. line by
line from the user and print their output to the console. Both control+D and
diff --git a/scripts/readmes/README_python_mac.txt b/scripts/readmes/README_python_mac.txt
new file mode 100644
index 0000000..1cf215d
--- /dev/null
+++ b/scripts/readmes/README_python_mac.txt
@@ -0,0 +1,27 @@
+
+This directory contains 64-bit python bindings for HFST for Mac OS X. The C++
+part is binary code and the python part is source code. The bindings work with
+all versions of python3. The structure of the directory is the following:
+
+ .
+ ├── hfst
+ │ ├── exceptions
+ │ │ └── __init__.py
+ │ ├── __init__.py
+ │ ├── sfst_rules
+ │ │ └── __init__.py
+ │ └── xerox_rules
+ │ └── __init__.py
+ ├── libhfst.py
+ ├── _libhfst.so
+ └── README
+
+
+The files 'libhfst.py' and '_libhfst.so' as well as folder 'hfst' should be
+installed under folder 'site-packages' of the version of python you are using,
+e.g. /usr/local/lib/python3.4/site-packages/. Alternatively, you can use python
+in the directory where you have these files or add the path to sys.path.
+
+For more information about the python API, see:
+
+ https://hfst.github.io/python/index.html
diff --git a/scripts/readmes/README_python_win.txt b/scripts/readmes/README_python_win.txt
new file mode 100644
index 0000000..c5b240b
--- /dev/null
+++ b/scripts/readmes/README_python_win.txt
@@ -0,0 +1,27 @@
+
+This directory contains 64-bit python bindings for HFST for Windows. The C++
+part is binary code and the python part is source code. The bindings work with
+python3.3 and python3.4 The structure of the directory is the following:
+
+ .
+ ├── hfst
+ │ ├── exceptions
+ │ │ └── __init__.py
+ │ ├── __init__.py
+ │ ├── sfst_rules
+ │ │ └── __init__.py
+ │ └── xerox_rules
+ │ └── __init__.py
+ ├── libhfst.py
+ ├── _libhfst.pyd
+ └── README.txt
+
+
+The files 'libhfst.py' and '_libhfst.pyd' as well as folder 'hfst' should be
+installed under folder 'site-packages' of the version of python you are using,
+e.g. 'C:\Python33\Lib\site-packages'. Alternatively, you can use python in the
+directory where you have these files or add the path to sys.path.
+
+For more information about the python API, see:
+
+ https://hfst.github.io/python/index.html
diff --git a/scripts/readmes/README_xfst_mac.txt b/scripts/readmes/README_xfst_mac.txt
index 9374d39..ade04a0 100644
--- a/scripts/readmes/README_xfst_mac.txt
+++ b/scripts/readmes/README_xfst_mac.txt
@@ -1,9 +1,9 @@
This package contains the command line tool hfst-xfst for [32|64]-bit Mac OS X.
-For more info about the tool, see:
+For more info about the tool, see:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstXfst>
-For more info about the HFST project, see:
+For more info about the HFST project, see:
<http://hfst.sourceforge.net/>
diff --git a/scripts/readmes/README_xfst_win.txt b/scripts/readmes/README_xfst_win.txt
index 5e1fb46..6456b55 100644
--- a/scripts/readmes/README_xfst_win.txt
+++ b/scripts/readmes/README_xfst_win.txt
@@ -1,10 +1,10 @@
This package contains the command line tool hfst-xfst.exe for 64-bit Windows.
-For more info about the tool, see:
+For more info about the tool, see:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstXfst>
-For more info about the HFST project, see:
+For more info about the HFST project, see:
<http://hfst.sourceforge.net/>
@@ -12,7 +12,7 @@ Usage
-----
The tool works by default in an interactive mode. It can be started either by
-double-clicking the icon (opens up a new console window) or executing
+double-clicking the icon (opens up a new console window) or executing
'hfst-xfst' in command prompt (runs the program in the same window).
A prompt 'hfst[N]:' is shown when the program runs, N being the number of
diff --git a/scripts/windows/make-hfst-lexc.bat b/scripts/windows/make-hfst-lexc.bat
index d77baa0..5705a15 100644
--- a/scripts/windows/make-hfst-lexc.bat
+++ b/scripts/windows/make-hfst-lexc.bat
@@ -11,7 +11,7 @@ if not exist "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" echo
exit /B
if exist "..\..\back-ends\foma\flags.c" echo Renaming foma's flags.c into _flags.c as there is a file with the same name in openfst && ^
-move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
+move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
@echo ON
@@ -44,6 +44,7 @@ HfstStrings2FstTokenizer.cpp ^
HfstPrintDot.cpp ^
HfstPrintPCKimmo.cpp ^
hfst-string-conversions.cpp ^
+string-utils.cpp ^
implementations\HfstTransitionGraph.cpp ^
implementations\ConvertTransducerFormat.cpp ^
implementations\HfstTropicalTransducerTransitionData.cpp ^
diff --git a/scripts/windows/make-hfst-proc.bat b/scripts/windows/make-hfst-proc.bat
index 1b3afff..f0f0a88 100644
--- a/scripts/windows/make-hfst-proc.bat
+++ b/scripts/windows/make-hfst-proc.bat
@@ -11,7 +11,7 @@ if not exist "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" echo
exit /B
if exist "..\..\back-ends\foma\flags.c" echo Renaming foma's flags.c into _flags.c as there is a file with the same name in openfst && ^
-move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
+move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
if exist "..\..\tools\src\hfst-proc\transducer.cpp" echo Renaming hfst-proc's transducer.cpp into _transducer.cpp as there is a file with the same name in optimized-lookup && ^
move ..\..\tools\src\hfst-proc\transducer.cpp ..\..\tools\src\hfst-proc\_transducer.cpp
@@ -54,6 +54,7 @@ HfstStrings2FstTokenizer.cpp ^
HfstPrintDot.cpp ^
HfstPrintPCKimmo.cpp ^
hfst-string-conversions.cpp ^
+string-utils.cpp ^
implementations\HfstTransitionGraph.cpp ^
implementations\ConvertTransducerFormat.cpp ^
implementations\HfstTropicalTransducerTransitionData.cpp ^
diff --git a/scripts/windows/make-hfst-tool.bat b/scripts/windows/make-hfst-tool.bat
index ba62127..da1f466 100644
--- a/scripts/windows/make-hfst-tool.bat
+++ b/scripts/windows/make-hfst-tool.bat
@@ -11,7 +11,7 @@ if not exist "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" echo
exit /B
if exist "..\..\back-ends\foma\flags.c" echo Renaming foma's flags.c into _flags.c as there is a file with the same name in openfst && ^
-move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
+move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
@echo ON
@@ -44,9 +44,11 @@ HfstStrings2FstTokenizer.cpp ^
HfstPrintDot.cpp ^
HfstPrintPCKimmo.cpp ^
hfst-string-conversions.cpp ^
-implementations\HfstTransitionGraph.cpp ^
-implementations\ConvertTransducerFormat.cpp ^
+string-utils.cpp ^
+implementations\HfstBasicTransducer.cpp ^
+implementations\HfstBasicTransition.cpp ^
implementations\HfstTropicalTransducerTransitionData.cpp ^
+implementations\ConvertTransducerFormat.cpp ^
implementations\ConvertTropicalWeightTransducer.cpp ^
implementations\ConvertLogWeightTransducer.cpp ^
implementations\ConvertFomaTransducer.cpp ^
@@ -82,6 +84,37 @@ parsers\pmatch_utils.cpp ^
parsers\xre_utils.cpp ^
parsers\xfst-utils.cpp ^
parsers\xfst_help_message.cpp ^
+parsers\sfst-scanner.cpp ^
+parsers\xfst-compiler.cpp ^
+parsers\XfstCompiler.cpp ^
+parsers\SfstAlphabet.cpp ^
+parsers\SfstBasic.cpp ^
+parsers\SfstUtf8.cpp ^
+parsers\htwolcpre1-parser.cpp ^
+parsers\htwolcpre2-parser.cpp ^
+parsers\htwolcpre3-parser.cpp ^
+parsers\htwolcpre1-lexer.cpp ^
+parsers\htwolcpre2-lexer.cpp ^
+parsers\htwolcpre3-lexer.cpp ^
+parsers\alphabet_src\Alphabet.cpp ^
+parsers\commandline_src\CommandLine.cpp ^
+parsers\io_src\InputReader.cpp ^
+parsers\rule_src\ConflictResolvingLeftArrowRule.cpp ^
+parsers\rule_src\ConflictResolvingRightArrowRule.cpp ^
+parsers\rule_src\LeftArrowRule.cpp ^
+parsers\rule_src\LeftArrowRuleContainer.cpp ^
+parsers\rule_src\LeftRestrictionArrowRule.cpp ^
+parsers\rule_src\OtherSymbolTransducer.cpp ^
+parsers\rule_src\RightArrowRule.cpp ^
+parsers\rule_src\RightArrowRuleContainer.cpp ^
+parsers\rule_src\Rule.cpp ^
+parsers\rule_src\RuleContainer.cpp ^
+parsers\rule_src\TwolCGrammar.cpp ^
+parsers\string_src\string_manipulation.cpp ^
+parsers\variable_src\RuleSymbolVector.cpp ^
+parsers\variable_src\RuleVariables.cpp ^
+parsers\variable_src\RuleVariablesConstIterator.cpp ^
+parsers\variable_src\VariableValues.cpp ^
..\..\back-ends\openfstwin\src\lib\compat.cpp ^
..\..\back-ends\openfstwin\src\lib\flags.cpp ^
..\..\back-ends\openfstwin\src\lib\fst.cpp ^
@@ -90,5 +123,3 @@ parsers\xfst_help_message.cpp ^
..\..\back-ends\openfstwin\src\lib\symbol-table-ops.cpp ^
..\..\back-ends\openfstwin\src\lib\util.cpp ^
..\..\back-ends\foma\*.c user32.lib
-
-
diff --git a/scripts/windows/make-hfst-xfst.bat b/scripts/windows/make-hfst-xfst.bat
index 4603681..0075cdd 100644
--- a/scripts/windows/make-hfst-xfst.bat
+++ b/scripts/windows/make-hfst-xfst.bat
@@ -11,7 +11,7 @@ if not exist "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" echo
exit /B
if exist "..\..\back-ends\foma\flags.c" echo Renaming foma's flags.c into _flags.c as there is a file with the same name in openfst && ^
-move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
+move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
@echo ON
@@ -44,6 +44,7 @@ HfstStrings2FstTokenizer.cpp ^
HfstPrintDot.cpp ^
HfstPrintPCKimmo.cpp ^
hfst-string-conversions.cpp ^
+string-utils.cpp ^
implementations\HfstTransitionGraph.cpp ^
implementations\ConvertTransducerFormat.cpp ^
implementations\HfstTropicalTransducerTransitionData.cpp ^
diff --git a/scripts/windows/make-htwolcpre1.bat b/scripts/windows/make-htwolcpre1.bat
index 6677960..e23055e 100644
--- a/scripts/windows/make-htwolcpre1.bat
+++ b/scripts/windows/make-htwolcpre1.bat
@@ -11,7 +11,7 @@ if not exist "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" echo
exit /B
if exist "..\..\back-ends\foma\flags.c" echo Renaming foma's flags.c into _flags.c as there is a file with the same name in openfst && ^
-move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
+move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
@echo ON
@@ -52,6 +52,7 @@ HfstStrings2FstTokenizer.cpp ^
HfstPrintDot.cpp ^
HfstPrintPCKimmo.cpp ^
hfst-string-conversions.cpp ^
+string-utils.cpp ^
implementations\HfstTransitionGraph.cpp ^
implementations\ConvertTransducerFormat.cpp ^
implementations\HfstTropicalTransducerTransitionData.cpp ^
diff --git a/scripts/windows/make-htwolcpre2.bat b/scripts/windows/make-htwolcpre2.bat
index 0cc0a55..c5fb840 100644
--- a/scripts/windows/make-htwolcpre2.bat
+++ b/scripts/windows/make-htwolcpre2.bat
@@ -11,7 +11,7 @@ if not exist "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" echo
exit /B
if exist "..\..\back-ends\foma\flags.c" echo Renaming foma's flags.c into _flags.c as there is a file with the same name in openfst && ^
-move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
+move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
@echo ON
@@ -46,6 +46,7 @@ HfstStrings2FstTokenizer.cpp ^
HfstPrintDot.cpp ^
HfstPrintPCKimmo.cpp ^
hfst-string-conversions.cpp ^
+string-utils.cpp ^
implementations\HfstTransitionGraph.cpp ^
implementations\ConvertTransducerFormat.cpp ^
implementations\HfstTropicalTransducerTransitionData.cpp ^
diff --git a/scripts/windows/make-htwolcpre3.bat b/scripts/windows/make-htwolcpre3.bat
index c43f4e7..2a9fae8 100644
--- a/scripts/windows/make-htwolcpre3.bat
+++ b/scripts/windows/make-htwolcpre3.bat
@@ -11,7 +11,7 @@ if not exist "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" echo
exit /B
if exist "..\..\back-ends\foma\flags.c" echo Renaming foma's flags.c into _flags.c as there is a file with the same name in openfst && ^
-move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
+move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
@echo ON
@@ -60,6 +60,7 @@ HfstStrings2FstTokenizer.cpp ^
HfstPrintDot.cpp ^
HfstPrintPCKimmo.cpp ^
hfst-string-conversions.cpp ^
+string-utils.cpp ^
implementations\HfstTransitionGraph.cpp ^
implementations\ConvertTransducerFormat.cpp ^
implementations\HfstTropicalTransducerTransitionData.cpp ^
diff --git a/scripts/windows/make-python-bindings.bat b/scripts/windows/make-python-bindings.bat
index 6154fa9..145cd63 100644
--- a/scripts/windows/make-python-bindings.bat
+++ b/scripts/windows/make-python-bindings.bat
@@ -8,7 +8,7 @@ rem copy ..\..\back-ends\foma\libfoma.* .
rem copy ..\..\back-ends\openfstwin\src\lib\openfst.* .
if not exist libhfst.i echo Error: missing file "libhfst.i" && ^
-exit /B
+exit /B
if not exist C:\python33\libs\python33.lib echo Error: missing file "C:\python33\libs\python33.lib" && ^
exit /B
@@ -20,7 +20,7 @@ if not exist "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" echo
exit /B
if exist "..\..\back-ends\foma\flags.c" echo Renaming foma's flags.c into _flags.c as there is a file with the same name in openfst ^
-move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
+move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
@echo ON
@@ -51,7 +51,9 @@ HfstStrings2FstTokenizer.cpp ^
HfstPrintDot.cpp ^
HfstPrintPCKimmo.cpp ^
hfst-string-conversions.cpp ^
-implementations\HfstTransitionGraph.cpp ^
+string-utils.cpp ^
+implementations\HfstBasicTransducer.cpp ^
+implementations\HfstBasicTransition.cpp ^
implementations\ConvertTransducerFormat.cpp ^
implementations\HfstTropicalTransducerTransitionData.cpp ^
implementations\ConvertTropicalWeightTransducer.cpp ^
diff --git a/scripts/windows/make-python-package.bat b/scripts/windows/make-python-package.bat
new file mode 100644
index 0000000..80146c7
--- /dev/null
+++ b/scripts/windows/make-python-package.bat
@@ -0,0 +1,18 @@
+echo off
+
+mkdir %1
+mkdir %1\hfst
+mkdir %1\hfst\exceptions
+mkdir %1\hfst\sfst_rules
+mkdir %1\hfst\xerox_rules
+
+copy README_python_win.txt %1\README.txt
+copy _libhfst.pyd %1\
+copy libhfst.py %1\
+copy hfst\__init__.py %1\hfst\__init__.py
+copy hfst\exceptions\__init__.py %1\hfst\exceptions\__init__.py
+copy hfst\sfst_rules\__init__.py %1\hfst\sfst_rules\__init__.py
+copy hfst\xerox_rules\__init__.py %1\hfst\xerox_rules\__init__.py
+
+"C:\Program Files\7-Zip\7z.exe" a -t7z %1".7z" %1
+"C:\Program Files\7-Zip\7z.exe" a -tzip %1".zip" %1
diff --git a/test/Makefile.am b/test/Makefile.am
index c76de03..c463070 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -17,4 +17,9 @@
## You should have received a copy of the GNU General Public License
## along with this program. If not, see <http://www.gnu.org/licenses/>.
-SUBDIRS=libhfst tools
+
+if ENABLE_NO_TOOLS
+ SUBDIRS=libhfst
+else
+ SUBDIRS=libhfst tools
+endif
diff --git a/test/libhfst/Makefile.am b/test/libhfst/Makefile.am
index 4d4b5a2..df407f4 100644
--- a/test/libhfst/Makefile.am
+++ b/test/libhfst/Makefile.am
@@ -20,7 +20,7 @@ test_transducer_functions test_hfst_basic_transducer test_flag_diacritics \
test_examples
# sources for programs
-test_rules_SOURCES=test_rules.cc
+test_rules_SOURCES=test_rules.cc
test_constructors_SOURCES=test_constructors.cc
test_streams_SOURCES=test_streams.cc
test_tokenizer_SOURCES=test_tokenizer.cc
diff --git a/test/libhfst/auxiliary_functions.cc b/test/libhfst/auxiliary_functions.cc
index d430f7b..2a6ea38 100644
--- a/test/libhfst/auxiliary_functions.cc
+++ b/test/libhfst/auxiliary_functions.cc
@@ -1,11 +1,11 @@
const bool VERBOSE=true;
-void verbose_print(const char *msg,
+void verbose_print(const char *msg,
hfst::ImplementationType type=hfst::ERROR_TYPE) {
if (VERBOSE) {
fprintf(stderr, "Testing:\t%s", msg);
fprintf(stderr, " for type ");
- switch (type)
+ switch (type)
{
case hfst::SFST_TYPE:
fprintf(stderr, "SFST_TYPE");
@@ -37,7 +37,7 @@ unsigned char get_bin(size_t fact_128, size_t fact_64, size_t fact_32,
size_t fact_16 , size_t fact_8 , size_t fact_4 ,
size_t fact_2 , size_t fact_1)
{
- return
+ return
fact_128*128 + fact_64*64 + fact_32*32 + fact_16*16 + fact_8*8 +
fact_4*4 + fact_2*2+ fact_1*1;
}
diff --git a/test/libhfst/test_constructors.cc b/test/libhfst/test_constructors.cc
index 76a69a7..d1b363f 100644
--- a/test/libhfst/test_constructors.cc
+++ b/test/libhfst/test_constructors.cc
@@ -11,13 +11,13 @@ using hfst::implementations::HfstBasicTransducer;
using hfst::implementations::HfstBasicTransition;
-int main(int argc, char **argv)
+int main(int argc, char **argv)
{
const unsigned int TYPES_SIZE=3;
- const ImplementationType types [] = {SFST_TYPE,
- TROPICAL_OPENFST_TYPE,
- /*LOG_OPENFST_TYPE,*/
+ const ImplementationType types [] = {SFST_TYPE,
+ TROPICAL_OPENFST_TYPE,
+ /*LOG_OPENFST_TYPE,*/
FOMA_TYPE};
for (unsigned int i=0; i<TYPES_SIZE; i++)
@@ -65,7 +65,7 @@ int main(int argc, char **argv)
/* From AT&T format. */
verbose_print("Construction from AT&T format", types[i]);
- FILE * file = fopen((std::string(getenv("srcdir")) +
+ FILE * file = fopen((std::string(getenv("srcdir")) +
std::string("/foobar.att")).c_str(), "rb");
assert(file != NULL);
unsigned int linecount = 0;
@@ -76,7 +76,7 @@ int main(int argc, char **argv)
assert(foobar.compare(foobar_att));
- /* From HfstInputStream.
+ /* From HfstInputStream.
Tests also functions get_type, set_name and get_name. */
verbose_print("Construction from HfstInputStream", types[i]);
HfstOutputStream out("testfile.hfst", foobar.get_type());
diff --git a/test/libhfst/test_examples.cc b/test/libhfst/test_examples.cc
index 7b6145a..193e02d 100644
--- a/test/libhfst/test_examples.cc
+++ b/test/libhfst/test_examples.cc
@@ -10,7 +10,7 @@ using namespace hfst;
using hfst::implementations::HfstBasicTransition;
using hfst::implementations::HfstBasicTransducer;
-bool function(const StringPair &sp, StringPairSet &sps)
+bool function(const StringPair &sp, StringPairSet &sps)
{
if (sp.second.compare(sp.first) != 0)
return false;
@@ -31,13 +31,13 @@ bool function(const StringPair &sp, StringPairSet &sps)
}
-int main(int argc, char **argv)
+int main(int argc, char **argv)
{
const unsigned int TYPES_SIZE=3;
- const ImplementationType types [] = {SFST_TYPE,
- TROPICAL_OPENFST_TYPE,
- /*LOG_OPENFST_TYPE,*/
+ const ImplementationType types [] = {SFST_TYPE,
+ TROPICAL_OPENFST_TYPE,
+ /*LOG_OPENFST_TYPE,*/
FOMA_TYPE};
HfstBasicTransducer tr1;
@@ -52,7 +52,7 @@ int main(int argc, char **argv)
tr2.add_state(2);
tr2.set_final_weight(2, 0);
tr2.add_transition
- (0, HfstBasicTransition(1, "@_IDENTITY_SYMBOL_@",
+ (0, HfstBasicTransition(1, "@_IDENTITY_SYMBOL_@",
"@_IDENTITY_SYMBOL_@", 0) );
tr2.add_transition
(1, HfstBasicTransition(2, "bar", "bar", 0) );
@@ -60,12 +60,12 @@ int main(int argc, char **argv)
// The disjunction should be
HfstBasicTransducer disj;
- disj.add_state(1);
- disj.add_state(2);
+ disj.add_state(1);
+ disj.add_state(2);
disj.set_final_weight(2, 0);
disj.add_transition
- (0, HfstBasicTransition(1, "@_IDENTITY_SYMBOL_@",
+ (0, HfstBasicTransition(1, "@_IDENTITY_SYMBOL_@",
"@_IDENTITY_SYMBOL_@", 0) );
disj.add_transition
(0, HfstBasicTransition(1, "foo", "foo", 0) );
@@ -99,7 +99,7 @@ int main(int argc, char **argv)
Tr1.disjunct(Tr2).minimize();
// Tr1 is expanded to [ @_UNKNOWN_SYMBOL_@:foo | bar:foo ]
- // Tr2 is expanded to
+ // Tr2 is expanded to
// [ [ @_IDENTITY_SYMBOL_@:@_IDENTITY_SYMBOL_@ | foo:foo ] [ bar:bar ] ]
assert(Tr1.compare(Disj));
@@ -127,7 +127,7 @@ int main(int argc, char **argv)
{
ImplementationType type=FOMA_TYPE;
- /* Create a simple lexicon transducer
+ /* Create a simple lexicon transducer
[[foo bar foo] | [foo bar baz]]. */
HfstTokenizer tok;
@@ -140,7 +140,7 @@ int main(int argc, char **argv)
words.disjunct(t);
- /* Create a rule transducer that optionally replaces
+ /* Create a rule transducer that optionally replaces
"bar" with "baz" between "foo" and "foo". */
HfstTransducerPair context
@@ -159,7 +159,7 @@ int main(int argc, char **argv)
(context, mapping, optional, alphabet);
- /* Apply the rule transducer to the lexicon. */
+ /* Apply the rule transducer to the lexicon. */
words.compose(rule).minimize();
@@ -170,7 +170,7 @@ int main(int argc, char **argv)
try {
words.extract_paths(results);
- }
+ }
catch (TransducerIsCyclicException e)
{
/* This should not happen because transducer is not cyclic. */
@@ -178,7 +178,7 @@ int main(int argc, char **argv)
exit(1);
}
- /* Go through all paths. */
+ /* Go through all paths. */
for (HfstTwoLevelPaths::const_iterator it = results.begin();
it != results.end(); it++)
{
@@ -193,8 +193,8 @@ int main(int argc, char **argv)
istring.append(IT->first);
ostring.append(IT->second);
}
- /*fprintf(stdout, "%s : %s\n",
- istring.c_str(),
+ /*fprintf(stdout, "%s : %s\n",
+ istring.c_str(),
ostring.c_str());*/
}
}
diff --git a/test/libhfst/test_flag_diacritics.cc b/test/libhfst/test_flag_diacritics.cc
index 4215028..8dae47b 100644
--- a/test/libhfst/test_flag_diacritics.cc
+++ b/test/libhfst/test_flag_diacritics.cc
@@ -10,7 +10,7 @@ using hfst::implementations::HfstState;
using hfst::implementations::HfstBasicTransducer;
using hfst::implementations::HfstBasicTransition;
-int main(int argc, char **argv)
+int main(int argc, char **argv)
{
HfstBasicTransducer t;
@@ -36,16 +36,16 @@ int main(int argc, char **argv)
const unsigned int TYPES_SIZE=5;
- const ImplementationType types [] = {SFST_TYPE,
- TROPICAL_OPENFST_TYPE,
- /*LOG_OPENFST_TYPE,*/
+ const ImplementationType types [] = {SFST_TYPE,
+ TROPICAL_OPENFST_TYPE,
+ /*LOG_OPENFST_TYPE,*/
FOMA_TYPE,
HFST_OL_TYPE,
HFST_OLW_TYPE};
/* For all transducer implementation types, perform the following tests: */
- for (unsigned int i=0; i<TYPES_SIZE-2; i++)
+ for (unsigned int i=0; i<TYPES_SIZE-2; i++)
// FIXME: infinite loop in HFST_OL_TYPE
{
if (not HfstTransducer::is_implementation_type_available(types[i]))
@@ -99,9 +99,9 @@ int main(int argc, char **argv)
result_strings.insert(StringPair(istring, ostring));
}
- assert(result_strings.find(StringPair("ac","ac"))
+ assert(result_strings.find(StringPair("ac","ac"))
!= result_strings.end());
- assert(result_strings.find(StringPair("bd","bd"))
+ assert(result_strings.find(StringPair("bd","bd"))
!= result_strings.end());
diff --git a/test/libhfst/test_hfst_basic_transducer.cc b/test/libhfst/test_hfst_basic_transducer.cc
index 50bd3fa..ddbfd67 100644
--- a/test/libhfst/test_hfst_basic_transducer.cc
+++ b/test/libhfst/test_hfst_basic_transducer.cc
@@ -1,7 +1,7 @@
/*
Test file for HfstBasicTransducer.
Some of the functions are already tested in other test files,
- because some HfstTransducer functions are implemented only
+ because some HfstTransducer functions are implemented only
for HfstBasicTransducer.
*/
@@ -16,7 +16,7 @@ using implementations::HfstBasicTransition;
using implementations::HfstBasicTransducer;
-int main(int argc, char **argv)
+int main(int argc, char **argv)
{
verbose_print("HfstBasicTransducer construction");
@@ -61,7 +61,7 @@ int main(int argc, char **argv)
FILE * ifile = fopen("test.att", "rb");
try {
unsigned int linecount = 0;
- HfstBasicTransducer foo
+ HfstBasicTransducer foo
= HfstBasicTransducer::read_in_att_format(ifile, "@0@", linecount);
(void)linecount;
fclose(ifile);
@@ -118,7 +118,7 @@ int main(int argc, char **argv)
{
// In the xerox formalism used here, "?" means the unknown symbol
- // and "?:?" the identity pair
+ // and "?:?" the identity pair
HfstBasicTransducer tr1;
tr1.add_state(1);
@@ -133,7 +133,7 @@ int main(int argc, char **argv)
tr2.add_state(2);
tr2.set_final_weight(2, 0);
tr2.add_transition
- (0, HfstBasicTransition(1, "@_IDENTITY_SYMBOL_@",
+ (0, HfstBasicTransition(1, "@_IDENTITY_SYMBOL_@",
"@_IDENTITY_SYMBOL_@", 0) );
tr2.add_transition
(1, HfstBasicTransition(2, "bar", "bar", 0) );
@@ -154,13 +154,13 @@ int main(int argc, char **argv)
verbose_print("HfstBasicTransducer: iterating through");
- {
+ {
unsigned int source_state=0;
for (HfstBasicTransducer::const_iterator it = t.begin();
it != t.end(); it++ )
{
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
+ for (hfst::implementations::HfstBasicTransitions::const_iterator tr_it
= it->begin(); tr_it != it->end(); tr_it++)
{
std::cerr << source_state << "\t"
@@ -170,7 +170,7 @@ int main(int argc, char **argv)
<< tr_it->get_weight() << std::endl;
}
- if (t.is_final_state(source_state))
+ if (t.is_final_state(source_state))
{
std::cerr << source_state << "\t"
<< t.get_final_weight(source_state) << std::endl;
diff --git a/test/libhfst/test_lexc.cc b/test/libhfst/test_lexc.cc
index b1d8c2d..31deeb4 100644
--- a/test/libhfst/test_lexc.cc
+++ b/test/libhfst/test_lexc.cc
@@ -1,14 +1,14 @@
#include "HfstTransducer.h"
-int main(int argc, char **argv)
+int main(int argc, char **argv)
{
using namespace hfst;
using namespace hfst::lexc;
const unsigned int TYPES_SIZE=3;//4;
- const ImplementationType types [] = {SFST_TYPE,
- TROPICAL_OPENFST_TYPE,
- /*LOG_OPENFST_TYPE,*/
+ const ImplementationType types [] = {SFST_TYPE,
+ TROPICAL_OPENFST_TYPE,
+ /*LOG_OPENFST_TYPE,*/
FOMA_TYPE};
const char* srcdirc = getenv("srcdir");
@@ -49,16 +49,16 @@ int main(int argc, char **argv)
- try
+ try
{
fprintf(stderr, "valid file, read_lexc... ", i);
- HfstTransducer * rlexc
+ HfstTransducer * rlexc
= HfstTransducer::read_lexc(srcdir + "/test_lexc.lexc", types[i]);
assert(rlexc != 0);
assert(animals.compare(*rlexc));
delete rlexc;
}
- catch (FunctionNotImplementedException e)
+ catch (FunctionNotImplementedException e)
{
assert(false);
}
@@ -94,7 +94,7 @@ int main(int argc, char **argv)
/*try {
HfstTransducer * rlexc
= HfstTransducer::read_lexc
- (std::string(getenv("srcdir")) +
+ (std::string(getenv("srcdir")) +
"/a_test_file_that_does_not_exist.o2f393480f31fsfgqe", types[i]);
assert(false);
}
diff --git a/test/libhfst/test_rules.cc b/test/libhfst/test_rules.cc
index 99cabc8..c13acac 100644
--- a/test/libhfst/test_rules.cc
+++ b/test/libhfst/test_rules.cc
@@ -62,21 +62,21 @@ void compare_and_delete(HfstTransducer * rule_transducers1[],
int main(int argc, char **argv) {
ImplementationType types [] = {SFST_TYPE, TROPICAL_OPENFST_TYPE, FOMA_TYPE};
- HfstTransducer * rule_transducers1 [3] = {NULL, NULL, NULL};
- HfstTransducer * rule_transducers2 [3] = {NULL, NULL, NULL};
- HfstTransducer * rule_transducers3 [3] = {NULL, NULL, NULL};
+ HfstTransducer * rule_transducers1 [3] = {NULL, NULL, NULL};
+ HfstTransducer * rule_transducers2 [3] = {NULL, NULL, NULL};
+ HfstTransducer * rule_transducers3 [3] = {NULL, NULL, NULL};
/* HfstTransducer two_level_if(
- HfstTransducerPair &context,
- StringPairSet &mappings,
- StringPairSet &alphabet,
+ HfstTransducerPair &context,
+ StringPairSet &mappings,
+ StringPairSet &alphabet,
ImplementationType type); */
verbose_print("HfstTransducer two_level_if("
- "HfstTransducerPair &context,"
- "StringPairSet &mappings,"
- "StringPairSet &alphabet,"
+ "HfstTransducerPair &context,"
+ "StringPairSet &mappings,"
+ "StringPairSet &alphabet,"
"ImplementationType type");
{
@@ -98,11 +98,11 @@ int main(int argc, char **argv) {
alphabet.insert(StringPair("c", "c"));
- HfstTransducer rule_transducer1
+ HfstTransducer rule_transducer1
= rules::two_level_if(context, mappings, alphabet);
- HfstTransducer rule_transducer2
+ HfstTransducer rule_transducer2
= rules::two_level_only_if(context, mappings, alphabet);
- HfstTransducer rule_transducer3
+ HfstTransducer rule_transducer3
= rules::two_level_if_and_only_if(context, mappings, alphabet);
rule_transducers1[i] = new HfstTransducer(rule_transducer1);
rule_transducers2[i] = new HfstTransducer(rule_transducer2);
@@ -138,7 +138,7 @@ int main(int argc, char **argv) {
HfstTransducer test_abababa("abababa", TOK, types[i]);
test_abababa.compose(replace_down_transducer);
- HfstTransducer abxaba("abababa",
+ HfstTransducer abxaba("abababa",
"abx at _EPSILON_SYMBOL_@aba", TOK, types[i]);
HfstTransducer ababxa("abababa",
"ababx at _EPSILON_SYMBOL_@a", TOK, types[i]);
@@ -151,17 +151,17 @@ int main(int argc, char **argv) {
#ifdef FOO
/* HfstTransducer &replace_in_context(
- HfstTransducerPair &context,
- ReplaceType repl_type,
- HfstTransducer &t,
- bool optional,
+ HfstTransducerPair &context,
+ ReplaceType repl_type,
+ HfstTransducer &t,
+ bool optional,
StringPairSet &alphabet) */
verbose_print("HfstTransducer &replace_in_context("
- "HfstTransducerPair &context,"
- "ReplaceType repl_type,"
- "HfstTransducer &t,"
- "bool optional,"
+ "HfstTransducerPair &context,"
+ "ReplaceType repl_type,"
+ "HfstTransducer &t,"
+ "bool optional,"
"StringPairSet &alphabet");
for (int i=0; i<3; i++) {
@@ -194,7 +194,7 @@ int main(int argc, char **argv) {
mapping.add_transition
(0, HfstBasicTransition(1, "@_EPSILON_SYMBOL_@", "X", 0));
- mapping.set_final_weight(1, 0);
+ mapping.set_final_weight(1, 0);
mapping.add_transition
(0, HfstBasicTransition(2, "@_UNKNOWN_SYMBOL_@", "X", 0));
@@ -220,7 +220,7 @@ int main(int argc, char **argv) {
HfstTransducerPair context(HfstTransducer("K", "K", type),
HfstTransducer("K", "K", type));
- HfstTransducer rule
+ HfstTransducer rule
= rules::replace_up(context, mapping_, optional, alphabet);
HfstTokenizer TOK;
diff --git a/test/libhfst/test_streams.cc b/test/libhfst/test_streams.cc
index 8dbf319..63f490c 100644
--- a/test/libhfst/test_streams.cc
+++ b/test/libhfst/test_streams.cc
@@ -8,12 +8,12 @@
using namespace hfst;
-int main(int argc, char **argv)
+int main(int argc, char **argv)
{
const unsigned int TYPES_SIZE=3;
- const ImplementationType types [] = {SFST_TYPE,
- TROPICAL_OPENFST_TYPE,
+ const ImplementationType types [] = {SFST_TYPE,
+ TROPICAL_OPENFST_TYPE,
/*LOG_OPENFST_TYPE,*/
FOMA_TYPE};
@@ -27,10 +27,10 @@ int main(int argc, char **argv)
verbose_print("Construction from AT&T format", types[i]);
unsigned int transducers_read=0;
- FILE * file = fopen((std::string(getenv("srcdir")) +
+ FILE * file = fopen((std::string(getenv("srcdir")) +
std::string("/test_transducers.att")).c_str(), "rb");
assert(file != NULL);
- try
+ try
{
while (not feof(file))
{
@@ -41,7 +41,7 @@ int main(int argc, char **argv)
}
}
//catch (hfst::exceptions::NotValidAttFormatException e)
- catch (const HfstException e)
+ catch (const HfstException e)
{
assert(transducers_read == 4);
}
@@ -68,7 +68,7 @@ int main(int argc, char **argv)
remove("transducer2.att");
remove("transducer.att");
- /* From HfstInputStream. */
+ /* From HfstInputStream. */
verbose_print("Writing to HfstOutputStream", types[i]);
HfstTransducer tr1("foo", types[i]);
@@ -90,7 +90,7 @@ int main(int argc, char **argv)
transducers_read=0;
while (not in.is_eof())
{
- HfstTransducer tr(in);
+ HfstTransducer tr(in);
transducers.push_back(tr);
transducers_read++;
}
diff --git a/test/libhfst/test_tokenizer.cc b/test/libhfst/test_tokenizer.cc
index 308555b..e458556 100644
--- a/test/libhfst/test_tokenizer.cc
+++ b/test/libhfst/test_tokenizer.cc
@@ -7,7 +7,7 @@
using namespace hfst;
-int main(int argc, char **argv)
+int main(int argc, char **argv)
{
/* From one string. */
@@ -106,8 +106,8 @@ int main(int argc, char **argv)
line[1] = get_bin(1,0,0,0,0,0,0,0);
line[2] = 0;
try
- {
- utf8_tokenizer.check_utf8_correctness(line);
+ {
+ utf8_tokenizer.check_utf8_correctness(line);
assert(false);
}
catch (const IncorrectUtf8CodingException & e) {};
@@ -117,8 +117,8 @@ int main(int argc, char **argv)
line[1] = get_bin(1,0,0,0,0,0,0,0);
line[2] = 0;
try
- {
- utf8_tokenizer.check_utf8_correctness(line);
+ {
+ utf8_tokenizer.check_utf8_correctness(line);
assert(false);
}
catch (const IncorrectUtf8CodingException & e) {};
@@ -130,8 +130,8 @@ int main(int argc, char **argv)
line[3] = get_bin(1,0,0,0,0,0,0,0);
line[4] = 0;
try
- {
- utf8_tokenizer.check_utf8_correctness(line);
+ {
+ utf8_tokenizer.check_utf8_correctness(line);
assert(false);
}
catch (const IncorrectUtf8CodingException & e) {};
@@ -143,8 +143,8 @@ int main(int argc, char **argv)
line[3] = get_bin(1,0,0,0,0,0,0,0);
line[4] = 0;
try
- {
- utf8_tokenizer.check_utf8_correctness(line);
+ {
+ utf8_tokenizer.check_utf8_correctness(line);
assert(false);
}
catch (const IncorrectUtf8CodingException & e) {};
@@ -156,8 +156,8 @@ int main(int argc, char **argv)
line[3] = get_bin(1,0,0,0,0,0,0,0);
line[4] = 0;
try
- {
- utf8_tokenizer.check_utf8_correctness(line);
+ {
+ utf8_tokenizer.check_utf8_correctness(line);
assert(false);
}
catch (const IncorrectUtf8CodingException & e) {};
@@ -167,8 +167,8 @@ int main(int argc, char **argv)
line[1] = get_bin(1,0,0,0,0,0,0,0);
line[2] = 0;
try
- {
- utf8_tokenizer.check_utf8_correctness(line);
+ {
+ utf8_tokenizer.check_utf8_correctness(line);
assert(false);
}
catch (const IncorrectUtf8CodingException & e) {};
@@ -178,8 +178,8 @@ int main(int argc, char **argv)
line[1] = get_bin(0,1,0,0,0,0,0,0);
line[2] = 0;
try
- {
- utf8_tokenizer.check_utf8_correctness(line);
+ {
+ utf8_tokenizer.check_utf8_correctness(line);
assert(false);
}
catch (const IncorrectUtf8CodingException & e) {};
@@ -190,8 +190,8 @@ int main(int argc, char **argv)
line[2] = get_bin(0,1,0,0,0,0,0,0);
line[3] = 0;
try
- {
- utf8_tokenizer.check_utf8_correctness(line);
+ {
+ utf8_tokenizer.check_utf8_correctness(line);
assert(false);
}
catch (const IncorrectUtf8CodingException & e) {};
@@ -203,8 +203,8 @@ int main(int argc, char **argv)
line[3] = get_bin(0,1,0,0,0,0,0,0);
line[4] = 0;
try
- {
- utf8_tokenizer.check_utf8_correctness(line);
+ {
+ utf8_tokenizer.check_utf8_correctness(line);
assert(false);
}
catch (const IncorrectUtf8CodingException & e) {};
@@ -217,8 +217,8 @@ int main(int argc, char **argv)
line[4] = get_bin(1,0,0,0,0,0,0,0);
line[5] = 0;
try
- {
- utf8_tokenizer.check_utf8_correctness(line);
+ {
+ utf8_tokenizer.check_utf8_correctness(line);
assert(false);
}
catch (const IncorrectUtf8CodingException & e) {};
diff --git a/test/libhfst/test_transducer_functions.cc b/test/libhfst/test_transducer_functions.cc
index 0330bfd..3766ca6 100644
--- a/test/libhfst/test_transducer_functions.cc
+++ b/test/libhfst/test_transducer_functions.cc
@@ -14,24 +14,24 @@
- transform_weights
Functions:
- concatenate
- determinize
- disjunct
- input_project
- intersect
- invert
- minimize
- optionalize
- output_project
- remove_epsilons
- repeat_n
- repeat_n_minus
- repeat_n_plus
- repeat_n_to_k
- repeat_plus
- repeat_star
- reverse
- subtract
+ concatenate
+ determinize
+ disjunct
+ input_project
+ intersect
+ invert
+ minimize
+ optionalize
+ output_project
+ remove_epsilons
+ repeat_n
+ repeat_n_minus
+ repeat_n_plus
+ repeat_n_to_k
+ repeat_plus
+ repeat_star
+ reverse
+ subtract
are tested in command line tool tests
*/
@@ -53,12 +53,12 @@ bool compare_alphabets(const HfstTransducer &t1, const HfstTransducer &t2)
// DEBUG
if (not retval)
{
- for (StringSet::const_iterator it = alpha1.begin();
+ for (StringSet::const_iterator it = alpha1.begin();
it != alpha1.end(); it++) {
std::cerr << *it << std::endl;
}
std::cerr << "--" << std::endl;
- for (StringSet::const_iterator it = alpha2.begin();
+ for (StringSet::const_iterator it = alpha2.begin();
it != alpha1.end(); it++) {
std::cerr << *it << std::endl;
}
@@ -78,10 +78,10 @@ bool compare_string_vectors(const StringVector &v1, const StringVector &v2,
std::string v1_string;
std::string v2_string;
- for (unsigned int i=0; i<v1.size(); i++) {
+ for (unsigned int i=0; i<v1.size(); i++) {
v1_string.append(v1[i]);
}
- for (unsigned int i=0; i<v2.size(); i++) {
+ for (unsigned int i=0; i<v2.size(); i++) {
v2_string.append(v2[i]);
}
// fprintf(stderr, "####### comparing strings '%s' and '%s'...\n", v1_string.c_str(), v2_string.c_str() ); // debug
@@ -112,7 +112,7 @@ bool do_hfst_lookup_paths_contain(const HfstOneLevelPaths &results,
for (HfstOneLevelPaths::const_iterator it = results.begin();
it != results.end(); it++)
{
- if (compare_string_vectors(it->second, expected_path, true))
+ if (compare_string_vectors(it->second, expected_path, true))
{
found = true;
weight = it->first;
@@ -125,12 +125,12 @@ bool do_hfst_lookup_paths_contain(const HfstOneLevelPaths &results,
if (not test_path_weight)
return true;
- if (weight > (path_weight - 0.01) &&
+ if (weight > (path_weight - 0.01) &&
weight < (path_weight + 0.01))
return true;
fprintf(stderr, "FAIL: The path weight is %f, %f expected.\n",
weight, path_weight);
- return false;
+ return false;
}
bool do_results_contain(const HfstTwoLevelPaths &paths,
@@ -158,7 +158,7 @@ bool do_results_contain(const HfstTwoLevelPaths &paths,
{
if (not test_path_weight)
return true;
- if (it->first > (weight - 0.01) &&
+ if (it->first > (weight - 0.01) &&
it->first < (weight + 0.01))
return true;
}
@@ -171,7 +171,7 @@ bool do_results_contain(const HfstTwoLevelPaths &paths,
for (WeightedPaths<float>::Set::const_iterator it = paths.begin();
it != paths.end(); it++)
{
- fprintf(stderr, "%s:%s\t%f\n",
+ fprintf(stderr, "%s:%s\t%f\n",
it->istring.c_str(), it->ostring.c_str(), it->weight);
}
}*/
@@ -218,13 +218,13 @@ bool modify_transitions(const StringPair &sp, StringPairSet &sps)
}
-int main(int argc, char **argv)
+int main(int argc, char **argv)
{
const unsigned int TYPES_SIZE=3;
- const ImplementationType types [] = {SFST_TYPE,
- TROPICAL_OPENFST_TYPE,
- /*LOG_OPENFST_TYPE,*/
+ const ImplementationType types [] = {SFST_TYPE,
+ TROPICAL_OPENFST_TYPE,
+ /*LOG_OPENFST_TYPE,*/
FOMA_TYPE};
/* For all transducer implementation types, perform the following tests: */
@@ -317,7 +317,7 @@ int main(int argc, char **argv)
HfstTransducer t1("foo", "bar", types[i]);
HfstTransducer t2("foo", "bar", types[i]);
- /* Go through all implementation formats
+ /* Go through all implementation formats
and get back to the original one.*/
for (unsigned int j=0; j<=TYPES_SIZE; j++)
{
@@ -361,8 +361,8 @@ int main(int argc, char **argv)
expected_results.insert(StringPair("mouse","mice"));
HfstTwoLevelPaths results;
- animals.extract_paths(results,
- 3, /* max_num */
+ animals.extract_paths(results,
+ 3, /* max_num */
0 /* cycles */ );
/* Test that results are as expected. */
@@ -411,10 +411,10 @@ int main(int argc, char **argv)
// FATAL: EncodeMapper: Weight-encoded arc has non-trivial weight
if (types[i] != LOG_OPENFST_TYPE)
{
- HfstTransducer hippopotamus1("hippopotamus", "hippopotami",
+ HfstTransducer hippopotamus1("hippopotamus", "hippopotami",
tok, types[i]);
hippopotamus1.set_final_weights(1.2);
- HfstTransducer hippopotamus2("hippopotamus", "hippopotamuses",
+ HfstTransducer hippopotamus2("hippopotamus", "hippopotamuses",
tok, types[i]);
hippopotamus2.set_final_weights(1.4);
animals.disjunct(hippopotamus1);
@@ -442,7 +442,7 @@ int main(int argc, char **argv)
StringVector lookup_cat = tok.tokenize_one_level("cat");
StringVector lookup_dog = tok.tokenize_one_level("dog");
StringVector lookup_mouse = tok.tokenize_one_level("mouse");
- StringVector lookup_hippopotamus
+ StringVector lookup_hippopotamus
= tok.tokenize_one_level("hippopotamus");
/* where results of lookup are stored */
@@ -508,7 +508,7 @@ int main(int argc, char **argv)
// FATAL: SingleShortestPath: Weight needs to have the path property
// and be right distributive: log
if (types[i] != LOG_OPENFST_TYPE)
- {
+ {
/* Function n_best. */
verbose_print("function n_best", types[i]);
@@ -555,7 +555,7 @@ int main(int argc, char **argv)
&&
do_results_contain
(results3, "mouse", "mice", 1.7, true));
- }
+ }
HfstTransducer animals4(animals);
animals4.n_best(4);
@@ -574,7 +574,7 @@ int main(int argc, char **argv)
&&
do_results_contain
(results4, "dog", "dogs", 2.5, true));
- }
+ }
HfstTransducer animals5(animals);
animals5.n_best(5);
@@ -596,7 +596,7 @@ int main(int argc, char **argv)
&&
do_results_contain
(results5, "cat", "cats", 3.0, true));
- }
+ }
}
@@ -673,14 +673,14 @@ int main(int argc, char **argv)
{
verbose_print("function push_weights", types[i]);
- /* Create an HFST basic transducer [a:b] with transition
+ /* Create an HFST basic transducer [a:b] with transition
weight 0.3 and final weight 0.5. */
HfstBasicTransducer t;
t.add_state(1);
t.add_transition(0, HfstBasicTransition(1, "a", "b", 0.3));
t.set_final_weight(1, 0.5);
- /* Convert to tropical OpenFst format and push weights
+ /* Convert to tropical OpenFst format and push weights
toward final and initial states. */
HfstTransducer T_final(t, TROPICAL_OPENFST_TYPE);
T_final.push_weights(TO_FINAL_STATE);
@@ -693,10 +693,10 @@ int main(int argc, char **argv)
/* Test the final weight. */
try {
- /* Rounding can affect the precision. */
+ /* Rounding can affect the precision. */
assert(0.79 < t_final.get_final_weight(1) &&
t_final.get_final_weight(1) < 0.81);
- }
+ }
/* If the state does not exist or is not final */
//catch (hfst::exceptions::HfstArgumentException e) {
catch (const HfstException e) {
@@ -705,10 +705,10 @@ int main(int argc, char **argv)
/* Test the transition weight. */
try {
- HfstBasicTransducer::HfstTransitions transitions = t_initial[0];
+ hfst::implementations::HfstBasicTransitions transitions = t_initial[0];
assert(transitions.size() == 1);
float weight = transitions.begin()->get_weight();
- /* Rounding can affect the precision. */
+ /* Rounding can affect the precision. */
assert(0.79 < weight &&
weight < 0.81);
}
@@ -730,7 +730,7 @@ int main(int argc, char **argv)
verbose_print("functions set_final_weights and "
"transform_weights", types[i]);
- /* Create an HFST basic transducer [a:b] with transition
+ /* Create an HFST basic transducer [a:b] with transition
weight 0.3 and final weight 0.5. */
HfstBasicTransducer t;
t.add_state(1);
@@ -744,8 +744,8 @@ int main(int argc, char **argv)
T.push_weights(TO_FINAL_STATE);
/* Convert back to HFST basic transducer and test the weight. */
- HfstBasicTransducer tc(T);
- try {
+ HfstBasicTransducer tc(T);
+ try {
assert(0.24 < tc.get_final_weight(1) &&
tc.get_final_weight(1) < 0.26);
}
@@ -773,7 +773,7 @@ int main(int argc, char **argv)
t1.substitute("t", "T", false, true);
t1.substitute("@_EPSILON_SYMBOL_@", "<eps>");
t1.substitute("a", "A");
- t1.substitute("T", "T"); // special
+ t1.substitute("T", "T"); // special
t1.substitute("foo", "bar"); // cases
HfstTransducer t1_("CAt<eps>", "cATs", tok, types[i]);
assert(t1.compare(t1_));
@@ -868,7 +868,7 @@ int main(int argc, char **argv)
HfstTransducer id2id_copy(id2id);
HfstTransducer id2id_copy2(id2id);
- a2b_copy.concatenate(id2id_copy);
+ a2b_copy.concatenate(id2id_copy);
assert(id2id_copy.compare(id2id_copy2));
assert(id2id_copy.get_alphabet() == id2id_copy2.get_alphabet());
}
@@ -878,7 +878,7 @@ int main(int argc, char **argv)
HfstTransducer id2id_copy(id2id);
HfstTransducer id2id_copy2(id2id);
- a2b_copy.disjunct(id2id_copy);
+ a2b_copy.disjunct(id2id_copy);
assert(id2id_copy.compare(id2id_copy2));
assert(id2id_copy.get_alphabet() == id2id_copy2.get_alphabet());
}
@@ -888,7 +888,7 @@ int main(int argc, char **argv)
HfstTransducer id2id_copy(id2id);
HfstTransducer id2id_copy2(id2id);
- a2b_copy.intersect(id2id_copy);
+ a2b_copy.intersect(id2id_copy);
assert(id2id_copy.compare(id2id_copy2));
assert(id2id_copy.get_alphabet() == id2id_copy2.get_alphabet());
}
@@ -898,7 +898,7 @@ int main(int argc, char **argv)
HfstTransducer id2id_copy(id2id);
HfstTransducer id2id_copy2(id2id);
- a2b_copy.subtract(id2id_copy);
+ a2b_copy.subtract(id2id_copy);
assert(id2id_copy.compare(id2id_copy2));
assert(id2id_copy.get_alphabet() == id2id_copy2.get_alphabet());
}
@@ -908,7 +908,7 @@ int main(int argc, char **argv)
HfstTransducer id2id_copy(id2id);
HfstTransducer id2id_copy2(id2id);
- a2b_copy.compose(id2id_copy);
+ a2b_copy.compose(id2id_copy);
assert(id2id_copy.compare(id2id_copy2));
assert(id2id_copy.get_alphabet() == id2id_copy2.get_alphabet());
}
@@ -918,12 +918,12 @@ int main(int argc, char **argv)
HfstTransducer id2id_copy(id2id);
HfstTransducer id2id_copy2(id2id);
- a2b_copy.insert_freely(id2id_copy);
+ a2b_copy.insert_freely(id2id_copy);
assert(id2id_copy.compare(id2id_copy2));
assert(id2id_copy.get_alphabet() == id2id_copy2.get_alphabet());
}
- /* Test that binary functions work when the argument and the calling
+ /* Test that binary functions work when the argument and the calling
object are the same. */
{
HfstTransducer foo("foo", types[i]);
diff --git a/test/tools/Makefile.am b/test/tools/Makefile.am
index 89f8a2c..2a9b90b 100644
--- a/test/tools/Makefile.am
+++ b/test/tools/Makefile.am
@@ -162,6 +162,9 @@ endif
if WANT_PROC
TESTS += proc-functionality.sh
endif
+if WANT_TOKENIZE
+TESTS += tokenize-functionality.sh tokenize-backtrack-functionality.sh tokenize-flushing-functionality.sh
+endif
TESTS += $(STRESSES)
TESTS += mismatching-input-streams.sh
@@ -308,6 +311,7 @@ LEXC_TXTS=basic.cat-dog-bird.lexc basic.colons.lexc basic.comments.lexc \
cat.lexc hfst.weights.lexc stress.random-lexicons-100.lexc \
xfail.bogus.lexc xfail.ISO-8859-1.lexc xfail.lexicon-semicolon.lexc \
xfail.sublexicon-defined-more-than-once.lexc \
+ xre.more-than-twice.lexc xre.less-than-twice.lexc \
xre.any-variations.lexc xre.at-file.lexc \
xre.automatic-multichar-symbols.lexc xre.basic.lexc \
xre.definitions.lexc xre.months.lexc xre.nested-definitions.lexc \
@@ -353,82 +357,101 @@ ALL_SRCS=$(FST_TXTS) $(FST_STRINGS) $(FST_PAIRS) $(FST_PAIRSTRINGS) \
EXTRA_FILES=cat2dog.substitute cat.strings cat.txt cats_and_dogs.xre utf-8.strings latin-1.strings c_a_t.strings dos.strings not-contains-a.xre not-contains-a-comment-emptyline.xre parallel-left-arrow-multicom-emptyline.xre parallel-left-arrow.xre cat.prolog negative_epsilon_cycles.txt no_negative_epsilon_cycles.txt substituting_transducer.txt substituted_transducer.txt
-RESULT_FILES=basic.cat-dog-bird.lexc.flag.result \
-basic.cat-dog-bird.lexc.result \
-basic.colons.lexc.flag.result \
-basic.colons.lexc.result \
-basic.comments.lexc.flag.result \
-basic.comments.lexc.result \
-basic.empty-sides.lexc.flag.result \
-basic.empty-sides.lexc.result \
-basic.escapes.lexc.flag.result \
-basic.escapes.lexc.result \
-basic.infostrings.lexc.flag.result \
-basic.infostrings.lexc.result \
-basic.initial-lexicon-empty.lexc.flag.result \
-basic.initial-lexicon-empty.lexc.result \
-basic.lowercase-lexicon-end.lexc.flag.result \
-basic.lowercase-lexicon-end.lexc.result \
-basic.multichar-symbols.lexc.flag.result \
-basic.multichar-symbols.lexc.result \
-basic.multi-entry-lines.lexc.flag.result \
-basic.multi-entry-lines.lexc.result \
-basic.multichar-escaped-zero.lexc.flag.result \
-basic.multichar-escaped-zero.lexc.result \
-basic.no-newline-at-end.lexc.flag.result \
-basic.no-newline-at-end.lexc.result \
-basic.no-Root.lexc.flag.result \
-basic.no-Root.lexc.result \
-basic.punctuation.lexc.flag.result \
-basic.punctuation.lexc.result \
-basic.almost-reserved-words.lexc.flag.result \
-basic.almost-reserved-words.lexc.result \
-basic.regexps.lexc.flag.result \
-basic.regexps.lexc.result \
-basic.root-loop.lexc.flag.result \
-basic.root-loop.lexc.result \
-basic.spurious-lexicon.lexc.flag.result \
-basic.spurious-lexicon.lexc.result \
-basic.string-pairs.lexc.flag.result \
-basic.string-pairs.lexc.result \
-basic.two-lexicons.lexc.flag.result \
-basic.two-lexicons.lexc.result \
-basic.UTF-8.lexc.flag.result \
-basic.UTF-8.lexc.result \
-basic.zeros-epsilons.lexc.flag.result \
-basic.zeros-epsilons.lexc.result \
-hfst.weights.lexc.flag.result \
-hfst.weights.lexc.result \
-xre.any-variations.lexc.flag.result \
-xre.any-variations.lexc.result \
-xre.automatic-multichar-symbols.lexc.flag.result \
-xre.automatic-multichar-symbols.lexc.result \
-xre.basic.lexc.flag.result \
-xre.basic.lexc.result \
-xre.definitions.lexc.flag.result \
-xre.definitions.lexc.result \
-xre.months.lexc.flag.result \
-xre.months.lexc.result \
-xre.nested-definitions.lexc.flag.result \
-xre.nested-definitions.lexc.result \
-xre.numeric-star.lexc.flag.result \
-xre.numeric-star.lexc.result \
-xre.quotations.lexc.flag.result \
-xre.quotations.lexc.result \
-xre.sharp.lexc.flag.result \
-xre.sharp.lexc.result \
-xre.star-plus-optional.lexc.flag.result \
-xre.star-plus-optional.lexc.result \
-no-newline-before-sublexicon.lexc.flag.result \
-no-newline-before-sublexicon.lexc.result \
-warn.sublexicon-mentioned-but-not-defined.lexc.flag.result \
-warn.sublexicon-mentioned-but-not-defined.lexc.result \
-warn.one-sided-flags.lexc.flag.result \
-warn.one-sided-flags.lexc.result
+TOKENIZE_FILES=tokenize-dog-in.lexc \
+tokenize-dog-out-cg.strings \
+tokenize-dog-out-giella-cg.strings \
+tokenize-dog-out.strings \
+tokenize-dog-out-xerox.strings \
+tokenize-dog.pmscript \
+tokenize-backtrack.lexc \
+tokenize-backtrack.pmscript \
+tokenize-backtrack-out-giella-cg-spaces.strings \
+tokenize-backtrack-out-giella-cg-contiguous.strings \
+tokenize-backtrack-out-giella-cg.strings \
+tokenize-dog-out-giella-cg-superblank.strings \
+tokenize-dog-out-giella-cg-flushing.strings
+
+
+RESULT_FILES=basic.cat-dog-bird.lexc.flag.result.prolog \
+basic.cat-dog-bird.lexc.result.prolog \
+basic.colons.lexc.flag.result.prolog \
+basic.colons.lexc.result.prolog \
+basic.comments.lexc.flag.result.prolog \
+basic.comments.lexc.result.prolog \
+basic.empty-sides.lexc.flag.result.prolog \
+basic.empty-sides.lexc.result.prolog \
+basic.escapes.lexc.flag.result.prolog \
+basic.escapes.lexc.result.prolog \
+basic.infostrings.lexc.flag.result.prolog \
+basic.infostrings.lexc.result.prolog \
+basic.initial-lexicon-empty.lexc.flag.result.prolog \
+basic.initial-lexicon-empty.lexc.result.prolog \
+basic.lowercase-lexicon-end.lexc.flag.result.prolog \
+basic.lowercase-lexicon-end.lexc.result.prolog \
+basic.multichar-symbols.lexc.flag.result.prolog \
+basic.multichar-symbols.lexc.result.prolog \
+basic.multi-entry-lines.lexc.flag.result.prolog \
+basic.multi-entry-lines.lexc.result.prolog \
+basic.multichar-escaped-zero.lexc.flag.result.prolog \
+basic.multichar-escaped-zero.lexc.result.prolog \
+basic.no-newline-at-end.lexc.flag.result.prolog \
+basic.no-newline-at-end.lexc.result.prolog \
+basic.no-Root.lexc.flag.result.prolog \
+basic.no-Root.lexc.result.prolog \
+basic.punctuation.lexc.flag.result.prolog \
+basic.punctuation.lexc.result.prolog \
+basic.almost-reserved-words.lexc.flag.result.prolog \
+basic.almost-reserved-words.lexc.result.prolog \
+basic.regexps.lexc.flag.result.prolog \
+basic.regexps.lexc.result.prolog \
+basic.root-loop.lexc.flag.result.prolog \
+basic.root-loop.lexc.result.prolog \
+basic.spurious-lexicon.lexc.flag.result.prolog \
+basic.spurious-lexicon.lexc.result.prolog \
+basic.string-pairs.lexc.flag.result.prolog \
+basic.string-pairs.lexc.result.prolog \
+basic.two-lexicons.lexc.flag.result.prolog \
+basic.two-lexicons.lexc.result.prolog \
+basic.UTF-8.lexc.flag.result.prolog \
+basic.UTF-8.lexc.result.prolog \
+basic.zeros-epsilons.lexc.flag.result.prolog \
+basic.zeros-epsilons.lexc.result.prolog \
+hfst.weights.lexc.flag.result.prolog \
+hfst.weights.lexc.result.prolog \
+xre.more-than-twice.lexc.flag.result.prolog \
+xre.more-than-twice.lexc.result.prolog \
+xre.less-than-twice.lexc.flag.result.prolog \
+xre.less-than-twice.lexc.result.prolog \
+xre.any-variations.lexc.flag.result.prolog \
+xre.any-variations.lexc.result.prolog \
+xre.automatic-multichar-symbols.lexc.flag.result.prolog \
+xre.automatic-multichar-symbols.lexc.result.prolog \
+xre.basic.lexc.flag.result.prolog \
+xre.basic.lexc.result.prolog \
+xre.definitions.lexc.flag.result.prolog \
+xre.definitions.lexc.result.prolog \
+xre.months.lexc.flag.result.prolog \
+xre.months.lexc.result.prolog \
+xre.nested-definitions.lexc.flag.result.prolog \
+xre.nested-definitions.lexc.result.prolog \
+xre.numeric-star.lexc.flag.result.prolog \
+xre.numeric-star.lexc.result.prolog \
+xre.quotations.lexc.flag.result.prolog \
+xre.quotations.lexc.result.prolog \
+xre.sharp.lexc.flag.result.prolog \
+xre.sharp.lexc.result.prolog \
+xre.star-plus-optional.lexc.flag.result.prolog \
+xre.star-plus-optional.lexc.result.prolog \
+no-newline-before-sublexicon.lexc.flag.result.prolog \
+no-newline-before-sublexicon.lexc.result.prolog \
+warn.sublexicon-mentioned-but-not-defined.lexc.flag.result.prolog \
+warn.sublexicon-mentioned-but-not-defined.lexc.result.prolog \
+warn.one-sided-flags.lexc.flag.result.prolog \
+warn.one-sided-flags.lexc.result.prolog
# files needed for test programs
EXTRA_DIST=empty-file $(FST_TXTS) $(FST_STRINGS) $(FST_PAIRS) $(FST_PAIRSTRINGS) $(FST_SPACESTRINGS) $(SUBSTITUTE_TXTS) $(XRE_TXTS) $(XFST_TXTS) $(TESTS) $(EXTRA_FILES) $(LEXC_TXTS) $(PMATCH_TXTS) $(PMATCHSCRIPTS) script.xfst lexc2fst-stress.sh lookup-stress.sh proc-stress.sh valgrind.sh \
-calculate-functionality.sh shuffle-functionality.sh $(RESULT_FILES)
+calculate-functionality.sh shuffle-functionality.sh $(RESULT_FILES) $(TOKENIZE_FILES)
valgrind: $(CHECK_DATA)
diff --git a/test/tools/basic.UTF-8.lexc.flag.result b/test/tools/basic.UTF-8.lexc.flag.result
deleted file mode 100644
index 49e5199..0000000
Binary files a/test/tools/basic.UTF-8.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.UTF-8.lexc.flag.result.prolog b/test/tools/basic.UTF-8.lexc.flag.result.prolog
new file mode 100644
index 0000000..c389a74
--- /dev/null
+++ b/test/tools/basic.UTF-8.lexc.flag.result.prolog
@@ -0,0 +1,26 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "ä", 0.000000).
+arc(NO_NAME_1, 2, 4, "ö", 0.000000).
+arc(NO_NAME_1, 2, 5, "Á":"á", 0.000000).
+arc(NO_NAME_1, 2, 6, "p", 0.000000).
+arc(NO_NAME_1, 3, 7, "g", 0.000000).
+arc(NO_NAME_1, 4, 8, "y", 0.000000).
+arc(NO_NAME_1, 5, 9, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 6, 10, "o", 0.000000).
+arc(NO_NAME_1, 7, 11, "r", 0.000000).
+arc(NO_NAME_1, 8, 12, "l", 0.000000).
+arc(NO_NAME_1, 9, 13, "@R.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 10, 14, "h", 0.000000).
+arc(NO_NAME_1, 11, 15, "ä", 0.000000).
+arc(NO_NAME_1, 12, 16, "ä", 0.000000).
+arc(NO_NAME_1, 14, 17, "á":"a", 0.000000).
+arc(NO_NAME_1, 15, 5, "s", 0.000000).
+arc(NO_NAME_1, 16, 18, "t", 0.000000).
+arc(NO_NAME_1, 17, 19, "r", 0.000000).
+arc(NO_NAME_1, 18, 20, "t", 0.000000).
+arc(NO_NAME_1, 19, 21, "0":"a", 0.000000).
+arc(NO_NAME_1, 20, 5, "i", 0.000000).
+arc(NO_NAME_1, 21, 5, "0":"t", 0.000000).
+final(NO_NAME_1, 13, 0.000000).
diff --git a/test/tools/basic.UTF-8.lexc.result b/test/tools/basic.UTF-8.lexc.result
deleted file mode 100644
index 2591c0f..0000000
--- a/test/tools/basic.UTF-8.lexc.result
+++ /dev/null
@@ -1,44 +0,0 @@
-##foma-net 1.0##
-##props##
-2 20 18 22 1 4 1 1 1 1 1 2 1FB91813
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 a
-4 g
-5 h
-6 i
-7 l
-8 o
-9 p
-10 r
-11 s
-12 t
-13 y
-14 Á
-15 á
-16 ä
-17 ö
-##states##
-0 9 11 0
-14 15 17
-17 5
-16 1
-1 4 2 0
-2 10 3 0
-3 16 4 0
-4 11 17 0
-5 13 6 0
-6 7 7 0
-7 16 8 0
-8 12 9 0
-9 12 10 0
-10 6 17 0
-11 8 12 0
-12 5 13 0
-13 15 3 14 0
-14 10 15 0
-15 0 3 16 0
-16 0 12 17 0
-17 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.UTF-8.lexc.result.prolog b/test/tools/basic.UTF-8.lexc.result.prolog
new file mode 100644
index 0000000..c593814
--- /dev/null
+++ b/test/tools/basic.UTF-8.lexc.result.prolog
@@ -0,0 +1,22 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 11, "p").
+arc(NO_NAME_1, 0, 17, "Á":"á").
+arc(NO_NAME_1, 0, 5, "ö").
+arc(NO_NAME_1, 0, 1, "ä").
+arc(NO_NAME_1, 1, 2, "g").
+arc(NO_NAME_1, 2, 3, "r").
+arc(NO_NAME_1, 3, 4, "ä").
+arc(NO_NAME_1, 4, 17, "s").
+arc(NO_NAME_1, 5, 6, "y").
+arc(NO_NAME_1, 6, 7, "l").
+arc(NO_NAME_1, 7, 8, "ä").
+arc(NO_NAME_1, 8, 9, "t").
+arc(NO_NAME_1, 9, 10, "t").
+arc(NO_NAME_1, 10, 17, "i").
+arc(NO_NAME_1, 11, 12, "o").
+arc(NO_NAME_1, 12, 13, "h").
+arc(NO_NAME_1, 13, 14, "á":"a").
+arc(NO_NAME_1, 14, 15, "r").
+arc(NO_NAME_1, 15, 16, "0":"a").
+arc(NO_NAME_1, 16, 17, "0":"t").
+final(NO_NAME_1, 17).
diff --git a/test/tools/basic.almost-reserved-words.lexc.flag.result b/test/tools/basic.almost-reserved-words.lexc.flag.result
deleted file mode 100644
index dd129df..0000000
Binary files a/test/tools/basic.almost-reserved-words.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.almost-reserved-words.lexc.flag.result.prolog b/test/tools/basic.almost-reserved-words.lexc.flag.result.prolog
new file mode 100644
index 0000000..6ffe5a9
--- /dev/null
+++ b/test/tools/basic.almost-reserved-words.lexc.flag.result.prolog
@@ -0,0 +1,49 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@").
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@").
+arc(NO_NAME_1, 2, 3, "m").
+arc(NO_NAME_1, 2, 4, "l").
+arc(NO_NAME_1, 2, 5, "e").
+arc(NO_NAME_1, 2, 6, "d").
+arc(NO_NAME_1, 2, 5, "E").
+arc(NO_NAME_1, 3, 29, "u").
+arc(NO_NAME_1, 4, 24, "e").
+arc(NO_NAME_1, 5, 23, "n").
+arc(NO_NAME_1, 6, 7, "e").
+arc(NO_NAME_1, 7, 8, "f").
+arc(NO_NAME_1, 7, 9, "c").
+arc(NO_NAME_1, 8, 21, "i").
+arc(NO_NAME_1, 9, 10, "l").
+arc(NO_NAME_1, 10, 11, "a").
+arc(NO_NAME_1, 11, 12, "r").
+arc(NO_NAME_1, 12, 13, "a").
+arc(NO_NAME_1, 13, 14, "t").
+arc(NO_NAME_1, 14, 15, "i").
+arc(NO_NAME_1, 15, 16, "o").
+arc(NO_NAME_1, 16, 17, "n").
+arc(NO_NAME_1, 17, 18, "s").
+arc(NO_NAME_1, 18, 19, "@P.LEXNAME.#@").
+arc(NO_NAME_1, 19, 20, "@R.LEXNAME.#@").
+arc(NO_NAME_1, 21, 22, "n").
+arc(NO_NAME_1, 22, 13, "i").
+arc(NO_NAME_1, 23, 18, "d").
+arc(NO_NAME_1, 24, 25, "x").
+arc(NO_NAME_1, 25, 26, "i").
+arc(NO_NAME_1, 26, 27, "c").
+arc(NO_NAME_1, 27, 28, "o").
+arc(NO_NAME_1, 28, 18, "n").
+arc(NO_NAME_1, 29, 30, "l").
+arc(NO_NAME_1, 30, 31, "t").
+arc(NO_NAME_1, 31, 32, "i").
+arc(NO_NAME_1, 32, 33, "c").
+arc(NO_NAME_1, 33, 34, "h").
+arc(NO_NAME_1, 34, 35, "a").
+arc(NO_NAME_1, 35, 36, "r").
+arc(NO_NAME_1, 36, 37, "_").
+arc(NO_NAME_1, 37, 38, "s").
+arc(NO_NAME_1, 38, 39, "y").
+arc(NO_NAME_1, 39, 40, "m").
+arc(NO_NAME_1, 40, 41, "b").
+arc(NO_NAME_1, 41, 42, "o").
+arc(NO_NAME_1, 42, 17, "l").
+final(NO_NAME_1, 20).
diff --git a/test/tools/basic.almost-reserved-words.lexc.result b/test/tools/basic.almost-reserved-words.lexc.result
deleted file mode 100644
index ed03fc3..0000000
Binary files a/test/tools/basic.almost-reserved-words.lexc.result and /dev/null differ
diff --git a/test/tools/basic.almost-reserved-words.lexc.result.prolog b/test/tools/basic.almost-reserved-words.lexc.result.prolog
new file mode 100644
index 0000000..890cc27
--- /dev/null
+++ b/test/tools/basic.almost-reserved-words.lexc.result.prolog
@@ -0,0 +1,45 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 38, "E").
+arc(NO_NAME_1, 0, 25, "d").
+arc(NO_NAME_1, 0, 38, "e").
+arc(NO_NAME_1, 0, 18, "l").
+arc(NO_NAME_1, 0, 1, "m").
+arc(NO_NAME_1, 1, 2, "u").
+arc(NO_NAME_1, 2, 3, "l").
+arc(NO_NAME_1, 3, 4, "t").
+arc(NO_NAME_1, 4, 5, "i").
+arc(NO_NAME_1, 5, 6, "c").
+arc(NO_NAME_1, 6, 7, "h").
+arc(NO_NAME_1, 7, 8, "a").
+arc(NO_NAME_1, 8, 9, "r").
+arc(NO_NAME_1, 9, 10, "_").
+arc(NO_NAME_1, 10, 11, "s").
+arc(NO_NAME_1, 11, 12, "y").
+arc(NO_NAME_1, 12, 13, "m").
+arc(NO_NAME_1, 13, 14, "b").
+arc(NO_NAME_1, 14, 15, "o").
+arc(NO_NAME_1, 15, 16, "l").
+arc(NO_NAME_1, 16, 17, "s").
+arc(NO_NAME_1, 18, 19, "e").
+arc(NO_NAME_1, 19, 20, "x").
+arc(NO_NAME_1, 20, 21, "i").
+arc(NO_NAME_1, 21, 22, "c").
+arc(NO_NAME_1, 22, 23, "o").
+arc(NO_NAME_1, 23, 17, "n").
+arc(NO_NAME_1, 24, 17, "d").
+arc(NO_NAME_1, 25, 26, "e").
+arc(NO_NAME_1, 26, 34, "c").
+arc(NO_NAME_1, 26, 27, "f").
+arc(NO_NAME_1, 27, 28, "i").
+arc(NO_NAME_1, 28, 29, "n").
+arc(NO_NAME_1, 29, 30, "i").
+arc(NO_NAME_1, 30, 31, "t").
+arc(NO_NAME_1, 31, 32, "i").
+arc(NO_NAME_1, 32, 33, "o").
+arc(NO_NAME_1, 33, 16, "n").
+arc(NO_NAME_1, 34, 35, "l").
+arc(NO_NAME_1, 35, 36, "a").
+arc(NO_NAME_1, 36, 37, "r").
+arc(NO_NAME_1, 37, 30, "a").
+arc(NO_NAME_1, 38, 24, "n").
+final(NO_NAME_1, 17).
diff --git a/test/tools/basic.cat-dog-bird.lexc.flag.result b/test/tools/basic.cat-dog-bird.lexc.flag.result
deleted file mode 100644
index 1990366..0000000
Binary files a/test/tools/basic.cat-dog-bird.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.cat-dog-bird.lexc.flag.result.prolog b/test/tools/basic.cat-dog-bird.lexc.flag.result.prolog
new file mode 100644
index 0000000..f5013f5
--- /dev/null
+++ b/test/tools/basic.cat-dog-bird.lexc.flag.result.prolog
@@ -0,0 +1,16 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "d", 0.000000).
+arc(NO_NAME_1, 2, 4, "c", 0.000000).
+arc(NO_NAME_1, 2, 5, "b", 0.000000).
+arc(NO_NAME_1, 3, 6, "o", 0.000000).
+arc(NO_NAME_1, 4, 7, "a", 0.000000).
+arc(NO_NAME_1, 5, 8, "i", 0.000000).
+arc(NO_NAME_1, 6, 9, "g", 0.000000).
+arc(NO_NAME_1, 7, 9, "t", 0.000000).
+arc(NO_NAME_1, 8, 10, "r", 0.000000).
+arc(NO_NAME_1, 9, 11, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 10, 9, "d", 0.000000).
+arc(NO_NAME_1, 11, 12, "@R.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 12, 0.000000).
diff --git a/test/tools/basic.cat-dog-bird.lexc.result b/test/tools/basic.cat-dog-bird.lexc.result
deleted file mode 100644
index 31ee8e8..0000000
--- a/test/tools/basic.cat-dog-bird.lexc.result
+++ /dev/null
@@ -1,27 +0,0 @@
-##foma-net 1.0##
-##props##
-1 10 9 12 1 3 1 1 1 1 1 2 314E36FD
-##sigma##
-3 a
-4 b
-5 c
-6 d
-7 g
-8 i
-9 o
-10 r
-11 t
-##states##
-0 4 5 0
-5 3
-6 1
-1 9 2 0
-2 7 8 0
-3 3 4 0
-4 11 8 0
-5 8 6 0
-6 10 7 0
-7 6 8 0
-8 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.cat-dog-bird.lexc.result.prolog b/test/tools/basic.cat-dog-bird.lexc.result.prolog
new file mode 100644
index 0000000..b3c9996
--- /dev/null
+++ b/test/tools/basic.cat-dog-bird.lexc.result.prolog
@@ -0,0 +1,12 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 5, "b").
+arc(NO_NAME_1, 0, 3, "c").
+arc(NO_NAME_1, 0, 1, "d").
+arc(NO_NAME_1, 1, 2, "o").
+arc(NO_NAME_1, 2, 8, "g").
+arc(NO_NAME_1, 3, 4, "a").
+arc(NO_NAME_1, 4, 8, "t").
+arc(NO_NAME_1, 5, 6, "i").
+arc(NO_NAME_1, 6, 7, "r").
+arc(NO_NAME_1, 7, 8, "d").
+final(NO_NAME_1, 8).
diff --git a/test/tools/basic.colons.lexc.flag.result b/test/tools/basic.colons.lexc.flag.result
deleted file mode 100644
index d79b5c3..0000000
Binary files a/test/tools/basic.colons.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.colons.lexc.flag.result.prolog b/test/tools/basic.colons.lexc.flag.result.prolog
new file mode 100644
index 0000000..64e223b
--- /dev/null
+++ b/test/tools/basic.colons.lexc.flag.result.prolog
@@ -0,0 +1,29 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "l":"0", 0.000000).
+arc(NO_NAME_1, 2, 4, "L":"0", 0.000000).
+arc(NO_NAME_1, 2, 5, "C", 0.000000).
+arc(NO_NAME_1, 2, 6, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 2, 7, "L", 0.000000).
+arc(NO_NAME_1, 2, 4, "0":"R", 0.000000).
+arc(NO_NAME_1, 2, 8, "0":"r", 0.000000).
+arc(NO_NAME_1, 2, 9, ":", 0.000000).
+arc(NO_NAME_1, 3, 10, "e":"0", 0.000000).
+arc(NO_NAME_1, 4, 6, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 5, 6, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 5, 11, "p", 0.000000).
+arc(NO_NAME_1, 6, 12, "@R.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 7, 13, "p", 0.000000).
+arc(NO_NAME_1, 8, 14, "0":"i", 0.000000).
+arc(NO_NAME_1, 9, 15, "R", 0.000000).
+arc(NO_NAME_1, 10, 16, "f":"0", 0.000000).
+arc(NO_NAME_1, 11, 17, ":", 0.000000).
+arc(NO_NAME_1, 13, 4, ":", 0.000000).
+arc(NO_NAME_1, 14, 18, "0":"g", 0.000000).
+arc(NO_NAME_1, 15, 4, "p", 0.000000).
+arc(NO_NAME_1, 16, 4, "t":"0", 0.000000).
+arc(NO_NAME_1, 17, 15, "C", 0.000000).
+arc(NO_NAME_1, 18, 19, "0":"h", 0.000000).
+arc(NO_NAME_1, 19, 4, "0":"t", 0.000000).
+final(NO_NAME_1, 12, 0.000000).
diff --git a/test/tools/basic.colons.lexc.result b/test/tools/basic.colons.lexc.result
deleted file mode 100644
index a80e6c2..0000000
--- a/test/tools/basic.colons.lexc.result
+++ /dev/null
@@ -1,43 +0,0 @@
-##foma-net 1.0##
-##props##
-2 21 16 23 3 9 1 1 1 1 1 2 314E36FD
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 :
-4 C
-5 L
-6 R
-7 e
-8 f
-9 g
-10 h
-11 i
-12 l
-13 p
-14 r
-15 t
-##states##
-0 3 13 1
-5 11
-4 8
-0 14 4
-12 0 1
-0 6 15
-5 0 15
-1 7 0 2 0
-2 8 0 3 0
-3 15 0 15 0
-4 0 11 5 0
-5 0 9 6 0
-6 0 10 7 0
-7 0 15 15 0
-8 13 9 1
-9 3 10 0
-10 4 14 0
-11 13 12 0
-12 3 15 0
-13 6 14 0
-14 13 15 0
-15 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.colons.lexc.result.prolog b/test/tools/basic.colons.lexc.result.prolog
new file mode 100644
index 0000000..082f841
--- /dev/null
+++ b/test/tools/basic.colons.lexc.result.prolog
@@ -0,0 +1,25 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 13, ":").
+arc(NO_NAME_1, 0, 11, "L").
+arc(NO_NAME_1, 0, 8, "C").
+arc(NO_NAME_1, 0, 4, "0":"r").
+arc(NO_NAME_1, 0, 1, "l":"0").
+arc(NO_NAME_1, 0, 15, "0":"R").
+arc(NO_NAME_1, 0, 15, "L":"0").
+arc(NO_NAME_1, 1, 2, "e":"0").
+arc(NO_NAME_1, 2, 3, "f":"0").
+arc(NO_NAME_1, 3, 15, "t":"0").
+arc(NO_NAME_1, 4, 5, "0":"i").
+arc(NO_NAME_1, 5, 6, "0":"g").
+arc(NO_NAME_1, 6, 7, "0":"h").
+arc(NO_NAME_1, 7, 15, "0":"t").
+arc(NO_NAME_1, 8, 9, "p").
+arc(NO_NAME_1, 9, 10, ":").
+arc(NO_NAME_1, 10, 14, "C").
+arc(NO_NAME_1, 11, 12, "p").
+arc(NO_NAME_1, 12, 15, ":").
+arc(NO_NAME_1, 13, 14, "R").
+arc(NO_NAME_1, 14, 15, "p").
+final(NO_NAME_1, 0).
+final(NO_NAME_1, 8).
+final(NO_NAME_1, 15).
diff --git a/test/tools/basic.comments.lexc.flag.result b/test/tools/basic.comments.lexc.flag.result
deleted file mode 100644
index b4d51a7..0000000
Binary files a/test/tools/basic.comments.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.comments.lexc.flag.result.prolog b/test/tools/basic.comments.lexc.flag.result.prolog
new file mode 100644
index 0000000..8a5c18b
--- /dev/null
+++ b/test/tools/basic.comments.lexc.flag.result.prolog
@@ -0,0 +1,31 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "f", 0.000000).
+arc(NO_NAME_1, 2, 4, "b", 0.000000).
+arc(NO_NAME_1, 3, 5, "o", 0.000000).
+arc(NO_NAME_1, 4, 6, "a", 0.000000).
+arc(NO_NAME_1, 5, 7, "o", 0.000000).
+arc(NO_NAME_1, 6, 8, "r", 0.000000).
+arc(NO_NAME_1, 7, 9, "@P.LEXNAME.A@", 0.000000).
+arc(NO_NAME_1, 8, 10, "@P.LEXNAME.B@", 0.000000).
+arc(NO_NAME_1, 9, 11, "@R.LEXNAME.A@", 0.000000).
+arc(NO_NAME_1, 10, 12, "@R.LEXNAME.B@", 0.000000).
+arc(NO_NAME_1, 11, 13, "b", 0.000000).
+arc(NO_NAME_1, 12, 14, "!", 0.000000).
+arc(NO_NAME_1, 13, 15, "a", 0.000000).
+arc(NO_NAME_1, 14, 16, "n", 0.000000).
+arc(NO_NAME_1, 15, 17, "z", 0.000000).
+arc(NO_NAME_1, 16, 18, "o", 0.000000).
+arc(NO_NAME_1, 17, 19, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 18, 20, "t", 0.000000).
+arc(NO_NAME_1, 19, 21, "@R.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 20, 22, "a", 0.000000).
+arc(NO_NAME_1, 22, 23, "c", 0.000000).
+arc(NO_NAME_1, 23, 24, "o", 0.000000).
+arc(NO_NAME_1, 24, 25, "m", 0.000000).
+arc(NO_NAME_1, 25, 26, "m", 0.000000).
+arc(NO_NAME_1, 26, 27, "e", 0.000000).
+arc(NO_NAME_1, 27, 28, "n", 0.000000).
+arc(NO_NAME_1, 28, 17, "t", 0.000000).
+final(NO_NAME_1, 21, 0.000000).
diff --git a/test/tools/basic.comments.lexc.result b/test/tools/basic.comments.lexc.result
deleted file mode 100644
index 1366422..0000000
--- a/test/tools/basic.comments.lexc.result
+++ /dev/null
@@ -1,41 +0,0 @@
-##foma-net 1.0##
-##props##
-1 21 21 23 1 2 1 1 1 1 1 2 314E36FD
-##sigma##
-3 !
-4 a
-5 b
-6 c
-7 e
-8 f
-9 m
-10 n
-11 o
-12 r
-13 t
-14 z
-##states##
-0 5 6 0
-8 1
-1 11 2 0
-2 11 3 0
-3 5 4 0
-4 4 5 0
-5 14 20 0
-6 4 7 0
-7 12 8 0
-8 3 9 0
-9 10 10 0
-10 11 11 0
-11 13 12 0
-12 4 13 0
-13 6 14 0
-14 11 15 0
-15 9 16 0
-16 9 17 0
-17 7 18 0
-18 10 19 0
-19 13 20 0
-20 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.comments.lexc.result.prolog b/test/tools/basic.comments.lexc.result.prolog
new file mode 100644
index 0000000..102c852
--- /dev/null
+++ b/test/tools/basic.comments.lexc.result.prolog
@@ -0,0 +1,23 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 6, "b").
+arc(NO_NAME_1, 0, 1, "f").
+arc(NO_NAME_1, 1, 2, "o").
+arc(NO_NAME_1, 2, 3, "o").
+arc(NO_NAME_1, 3, 4, "b").
+arc(NO_NAME_1, 4, 5, "a").
+arc(NO_NAME_1, 5, 20, "z").
+arc(NO_NAME_1, 6, 7, "a").
+arc(NO_NAME_1, 7, 8, "r").
+arc(NO_NAME_1, 8, 9, "!").
+arc(NO_NAME_1, 9, 10, "n").
+arc(NO_NAME_1, 10, 11, "o").
+arc(NO_NAME_1, 11, 12, "t").
+arc(NO_NAME_1, 12, 13, "a").
+arc(NO_NAME_1, 13, 14, "c").
+arc(NO_NAME_1, 14, 15, "o").
+arc(NO_NAME_1, 15, 16, "m").
+arc(NO_NAME_1, 16, 17, "m").
+arc(NO_NAME_1, 17, 18, "e").
+arc(NO_NAME_1, 18, 19, "n").
+arc(NO_NAME_1, 19, 20, "t").
+final(NO_NAME_1, 20).
diff --git a/test/tools/basic.empty-sides.lexc.flag.result b/test/tools/basic.empty-sides.lexc.flag.result
deleted file mode 100644
index d31a350..0000000
Binary files a/test/tools/basic.empty-sides.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.empty-sides.lexc.flag.result.prolog b/test/tools/basic.empty-sides.lexc.flag.result.prolog
new file mode 100644
index 0000000..5f1b46d
--- /dev/null
+++ b/test/tools/basic.empty-sides.lexc.flag.result.prolog
@@ -0,0 +1,25 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "f":"0", 0.000000).
+arc(NO_NAME_1, 2, 4, "b", 0.000000).
+arc(NO_NAME_1, 2, 5, "0":"b", 0.000000).
+arc(NO_NAME_1, 2, 6, "q", 0.000000).
+arc(NO_NAME_1, 3, 7, "o":"0", 0.000000).
+arc(NO_NAME_1, 4, 8, "a", 0.000000).
+arc(NO_NAME_1, 5, 9, "0":"a", 0.000000).
+arc(NO_NAME_1, 6, 10, "u", 0.000000).
+arc(NO_NAME_1, 7, 11, "o":"0", 0.000000).
+arc(NO_NAME_1, 8, 11, "z", 0.000000).
+arc(NO_NAME_1, 9, 12, "0":"r", 0.000000).
+arc(NO_NAME_1, 10, 13, "u", 0.000000).
+arc(NO_NAME_1, 11, 14, "@P.LEXNAME.A@", 0.000000).
+arc(NO_NAME_1, 12, 15, "@P.LEXNAME.B@", 0.000000).
+arc(NO_NAME_1, 13, 12, "x", 0.000000).
+arc(NO_NAME_1, 14, 16, "@R.LEXNAME.A@", 0.000000).
+arc(NO_NAME_1, 15, 17, "@R.LEXNAME.B@", 0.000000).
+arc(NO_NAME_1, 16, 18, "0":"a", 0.000000).
+arc(NO_NAME_1, 17, 18, "b":"0", 0.000000).
+arc(NO_NAME_1, 18, 19, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 19, 20, "@R.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 20, 0.000000).
diff --git a/test/tools/basic.empty-sides.lexc.result b/test/tools/basic.empty-sides.lexc.result
deleted file mode 100644
index f9add21..0000000
--- a/test/tools/basic.empty-sides.lexc.result
+++ /dev/null
@@ -1,33 +0,0 @@
-##foma-net 1.0##
-##props##
-2 15 13 17 1 4 1 1 1 1 1 2 314E36FD
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 a
-4 b
-5 f
-6 o
-7 q
-8 r
-9 u
-10 x
-11 z
-##states##
-0 7 8 0
-4 5
-0 4 3
-5 0 1
-1 6 0 2 0
-2 6 0 7 0
-3 0 3 4 0
-4 0 8 11 0
-5 3 6 0
-6 11 7 0
-7 0 3 12 0
-8 9 9 0
-9 9 10 0
-10 10 11 0
-11 4 0 12 0
-12 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.empty-sides.lexc.result.prolog b/test/tools/basic.empty-sides.lexc.result.prolog
new file mode 100644
index 0000000..859cd6d
--- /dev/null
+++ b/test/tools/basic.empty-sides.lexc.result.prolog
@@ -0,0 +1,17 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 8, "q").
+arc(NO_NAME_1, 0, 5, "b").
+arc(NO_NAME_1, 0, 3, "0":"b").
+arc(NO_NAME_1, 0, 1, "f":"0").
+arc(NO_NAME_1, 1, 2, "o":"0").
+arc(NO_NAME_1, 2, 7, "o":"0").
+arc(NO_NAME_1, 3, 4, "0":"a").
+arc(NO_NAME_1, 4, 11, "0":"r").
+arc(NO_NAME_1, 5, 6, "a").
+arc(NO_NAME_1, 6, 7, "z").
+arc(NO_NAME_1, 7, 12, "0":"a").
+arc(NO_NAME_1, 8, 9, "u").
+arc(NO_NAME_1, 9, 10, "u").
+arc(NO_NAME_1, 10, 11, "x").
+arc(NO_NAME_1, 11, 12, "b":"0").
+final(NO_NAME_1, 12).
diff --git a/test/tools/basic.escapes.lexc.flag.result b/test/tools/basic.escapes.lexc.flag.result
deleted file mode 100644
index 25a3eff..0000000
Binary files a/test/tools/basic.escapes.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.escapes.lexc.flag.result.prolog b/test/tools/basic.escapes.lexc.flag.result.prolog
new file mode 100644
index 0000000..f667d03
--- /dev/null
+++ b/test/tools/basic.escapes.lexc.flag.result.prolog
@@ -0,0 +1,13 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "#", 0.000000).
+arc(NO_NAME_1, 2, 4, ";", 0.000000).
+arc(NO_NAME_1, 2, 5, "2", 0.000000).
+arc(NO_NAME_1, 3, 6, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 4, 7, "b", 0.000000).
+arc(NO_NAME_1, 5, 3, "%", 0.000000).
+arc(NO_NAME_1, 6, 8, "@R.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 7, 9, "a", 0.000000).
+arc(NO_NAME_1, 9, 3, "r", 0.000000).
+final(NO_NAME_1, 8, 0.000000).
diff --git a/test/tools/basic.escapes.lexc.result b/test/tools/basic.escapes.lexc.result
deleted file mode 100644
index 70cfd22..0000000
--- a/test/tools/basic.escapes.lexc.result
+++ /dev/null
@@ -1,22 +0,0 @@
-##foma-net 1.0##
-##props##
-1 7 6 9 1 3 1 1 1 1 1 2 1FB91813
-##sigma##
-3 #
-4 %
-5 2
-6 ;
-7 a
-8 b
-9 r
-##states##
-0 5 4 0
-6 1
-3 5
-1 8 2 0
-2 7 3 0
-3 9 5 0
-4 4 5 0
-5 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.escapes.lexc.result.prolog b/test/tools/basic.escapes.lexc.result.prolog
new file mode 100644
index 0000000..995202f
--- /dev/null
+++ b/test/tools/basic.escapes.lexc.result.prolog
@@ -0,0 +1,9 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 4, "2").
+arc(NO_NAME_1, 0, 1, ";").
+arc(NO_NAME_1, 0, 5, "#").
+arc(NO_NAME_1, 1, 2, "b").
+arc(NO_NAME_1, 2, 3, "a").
+arc(NO_NAME_1, 3, 5, "r").
+arc(NO_NAME_1, 4, 5, "%").
+final(NO_NAME_1, 5).
diff --git a/test/tools/basic.infostrings.lexc.flag.result b/test/tools/basic.infostrings.lexc.flag.result
deleted file mode 100644
index a411b15..0000000
Binary files a/test/tools/basic.infostrings.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.infostrings.lexc.flag.result.prolog b/test/tools/basic.infostrings.lexc.flag.result.prolog
new file mode 100644
index 0000000..3a8b155
--- /dev/null
+++ b/test/tools/basic.infostrings.lexc.flag.result.prolog
@@ -0,0 +1,20 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "k", 0.000000).
+arc(NO_NAME_1, 2, 4, "j", 0.000000).
+arc(NO_NAME_1, 3, 5, "i", 0.000000).
+arc(NO_NAME_1, 3, 6, "o", 0.000000).
+arc(NO_NAME_1, 4, 7, "a", 0.000000).
+arc(NO_NAME_1, 5, 8, "s", 0.000000).
+arc(NO_NAME_1, 6, 9, "i", 0.000000).
+arc(NO_NAME_1, 7, 10, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 8, 11, "s", 0.000000).
+arc(NO_NAME_1, 9, 11, "r", 0.000000).
+arc(NO_NAME_1, 10, 12, "@R.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 11, 13, "a", 0.000000).
+arc(NO_NAME_1, 13, 14, "@P.LEXNAME.Num@", 0.000000).
+arc(NO_NAME_1, 14, 15, "@R.LEXNAME.Num@", 0.000000).
+arc(NO_NAME_1, 15, 10, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 15, 7, "t", 0.000000).
+final(NO_NAME_1, 12, 0.000000).
diff --git a/test/tools/basic.infostrings.lexc.result b/test/tools/basic.infostrings.lexc.result
deleted file mode 100644
index e8584ff..0000000
--- a/test/tools/basic.infostrings.lexc.result
+++ /dev/null
@@ -1,28 +0,0 @@
-##foma-net 1.0##
-##props##
-1 11 10 13 2 5 1 1 1 1 1 2 1FB91813
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 a
-4 i
-5 j
-6 k
-7 o
-8 r
-9 s
-10 t
-##states##
-0 5 8 0
-6 1
-1 4 4 0
-7 2
-2 4 3 0
-3 8 6 0
-4 9 5 0
-5 9 6 0
-6 3 7 0
-7 10 9 1
-8 3 9 0
-9 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.infostrings.lexc.result.prolog b/test/tools/basic.infostrings.lexc.result.prolog
new file mode 100644
index 0000000..21967e1
--- /dev/null
+++ b/test/tools/basic.infostrings.lexc.result.prolog
@@ -0,0 +1,14 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 8, "j").
+arc(NO_NAME_1, 0, 1, "k").
+arc(NO_NAME_1, 1, 4, "i").
+arc(NO_NAME_1, 1, 2, "o").
+arc(NO_NAME_1, 2, 3, "i").
+arc(NO_NAME_1, 3, 6, "r").
+arc(NO_NAME_1, 4, 5, "s").
+arc(NO_NAME_1, 5, 6, "s").
+arc(NO_NAME_1, 6, 7, "a").
+arc(NO_NAME_1, 7, 9, "t").
+arc(NO_NAME_1, 8, 9, "a").
+final(NO_NAME_1, 7).
+final(NO_NAME_1, 9).
diff --git a/test/tools/basic.initial-lexicon-empty.lexc.flag.result b/test/tools/basic.initial-lexicon-empty.lexc.flag.result
deleted file mode 100644
index 84eb891..0000000
Binary files a/test/tools/basic.initial-lexicon-empty.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.initial-lexicon-empty.lexc.flag.result.prolog b/test/tools/basic.initial-lexicon-empty.lexc.flag.result.prolog
new file mode 100644
index 0000000..a323416
--- /dev/null
+++ b/test/tools/basic.initial-lexicon-empty.lexc.flag.result.prolog
@@ -0,0 +1,11 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "@P.LEXNAME.Continuation@", 0.000000).
+arc(NO_NAME_1, 3, 4, "@R.LEXNAME.Continuation@", 0.000000).
+arc(NO_NAME_1, 4, 5, "f", 0.000000).
+arc(NO_NAME_1, 5, 6, "o", 0.000000).
+arc(NO_NAME_1, 6, 7, "o", 0.000000).
+arc(NO_NAME_1, 7, 8, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 8, 9, "@R.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 9, 0.000000).
diff --git a/test/tools/basic.initial-lexicon-empty.lexc.result b/test/tools/basic.initial-lexicon-empty.lexc.result
deleted file mode 100644
index 4d7852b..0000000
--- a/test/tools/basic.initial-lexicon-empty.lexc.result
+++ /dev/null
@@ -1,14 +0,0 @@
-##foma-net 1.0##
-##props##
-1 3 4 5 1 1 1 1 1 1 1 2 1FB91813
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 f
-4 o
-##states##
-0 3 1 0
-1 4 2 0
-2 4 3 0
-3 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.initial-lexicon-empty.lexc.result.prolog b/test/tools/basic.initial-lexicon-empty.lexc.result.prolog
new file mode 100644
index 0000000..4852db1
--- /dev/null
+++ b/test/tools/basic.initial-lexicon-empty.lexc.result.prolog
@@ -0,0 +1,5 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "f").
+arc(NO_NAME_1, 1, 2, "o").
+arc(NO_NAME_1, 2, 3, "o").
+final(NO_NAME_1, 3).
diff --git a/test/tools/basic.lowercase-lexicon-end.lexc.flag.result b/test/tools/basic.lowercase-lexicon-end.lexc.flag.result
deleted file mode 100644
index 741f49b..0000000
Binary files a/test/tools/basic.lowercase-lexicon-end.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.lowercase-lexicon-end.lexc.flag.result.prolog b/test/tools/basic.lowercase-lexicon-end.lexc.flag.result.prolog
new file mode 100644
index 0000000..2514334
--- /dev/null
+++ b/test/tools/basic.lowercase-lexicon-end.lexc.flag.result.prolog
@@ -0,0 +1,46 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "x", 0.000000).
+arc(NO_NAME_1, 3, 4, "z", 0.000000).
+arc(NO_NAME_1, 3, 5, "y", 0.000000).
+arc(NO_NAME_1, 4, 6, "@P.LEXNAME.DER@", 0.000000).
+arc(NO_NAME_1, 5, 7, "z", 0.000000).
+arc(NO_NAME_1, 5, 8, "y", 0.000000).
+arc(NO_NAME_1, 6, 9, "@R.LEXNAME.DER@", 0.000000).
+arc(NO_NAME_1, 7, 10, "@P.LEXNAME.TRANS@", 0.000000).
+arc(NO_NAME_1, 8, 11, "z", 0.000000).
+arc(NO_NAME_1, 9, 12, "@P.LEXNAME.END-check@", 0.000000).
+arc(NO_NAME_1, 9, 13, "g", 0.000000).
+arc(NO_NAME_1, 9, 14, "d", 0.000000).
+arc(NO_NAME_1, 9, 15, "b", 0.000000).
+arc(NO_NAME_1, 10, 16, "@R.LEXNAME.TRANS@", 0.000000).
+arc(NO_NAME_1, 11, 17, "@P.LEXNAME.INTRANS@", 0.000000).
+arc(NO_NAME_1, 12, 18, "@R.LEXNAME.END-check@", 0.000000).
+arc(NO_NAME_1, 13, 19, "h", 0.000000).
+arc(NO_NAME_1, 14, 20, "e", 0.000000).
+arc(NO_NAME_1, 15, 21, "h", 0.000000).
+arc(NO_NAME_1, 16, 4, "@P.TRANS.on@", 0.000000).
+arc(NO_NAME_1, 17, 22, "@R.LEXNAME.INTRANS@", 0.000000).
+arc(NO_NAME_1, 18, 23, "@U.TRANS.on@", 0.000000).
+arc(NO_NAME_1, 18, 24, "@U.TRANS.off@", 0.000000).
+arc(NO_NAME_1, 19, 25, "t", 0.000000).
+arc(NO_NAME_1, 20, 4, "f", 0.000000).
+arc(NO_NAME_1, 21, 26, "h", 0.000000).
+arc(NO_NAME_1, 21, 7, "j", 0.000000).
+arc(NO_NAME_1, 22, 4, "@P.TRANS.off@", 0.000000).
+arc(NO_NAME_1, 23, 27, "@P.LEXNAME.TRANS-end@", 0.000000).
+arc(NO_NAME_1, 24, 28, "@P.LEXNAME.INTRANS-end@", 0.000000).
+arc(NO_NAME_1, 25, 12, "@P.LEXNAME.END-check@", 0.000000).
+arc(NO_NAME_1, 26, 11, "j", 0.000000).
+arc(NO_NAME_1, 27, 29, "@R.LEXNAME.TRANS-end@", 0.000000).
+arc(NO_NAME_1, 28, 30, "@R.LEXNAME.INTRANS-end@", 0.000000).
+arc(NO_NAME_1, 29, 31, "t", 0.000000).
+arc(NO_NAME_1, 30, 32, "i", 0.000000).
+arc(NO_NAME_1, 31, 33, "r", 0.000000).
+arc(NO_NAME_1, 32, 29, "n", 0.000000).
+arc(NO_NAME_1, 33, 34, "@P.LEXNAME.##@", 0.000000).
+arc(NO_NAME_1, 34, 35, "@R.LEXNAME.##@", 0.000000).
+arc(NO_NAME_1, 35, 36, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 36, 37, "@R.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 37, 0.000000).
diff --git a/test/tools/basic.lowercase-lexicon-end.lexc.result b/test/tools/basic.lowercase-lexicon-end.lexc.result
deleted file mode 100644
index d4b381a..0000000
--- a/test/tools/basic.lowercase-lexicon-end.lexc.result
+++ /dev/null
@@ -1,54 +0,0 @@
-##foma-net 1.0##
-##props##
-1 27 20 29 1 -1 1 1 1 1 0 2 1FB91813
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 @P.TRANS.off@
-4 @P.TRANS.on@
-5 @U.TRANS.off@
-6 @U.TRANS.on@
-7 b
-8 d
-9 e
-10 f
-11 g
-12 h
-13 i
-14 j
-15 n
-16 r
-17 t
-18 x
-19 y
-20 z
-##states##
-0 18 1 0
-1 20 6 0
-19 2
-2 20 5 0
-19 3
-3 20 4 0
-4 3 6 0
-5 4 6 0
-6 5 18 0
-6 15
-7 12
-8 10
-11 7
-7 12 8 0
-8 17 9 0
-9 5 18 0
-6 15
-10 9 11 0
-11 10 6 0
-12 12 13 0
-13 14 5 0
-12 14
-14 14 4 0
-15 17 16 0
-16 16 17 0
-17 -1 -1 1
-18 13 19 0
-19 15 15 0
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.lowercase-lexicon-end.lexc.result.prolog b/test/tools/basic.lowercase-lexicon-end.lexc.result.prolog
new file mode 100644
index 0000000..9c7c7cb
--- /dev/null
+++ b/test/tools/basic.lowercase-lexicon-end.lexc.result.prolog
@@ -0,0 +1,29 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "x").
+arc(NO_NAME_1, 1, 6, "z").
+arc(NO_NAME_1, 1, 2, "y").
+arc(NO_NAME_1, 2, 5, "z").
+arc(NO_NAME_1, 2, 3, "y").
+arc(NO_NAME_1, 3, 4, "z").
+arc(NO_NAME_1, 4, 6, "@P.TRANS.off@").
+arc(NO_NAME_1, 5, 6, "@P.TRANS.on@").
+arc(NO_NAME_1, 6, 18, "@U.TRANS.off@").
+arc(NO_NAME_1, 6, 15, "@U.TRANS.on@").
+arc(NO_NAME_1, 6, 12, "b").
+arc(NO_NAME_1, 6, 10, "d").
+arc(NO_NAME_1, 6, 7, "g").
+arc(NO_NAME_1, 7, 8, "h").
+arc(NO_NAME_1, 8, 9, "t").
+arc(NO_NAME_1, 9, 18, "@U.TRANS.off@").
+arc(NO_NAME_1, 9, 15, "@U.TRANS.on@").
+arc(NO_NAME_1, 10, 11, "e").
+arc(NO_NAME_1, 11, 6, "f").
+arc(NO_NAME_1, 12, 13, "h").
+arc(NO_NAME_1, 13, 5, "j").
+arc(NO_NAME_1, 13, 14, "h").
+arc(NO_NAME_1, 14, 4, "j").
+arc(NO_NAME_1, 15, 16, "t").
+arc(NO_NAME_1, 16, 17, "r").
+arc(NO_NAME_1, 18, 19, "i").
+arc(NO_NAME_1, 19, 15, "n").
+final(NO_NAME_1, 17).
diff --git a/test/tools/basic.multi-entry-lines.lexc.flag.result b/test/tools/basic.multi-entry-lines.lexc.flag.result
deleted file mode 100644
index c845ee3..0000000
Binary files a/test/tools/basic.multi-entry-lines.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.multi-entry-lines.lexc.flag.result.prolog b/test/tools/basic.multi-entry-lines.lexc.flag.result.prolog
new file mode 100644
index 0000000..8efca41
--- /dev/null
+++ b/test/tools/basic.multi-entry-lines.lexc.flag.result.prolog
@@ -0,0 +1,14 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "d", 0.000000).
+arc(NO_NAME_1, 2, 3, "l", 0.000000).
+arc(NO_NAME_1, 3, 4, "i", 0.000000).
+arc(NO_NAME_1, 4, 5, "n", 0.000000).
+arc(NO_NAME_1, 5, 6, "e", 0.000000).
+arc(NO_NAME_1, 6, 7, "d", 0.000000).
+arc(NO_NAME_1, 6, 8, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 6, 7, "s", 0.000000).
+arc(NO_NAME_1, 7, 8, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 8, 9, "@R.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 9, 0.000000).
diff --git a/test/tools/basic.multi-entry-lines.lexc.result b/test/tools/basic.multi-entry-lines.lexc.result
deleted file mode 100644
index 2dee36d..0000000
--- a/test/tools/basic.multi-entry-lines.lexc.result
+++ /dev/null
@@ -1,21 +0,0 @@
-##foma-net 1.0##
-##props##
-1 7 6 9 2 6 1 1 1 1 1 2 1FB91813
-##sigma##
-3 d
-4 e
-5 i
-6 l
-7 n
-8 s
-##states##
-0 6 1 0
-3 1
-1 5 2 0
-2 7 3 0
-3 4 4 0
-4 3 5 1
-8 5
-5 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.multi-entry-lines.lexc.result.prolog b/test/tools/basic.multi-entry-lines.lexc.result.prolog
new file mode 100644
index 0000000..978f69b
--- /dev/null
+++ b/test/tools/basic.multi-entry-lines.lexc.result.prolog
@@ -0,0 +1,10 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "l").
+arc(NO_NAME_1, 0, 1, "d").
+arc(NO_NAME_1, 1, 2, "i").
+arc(NO_NAME_1, 2, 3, "n").
+arc(NO_NAME_1, 3, 4, "e").
+arc(NO_NAME_1, 4, 5, "d").
+arc(NO_NAME_1, 4, 5, "s").
+final(NO_NAME_1, 4).
+final(NO_NAME_1, 5).
diff --git a/test/tools/basic.multichar-escaped-zero.lexc.flag.result b/test/tools/basic.multichar-escaped-zero.lexc.flag.result
deleted file mode 100644
index 951c484..0000000
Binary files a/test/tools/basic.multichar-escaped-zero.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.multichar-escaped-zero.lexc.flag.result.prolog b/test/tools/basic.multichar-escaped-zero.lexc.flag.result.prolog
new file mode 100644
index 0000000..b131a40
--- /dev/null
+++ b/test/tools/basic.multichar-escaped-zero.lexc.flag.result.prolog
@@ -0,0 +1,13 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "A0", 0.000000).
+arc(NO_NAME_1, 2, 4, "a", 0.000000).
+arc(NO_NAME_1, 3, 5, "@P.LEXNAME.A@", 0.000000).
+arc(NO_NAME_1, 3, 6, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 4, 7, "A0", 0.000000).
+arc(NO_NAME_1, 5, 8, "@R.LEXNAME.A@", 0.000000).
+arc(NO_NAME_1, 6, 9, "@R.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 7, 6, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 8, 7, "b", 0.000000).
+final(NO_NAME_1, 9, 0.000000).
diff --git a/test/tools/basic.multichar-escaped-zero.lexc.result b/test/tools/basic.multichar-escaped-zero.lexc.result
deleted file mode 100644
index 0182b06..0000000
--- a/test/tools/basic.multichar-escaped-zero.lexc.result
+++ /dev/null
@@ -1,16 +0,0 @@
-##foma-net 1.0##
-##props##
-1 4 4 6 2 3 1 1 1 1 1 2 8730B6
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 A0
-4 a
-5 b
-##states##
-0 3 2 0
-4 1
-1 3 3 0
-2 5 3 1
-3 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.multichar-escaped-zero.lexc.result.prolog b/test/tools/basic.multichar-escaped-zero.lexc.result.prolog
new file mode 100644
index 0000000..2df5e33
--- /dev/null
+++ b/test/tools/basic.multichar-escaped-zero.lexc.result.prolog
@@ -0,0 +1,7 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 2, "A0").
+arc(NO_NAME_1, 0, 1, "a").
+arc(NO_NAME_1, 1, 3, "A0").
+arc(NO_NAME_1, 2, 3, "b").
+final(NO_NAME_1, 2).
+final(NO_NAME_1, 3).
diff --git a/test/tools/basic.multichar-symbols.lexc.flag.result b/test/tools/basic.multichar-symbols.lexc.flag.result
deleted file mode 100644
index fbc6b64..0000000
Binary files a/test/tools/basic.multichar-symbols.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.multichar-symbols.lexc.flag.result.prolog b/test/tools/basic.multichar-symbols.lexc.flag.result.prolog
new file mode 100644
index 0000000..e09af61
--- /dev/null
+++ b/test/tools/basic.multichar-symbols.lexc.flag.result.prolog
@@ -0,0 +1,14 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "+1P":"o", 0.000000).
+arc(NO_NAME_1, 2, 4, "+1P":"a", 0.000000).
+arc(NO_NAME_1, 2, 5, "2P":"a", 0.000000).
+arc(NO_NAME_1, 3, 6, "+Sg":"0", 0.000000).
+arc(NO_NAME_1, 4, 7, "X":"m", 0.000000).
+arc(NO_NAME_1, 5, 6, "+Sg":"s", 0.000000).
+arc(NO_NAME_1, 6, 8, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 7, 9, "0":"u", 0.000000).
+arc(NO_NAME_1, 8, 10, "@R.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 9, 6, "0":"s", 0.000000).
+final(NO_NAME_1, 10, 0.000000).
diff --git a/test/tools/basic.multichar-symbols.lexc.result b/test/tools/basic.multichar-symbols.lexc.result
deleted file mode 100644
index a98e4fc..0000000
--- a/test/tools/basic.multichar-symbols.lexc.result
+++ /dev/null
@@ -1,26 +0,0 @@
-##foma-net 1.0##
-##props##
-2 8 7 10 1 3 1 1 1 1 1 2 1FB91813
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 +1P
-4 +Sg
-5 2P
-6 X
-7 a
-8 m
-9 o
-10 s
-11 u
-##states##
-0 3 7 3 0
-5 7 2
-3 9 1
-1 4 0 6 0
-2 4 10 6 0
-3 6 8 4 0
-4 0 11 5 0
-5 0 10 6 0
-6 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.multichar-symbols.lexc.result.prolog b/test/tools/basic.multichar-symbols.lexc.result.prolog
new file mode 100644
index 0000000..79c9d8b
--- /dev/null
+++ b/test/tools/basic.multichar-symbols.lexc.result.prolog
@@ -0,0 +1,10 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 3, "+1P":"a").
+arc(NO_NAME_1, 0, 2, "2P":"a").
+arc(NO_NAME_1, 0, 1, "+1P":"o").
+arc(NO_NAME_1, 1, 6, "+Sg":"0").
+arc(NO_NAME_1, 2, 6, "+Sg":"s").
+arc(NO_NAME_1, 3, 4, "X":"m").
+arc(NO_NAME_1, 4, 5, "0":"u").
+arc(NO_NAME_1, 5, 6, "0":"s").
+final(NO_NAME_1, 6).
diff --git a/test/tools/basic.no-Root.lexc.flag.result b/test/tools/basic.no-Root.lexc.flag.result
deleted file mode 100644
index 60d19ae..0000000
Binary files a/test/tools/basic.no-Root.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.no-Root.lexc.flag.result.prolog b/test/tools/basic.no-Root.lexc.flag.result.prolog
new file mode 100644
index 0000000..98e5987
--- /dev/null
+++ b/test/tools/basic.no-Root.lexc.flag.result.prolog
@@ -0,0 +1,16 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Foot@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Foot@", 0.000000).
+arc(NO_NAME_1, 2, 3, "d", 0.000000).
+arc(NO_NAME_1, 2, 4, "c", 0.000000).
+arc(NO_NAME_1, 2, 5, "b", 0.000000).
+arc(NO_NAME_1, 3, 6, "o", 0.000000).
+arc(NO_NAME_1, 4, 7, "a", 0.000000).
+arc(NO_NAME_1, 5, 8, "i", 0.000000).
+arc(NO_NAME_1, 6, 9, "g", 0.000000).
+arc(NO_NAME_1, 7, 9, "t", 0.000000).
+arc(NO_NAME_1, 8, 10, "r", 0.000000).
+arc(NO_NAME_1, 9, 11, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 10, 9, "d", 0.000000).
+arc(NO_NAME_1, 11, 12, "@R.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 12, 0.000000).
diff --git a/test/tools/basic.no-Root.lexc.result b/test/tools/basic.no-Root.lexc.result
deleted file mode 100644
index 9d23e8b..0000000
--- a/test/tools/basic.no-Root.lexc.result
+++ /dev/null
@@ -1,27 +0,0 @@
-##foma-net 1.0##
-##props##
-1 10 9 12 1 3 1 1 1 1 1 2 1FB91813
-##sigma##
-3 a
-4 b
-5 c
-6 d
-7 g
-8 i
-9 o
-10 r
-11 t
-##states##
-0 4 5 0
-5 3
-6 1
-1 9 2 0
-2 7 8 0
-3 3 4 0
-4 11 8 0
-5 8 6 0
-6 10 7 0
-7 6 8 0
-8 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.no-Root.lexc.result.prolog b/test/tools/basic.no-Root.lexc.result.prolog
new file mode 100644
index 0000000..b3c9996
--- /dev/null
+++ b/test/tools/basic.no-Root.lexc.result.prolog
@@ -0,0 +1,12 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 5, "b").
+arc(NO_NAME_1, 0, 3, "c").
+arc(NO_NAME_1, 0, 1, "d").
+arc(NO_NAME_1, 1, 2, "o").
+arc(NO_NAME_1, 2, 8, "g").
+arc(NO_NAME_1, 3, 4, "a").
+arc(NO_NAME_1, 4, 8, "t").
+arc(NO_NAME_1, 5, 6, "i").
+arc(NO_NAME_1, 6, 7, "r").
+arc(NO_NAME_1, 7, 8, "d").
+final(NO_NAME_1, 8).
diff --git a/test/tools/basic.no-newline-at-end.lexc.flag.result b/test/tools/basic.no-newline-at-end.lexc.flag.result
deleted file mode 100644
index 72687ee..0000000
Binary files a/test/tools/basic.no-newline-at-end.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.no-newline-at-end.lexc.flag.result.prolog b/test/tools/basic.no-newline-at-end.lexc.flag.result.prolog
new file mode 100644
index 0000000..20729ff
--- /dev/null
+++ b/test/tools/basic.no-newline-at-end.lexc.flag.result.prolog
@@ -0,0 +1,9 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "c", 0.000000).
+arc(NO_NAME_1, 3, 4, "a", 0.000000).
+arc(NO_NAME_1, 4, 5, "t", 0.000000).
+arc(NO_NAME_1, 5, 6, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 6, 7, "@R.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 7, 0.000000).
diff --git a/test/tools/basic.no-newline-at-end.lexc.result b/test/tools/basic.no-newline-at-end.lexc.result
deleted file mode 100644
index f079008..0000000
--- a/test/tools/basic.no-newline-at-end.lexc.result
+++ /dev/null
@@ -1,14 +0,0 @@
-##foma-net 1.0##
-##props##
-1 3 4 5 1 1 1 1 1 1 1 2 1FB91813
-##sigma##
-3 a
-4 c
-5 t
-##states##
-0 4 1 0
-1 3 2 0
-2 5 3 0
-3 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.no-newline-at-end.lexc.result.prolog b/test/tools/basic.no-newline-at-end.lexc.result.prolog
new file mode 100644
index 0000000..721a46d
--- /dev/null
+++ b/test/tools/basic.no-newline-at-end.lexc.result.prolog
@@ -0,0 +1,5 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "c").
+arc(NO_NAME_1, 1, 2, "a").
+arc(NO_NAME_1, 2, 3, "t").
+final(NO_NAME_1, 3).
diff --git a/test/tools/basic.punctuation.lexc.flag.result b/test/tools/basic.punctuation.lexc.flag.result
deleted file mode 100644
index 7813424..0000000
Binary files a/test/tools/basic.punctuation.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.punctuation.lexc.flag.result.prolog b/test/tools/basic.punctuation.lexc.flag.result.prolog
new file mode 100644
index 0000000..e5325ff
--- /dev/null
+++ b/test/tools/basic.punctuation.lexc.flag.result.prolog
@@ -0,0 +1,73 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, ",", 0.000000).
+arc(NO_NAME_1, 2, 4, ".", 0.000000).
+arc(NO_NAME_1, 2, 3, "%?", 0.000000).
+arc(NO_NAME_1, 2, 3, "…", 0.000000).
+arc(NO_NAME_1, 2, 3, "¿", 0.000000).
+arc(NO_NAME_1, 2, 3, "¶", 0.000000).
+arc(NO_NAME_1, 2, 5, "§", 0.000000).
+arc(NO_NAME_1, 2, 3, "$", 0.000000).
+arc(NO_NAME_1, 2, 3, "€", 0.000000).
+arc(NO_NAME_1, 2, 3, "£", 0.000000).
+arc(NO_NAME_1, 2, 3, "®", 0.000000).
+arc(NO_NAME_1, 2, 3, "©", 0.000000).
+arc(NO_NAME_1, 2, 3, "•", 0.000000).
+arc(NO_NAME_1, 2, 3, "●", 0.000000).
+arc(NO_NAME_1, 2, 3, "·", 0.000000).
+arc(NO_NAME_1, 2, 3, "ּ", 0.000000).
+arc(NO_NAME_1, 2, 3, "·", 0.000000).
+arc(NO_NAME_1, 2, 6, "-", 0.000000).
+arc(NO_NAME_1, 2, 3, "–", 0.000000).
+arc(NO_NAME_1, 2, 3, "—", 0.000000).
+arc(NO_NAME_1, 2, 3, "―", 0.000000).
+arc(NO_NAME_1, 2, 3, "=", 0.000000).
+arc(NO_NAME_1, 2, 3, "≈", 0.000000).
+arc(NO_NAME_1, 2, 3, "&", 0.000000).
+arc(NO_NAME_1, 2, 3, "'", 0.000000).
+arc(NO_NAME_1, 2, 3, "*", 0.000000).
+arc(NO_NAME_1, 2, 7, "+", 0.000000).
+arc(NO_NAME_1, 2, 3, "±", 0.000000).
+arc(NO_NAME_1, 2, 3, "`", 0.000000).
+arc(NO_NAME_1, 2, 3, "´", 0.000000).
+arc(NO_NAME_1, 2, 3, "/", 0.000000).
+arc(NO_NAME_1, 2, 3, "~", 0.000000).
+arc(NO_NAME_1, 2, 3, "«", 0.000000).
+arc(NO_NAME_1, 2, 3, "»", 0.000000).
+arc(NO_NAME_1, 2, 3, "“", 0.000000).
+arc(NO_NAME_1, 2, 3, "”", 0.000000).
+arc(NO_NAME_1, 2, 3, "‘", 0.000000).
+arc(NO_NAME_1, 2, 3, "’", 0.000000).
+arc(NO_NAME_1, 2, 3, "„", 0.000000).
+arc(NO_NAME_1, 2, 3, "‹", 0.000000).
+arc(NO_NAME_1, 2, 3, "›", 0.000000).
+arc(NO_NAME_1, 2, 3, "(", 0.000000).
+arc(NO_NAME_1, 2, 3, ")", 0.000000).
+arc(NO_NAME_1, 2, 3, "[", 0.000000).
+arc(NO_NAME_1, 2, 3, "]", 0.000000).
+arc(NO_NAME_1, 2, 3, "{", 0.000000).
+arc(NO_NAME_1, 2, 3, "}", 0.000000).
+arc(NO_NAME_1, 2, 3, "\"", 0.000000).
+arc(NO_NAME_1, 2, 3, "!", 0.000000).
+arc(NO_NAME_1, 2, 3, "<", 0.000000).
+arc(NO_NAME_1, 2, 3, ">", 0.000000).
+arc(NO_NAME_1, 2, 3, "%", 0.000000).
+arc(NO_NAME_1, 2, 3, ":", 0.000000).
+arc(NO_NAME_1, 2, 3, ";", 0.000000).
+arc(NO_NAME_1, 2, 3, "#", 0.000000).
+arc(NO_NAME_1, 3, 8, "+PUNCT":"0", 0.000000).
+arc(NO_NAME_1, 4, 9, ".", 0.000000).
+arc(NO_NAME_1, 4, 8, "+PUNCT":"0", 0.000000).
+arc(NO_NAME_1, 5, 3, "§", 0.000000).
+arc(NO_NAME_1, 5, 8, "+PUNCT":"0", 0.000000).
+arc(NO_NAME_1, 6, 3, "-", 0.000000).
+arc(NO_NAME_1, 6, 8, "+PUNCT":"0", 0.000000).
+arc(NO_NAME_1, 7, 10, "/", 0.000000).
+arc(NO_NAME_1, 7, 8, "+PUNCT":"0", 0.000000).
+arc(NO_NAME_1, 8, 11, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 9, 3, ".", 0.000000).
+arc(NO_NAME_1, 9, 8, "+PUNCT":"0", 0.000000).
+arc(NO_NAME_1, 10, 3, "-", 0.000000).
+arc(NO_NAME_1, 11, 12, "@R.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 12, 0.000000).
diff --git a/test/tools/basic.punctuation.lexc.result b/test/tools/basic.punctuation.lexc.result
deleted file mode 100644
index 161c419..0000000
--- a/test/tools/basic.punctuation.lexc.result
+++ /dev/null
@@ -1,132 +0,0 @@
-##foma-net 1.0##
-##props##
-2 67 9 69 1 60 1 1 1 1 1 2 1FB91813
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 !
-4 "
-5 #
-6 $
-7 %
-8 &
-9 '
-10 (
-11 )
-12 *
-13 +
-14 +PUNCT
-15 ,
-16 -
-17 .
-18 /
-19 :
-20 ;
-21 <
-22 =
-23 >
-24 ?
-25 [
-26 ]
-27 `
-28 {
-29 }
-30 ~
-31 £
-32 §
-33 ©
-34 «
-35 ®
-36 ±
-37 ´
-38 ¶
-39 ·
-40 »
-41 ¿
-42 ·
-43 ּ
-44 –
-45 —
-46 ―
-47 ‘
-48 ’
-49 “
-50 ”
-51 „
-52 •
-53 …
-54 ‹
-55 ›
-56 €
-57 ≈
-58 ●
-##states##
-0 5 7 0
-20 7
-19 7
-7 7
-23 7
-21 7
-3 7
-4 7
-29 7
-28 7
-26 7
-25 7
-11 7
-10 7
-55 7
-54 7
-51 7
-48 7
-47 7
-50 7
-49 7
-40 7
-34 7
-30 7
-18 7
-37 7
-27 7
-36 7
-13 5
-12 7
-9 7
-8 7
-57 7
-22 7
-46 7
-45 7
-44 7
-16 4
-42 7
-43 7
-39 7
-58 7
-52 7
-33 7
-35 7
-31 7
-56 7
-6 7
-32 3
-38 7
-41 7
-53 7
-24 7
-17 1
-15 7
-1 17 2 0
-14 0 8
-2 17 7 0
-14 0 8
-3 32 7 0
-14 0 8
-4 16 7 0
-14 0 8
-5 18 6 0
-14 0 8
-6 16 7 0
-7 14 0 8 0
-8 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.punctuation.lexc.result.prolog b/test/tools/basic.punctuation.lexc.result.prolog
new file mode 100644
index 0000000..eddc74a
--- /dev/null
+++ b/test/tools/basic.punctuation.lexc.result.prolog
@@ -0,0 +1,69 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 7, "#").
+arc(NO_NAME_1, 0, 7, ";").
+arc(NO_NAME_1, 0, 7, ":").
+arc(NO_NAME_1, 0, 7, "%").
+arc(NO_NAME_1, 0, 7, ">").
+arc(NO_NAME_1, 0, 7, "<").
+arc(NO_NAME_1, 0, 7, "!").
+arc(NO_NAME_1, 0, 7, "\"").
+arc(NO_NAME_1, 0, 7, "}").
+arc(NO_NAME_1, 0, 7, "{").
+arc(NO_NAME_1, 0, 7, "]").
+arc(NO_NAME_1, 0, 7, "[").
+arc(NO_NAME_1, 0, 7, ")").
+arc(NO_NAME_1, 0, 7, "(").
+arc(NO_NAME_1, 0, 7, "›").
+arc(NO_NAME_1, 0, 7, "‹").
+arc(NO_NAME_1, 0, 7, "„").
+arc(NO_NAME_1, 0, 7, "’").
+arc(NO_NAME_1, 0, 7, "‘").
+arc(NO_NAME_1, 0, 7, "”").
+arc(NO_NAME_1, 0, 7, "“").
+arc(NO_NAME_1, 0, 7, "»").
+arc(NO_NAME_1, 0, 7, "«").
+arc(NO_NAME_1, 0, 7, "~").
+arc(NO_NAME_1, 0, 7, "/").
+arc(NO_NAME_1, 0, 7, "´").
+arc(NO_NAME_1, 0, 7, "`").
+arc(NO_NAME_1, 0, 7, "±").
+arc(NO_NAME_1, 0, 5, "+").
+arc(NO_NAME_1, 0, 7, "*").
+arc(NO_NAME_1, 0, 7, "'").
+arc(NO_NAME_1, 0, 7, "&").
+arc(NO_NAME_1, 0, 7, "≈").
+arc(NO_NAME_1, 0, 7, "=").
+arc(NO_NAME_1, 0, 7, "―").
+arc(NO_NAME_1, 0, 7, "—").
+arc(NO_NAME_1, 0, 7, "–").
+arc(NO_NAME_1, 0, 4, "-").
+arc(NO_NAME_1, 0, 7, "·").
+arc(NO_NAME_1, 0, 7, "ּ").
+arc(NO_NAME_1, 0, 7, "·").
+arc(NO_NAME_1, 0, 7, "●").
+arc(NO_NAME_1, 0, 7, "•").
+arc(NO_NAME_1, 0, 7, "©").
+arc(NO_NAME_1, 0, 7, "®").
+arc(NO_NAME_1, 0, 7, "£").
+arc(NO_NAME_1, 0, 7, "€").
+arc(NO_NAME_1, 0, 7, "$").
+arc(NO_NAME_1, 0, 3, "§").
+arc(NO_NAME_1, 0, 7, "¶").
+arc(NO_NAME_1, 0, 7, "¿").
+arc(NO_NAME_1, 0, 7, "…").
+arc(NO_NAME_1, 0, 7, "%?").
+arc(NO_NAME_1, 0, 1, ".").
+arc(NO_NAME_1, 0, 7, ",").
+arc(NO_NAME_1, 1, 2, ".").
+arc(NO_NAME_1, 1, 8, "+PUNCT":"0").
+arc(NO_NAME_1, 2, 7, ".").
+arc(NO_NAME_1, 2, 8, "+PUNCT":"0").
+arc(NO_NAME_1, 3, 7, "§").
+arc(NO_NAME_1, 3, 8, "+PUNCT":"0").
+arc(NO_NAME_1, 4, 7, "-").
+arc(NO_NAME_1, 4, 8, "+PUNCT":"0").
+arc(NO_NAME_1, 5, 6, "/").
+arc(NO_NAME_1, 5, 8, "+PUNCT":"0").
+arc(NO_NAME_1, 6, 7, "-").
+arc(NO_NAME_1, 7, 8, "+PUNCT":"0").
+final(NO_NAME_1, 8).
diff --git a/test/tools/basic.regexps.lexc.flag.result b/test/tools/basic.regexps.lexc.flag.result
deleted file mode 100644
index be59c97..0000000
Binary files a/test/tools/basic.regexps.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.regexps.lexc.flag.result.prolog b/test/tools/basic.regexps.lexc.flag.result.prolog
new file mode 100644
index 0000000..fdf6dcf
--- /dev/null
+++ b/test/tools/basic.regexps.lexc.flag.result.prolog
@@ -0,0 +1,14 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@").
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@").
+arc(NO_NAME_1, 2, 3, "a":"C").
+arc(NO_NAME_1, 2, 4, "a":"B").
+arc(NO_NAME_1, 3, 10, "@P.LEXNAME.BAZ@").
+arc(NO_NAME_1, 4, 5, "@P.LEXNAME.BAR@").
+arc(NO_NAME_1, 5, 6, "@R.LEXNAME.BAR@").
+arc(NO_NAME_1, 6, 7, "a":"b").
+arc(NO_NAME_1, 7, 8, "@P.LEXNAME.#@").
+arc(NO_NAME_1, 8, 9, "@R.LEXNAME.#@").
+arc(NO_NAME_1, 10, 11, "@R.LEXNAME.BAZ@").
+arc(NO_NAME_1, 11, 7, "a":"c").
+final(NO_NAME_1, 9).
diff --git a/test/tools/basic.regexps.lexc.result b/test/tools/basic.regexps.lexc.result
deleted file mode 100644
index 961fac4..0000000
Binary files a/test/tools/basic.regexps.lexc.result and /dev/null differ
diff --git a/test/tools/basic.regexps.lexc.result.prolog b/test/tools/basic.regexps.lexc.result.prolog
new file mode 100644
index 0000000..1e20a0d
--- /dev/null
+++ b/test/tools/basic.regexps.lexc.result.prolog
@@ -0,0 +1,6 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "a":"B").
+arc(NO_NAME_1, 0, 2, "a":"C").
+arc(NO_NAME_1, 1, 3, "a":"b").
+arc(NO_NAME_1, 2, 3, "a":"c").
+final(NO_NAME_1, 3).
diff --git a/test/tools/basic.root-loop.lexc.flag.result b/test/tools/basic.root-loop.lexc.flag.result
deleted file mode 100644
index b055d8a..0000000
Binary files a/test/tools/basic.root-loop.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.root-loop.lexc.flag.result.prolog b/test/tools/basic.root-loop.lexc.flag.result.prolog
new file mode 100644
index 0000000..1c8c7e1
--- /dev/null
+++ b/test/tools/basic.root-loop.lexc.flag.result.prolog
@@ -0,0 +1,13 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "v", 0.000000).
+arc(NO_NAME_1, 2, 4, "b", 0.000000).
+arc(NO_NAME_1, 3, 5, "e", 0.000000).
+arc(NO_NAME_1, 4, 6, "i", 0.000000).
+arc(NO_NAME_1, 5, 7, "r", 0.000000).
+arc(NO_NAME_1, 6, 8, "g", 0.000000).
+arc(NO_NAME_1, 7, 0, "y", 0.000000).
+arc(NO_NAME_1, 8, 9, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 9, 10, "@R.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 10, 0.000000).
diff --git a/test/tools/basic.root-loop.lexc.result b/test/tools/basic.root-loop.lexc.result
deleted file mode 100644
index 31a523c..0000000
--- a/test/tools/basic.root-loop.lexc.result
+++ /dev/null
@@ -1,22 +0,0 @@
-##foma-net 1.0##
-##props##
-1 7 7 9 1 -1 1 1 1 1 0 2 1FB91813
-##sigma##
-3 b
-4 e
-5 g
-6 i
-7 r
-8 v
-9 y
-##states##
-0 3 2 0
-8 5
-1 5 6 0
-2 6 1 0
-3 9 0 0
-4 7 3 0
-5 4 4 0
-6 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.root-loop.lexc.result.prolog b/test/tools/basic.root-loop.lexc.result.prolog
new file mode 100644
index 0000000..a0b7109
--- /dev/null
+++ b/test/tools/basic.root-loop.lexc.result.prolog
@@ -0,0 +1,9 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 2, "b").
+arc(NO_NAME_1, 0, 5, "v").
+arc(NO_NAME_1, 1, 6, "g").
+arc(NO_NAME_1, 2, 1, "i").
+arc(NO_NAME_1, 3, 0, "y").
+arc(NO_NAME_1, 4, 3, "r").
+arc(NO_NAME_1, 5, 4, "e").
+final(NO_NAME_1, 6).
diff --git a/test/tools/basic.spurious-lexicon.lexc.flag.result b/test/tools/basic.spurious-lexicon.lexc.flag.result
deleted file mode 100644
index bb64e4f..0000000
Binary files a/test/tools/basic.spurious-lexicon.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.spurious-lexicon.lexc.flag.result.prolog b/test/tools/basic.spurious-lexicon.lexc.flag.result.prolog
new file mode 100644
index 0000000..5fb8ed1
--- /dev/null
+++ b/test/tools/basic.spurious-lexicon.lexc.flag.result.prolog
@@ -0,0 +1,14 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "w", 0.000000).
+arc(NO_NAME_1, 3, 4, "a", 0.000000).
+arc(NO_NAME_1, 4, 5, "l", 0.000000).
+arc(NO_NAME_1, 5, 6, "k", 0.000000).
+arc(NO_NAME_1, 6, 7, "@P.LEXNAME.V@", 0.000000).
+arc(NO_NAME_1, 7, 8, "@R.LEXNAME.V@", 0.000000).
+arc(NO_NAME_1, 8, 9, "s", 0.000000).
+arc(NO_NAME_1, 8, 10, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 9, 10, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 10, 11, "@R.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 11, 0.000000).
diff --git a/test/tools/basic.spurious-lexicon.lexc.result b/test/tools/basic.spurious-lexicon.lexc.result
deleted file mode 100644
index e98dcf3..0000000
--- a/test/tools/basic.spurious-lexicon.lexc.result
+++ /dev/null
@@ -1,19 +0,0 @@
-##foma-net 1.0##
-##props##
-1 5 6 7 2 2 1 1 1 1 1 2 1FB91813
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 a
-4 k
-5 l
-6 s
-7 w
-##states##
-0 7 1 0
-1 3 2 0
-2 5 3 0
-3 4 4 0
-4 6 5 1
-5 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.spurious-lexicon.lexc.result.prolog b/test/tools/basic.spurious-lexicon.lexc.result.prolog
new file mode 100644
index 0000000..21013b1
--- /dev/null
+++ b/test/tools/basic.spurious-lexicon.lexc.result.prolog
@@ -0,0 +1,8 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "w").
+arc(NO_NAME_1, 1, 2, "a").
+arc(NO_NAME_1, 2, 3, "l").
+arc(NO_NAME_1, 3, 4, "k").
+arc(NO_NAME_1, 4, 5, "s").
+final(NO_NAME_1, 4).
+final(NO_NAME_1, 5).
diff --git a/test/tools/basic.string-pairs.lexc.flag.result b/test/tools/basic.string-pairs.lexc.flag.result
deleted file mode 100644
index d5eadb3..0000000
Binary files a/test/tools/basic.string-pairs.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.string-pairs.lexc.flag.result.prolog b/test/tools/basic.string-pairs.lexc.flag.result.prolog
new file mode 100644
index 0000000..35649ce
--- /dev/null
+++ b/test/tools/basic.string-pairs.lexc.flag.result.prolog
@@ -0,0 +1,18 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "s", 0.000000).
+arc(NO_NAME_1, 2, 4, "f", 0.000000).
+arc(NO_NAME_1, 3, 5, "w", 0.000000).
+arc(NO_NAME_1, 4, 6, "i":"o", 0.000000).
+arc(NO_NAME_1, 5, 7, "i":"a", 0.000000).
+arc(NO_NAME_1, 6, 8, "g":"u", 0.000000).
+arc(NO_NAME_1, 7, 9, "m", 0.000000).
+arc(NO_NAME_1, 8, 10, "h":"g", 0.000000).
+arc(NO_NAME_1, 9, 11, "s":"0", 0.000000).
+arc(NO_NAME_1, 9, 12, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 10, 13, "t":"h", 0.000000).
+arc(NO_NAME_1, 11, 12, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 12, 14, "@R.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 13, 11, "0":"t", 0.000000).
+final(NO_NAME_1, 14, 0.000000).
diff --git a/test/tools/basic.string-pairs.lexc.result b/test/tools/basic.string-pairs.lexc.result
deleted file mode 100644
index 267c70f..0000000
--- a/test/tools/basic.string-pairs.lexc.result
+++ /dev/null
@@ -1,31 +0,0 @@
-##foma-net 1.0##
-##props##
-2 11 11 13 2 3 1 1 1 1 1 2 1FB91813
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 a
-4 f
-5 g
-6 h
-7 i
-8 m
-9 o
-10 s
-11 t
-12 u
-13 w
-##states##
-0 4 5 0
-10 1
-1 13 2 0
-2 7 3 3 0
-3 8 4 0
-4 10 0 10 1
-5 7 9 6 0
-6 5 12 7 0
-7 6 5 8 0
-8 11 6 9 0
-9 0 11 10 0
-10 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.string-pairs.lexc.result.prolog b/test/tools/basic.string-pairs.lexc.result.prolog
new file mode 100644
index 0000000..aac106c
--- /dev/null
+++ b/test/tools/basic.string-pairs.lexc.result.prolog
@@ -0,0 +1,14 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 5, "f").
+arc(NO_NAME_1, 0, 1, "s").
+arc(NO_NAME_1, 1, 2, "w").
+arc(NO_NAME_1, 2, 3, "i":"a").
+arc(NO_NAME_1, 3, 4, "m").
+arc(NO_NAME_1, 4, 10, "s":"0").
+arc(NO_NAME_1, 5, 6, "i":"o").
+arc(NO_NAME_1, 6, 7, "g":"u").
+arc(NO_NAME_1, 7, 8, "h":"g").
+arc(NO_NAME_1, 8, 9, "t":"h").
+arc(NO_NAME_1, 9, 10, "0":"t").
+final(NO_NAME_1, 4).
+final(NO_NAME_1, 10).
diff --git a/test/tools/basic.two-lexicons.lexc.flag.result b/test/tools/basic.two-lexicons.lexc.flag.result
deleted file mode 100644
index e0c9bbe..0000000
Binary files a/test/tools/basic.two-lexicons.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.two-lexicons.lexc.flag.result.prolog b/test/tools/basic.two-lexicons.lexc.flag.result.prolog
new file mode 100644
index 0000000..7dccb62
--- /dev/null
+++ b/test/tools/basic.two-lexicons.lexc.flag.result.prolog
@@ -0,0 +1,22 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "w", 0.000000).
+arc(NO_NAME_1, 2, 3, "t", 0.000000).
+arc(NO_NAME_1, 2, 4, "p", 0.000000).
+arc(NO_NAME_1, 3, 5, "a", 0.000000).
+arc(NO_NAME_1, 4, 6, "a", 0.000000).
+arc(NO_NAME_1, 5, 7, "l", 0.000000).
+arc(NO_NAME_1, 6, 7, "c", 0.000000).
+arc(NO_NAME_1, 7, 8, "k", 0.000000).
+arc(NO_NAME_1, 8, 9, "@P.LEXNAME.V@", 0.000000).
+arc(NO_NAME_1, 9, 10, "@R.LEXNAME.V@", 0.000000).
+arc(NO_NAME_1, 10, 11, "s", 0.000000).
+arc(NO_NAME_1, 10, 12, "e", 0.000000).
+arc(NO_NAME_1, 10, 13, "i", 0.000000).
+arc(NO_NAME_1, 11, 14, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 12, 11, "d", 0.000000).
+arc(NO_NAME_1, 13, 15, "n", 0.000000).
+arc(NO_NAME_1, 14, 16, "@R.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 15, 11, "g", 0.000000).
+final(NO_NAME_1, 16, 0.000000).
diff --git a/test/tools/basic.two-lexicons.lexc.result b/test/tools/basic.two-lexicons.lexc.result
deleted file mode 100644
index 3d35c28..0000000
--- a/test/tools/basic.two-lexicons.lexc.result
+++ /dev/null
@@ -1,35 +0,0 @@
-##foma-net 1.0##
-##props##
-1 14 11 16 1 9 1 1 1 1 1 2 1FB91813
-##sigma##
-3 a
-4 c
-5 d
-6 e
-7 g
-8 i
-9 k
-10 l
-11 n
-12 p
-13 s
-14 t
-15 w
-##states##
-0 12 3 0
-14 1
-15 1
-1 3 2 0
-2 10 5 0
-3 3 4 0
-4 4 5 0
-5 9 6 0
-6 8 8 0
-6 7
-13 10
-7 5 10 0
-8 11 9 0
-9 7 10 0
-10 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.two-lexicons.lexc.result.prolog b/test/tools/basic.two-lexicons.lexc.result.prolog
new file mode 100644
index 0000000..4110c97
--- /dev/null
+++ b/test/tools/basic.two-lexicons.lexc.result.prolog
@@ -0,0 +1,16 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 3, "p").
+arc(NO_NAME_1, 0, 1, "t").
+arc(NO_NAME_1, 0, 1, "w").
+arc(NO_NAME_1, 1, 2, "a").
+arc(NO_NAME_1, 2, 5, "l").
+arc(NO_NAME_1, 3, 4, "a").
+arc(NO_NAME_1, 4, 5, "c").
+arc(NO_NAME_1, 5, 6, "k").
+arc(NO_NAME_1, 6, 8, "i").
+arc(NO_NAME_1, 6, 7, "e").
+arc(NO_NAME_1, 6, 10, "s").
+arc(NO_NAME_1, 7, 10, "d").
+arc(NO_NAME_1, 8, 9, "n").
+arc(NO_NAME_1, 9, 10, "g").
+final(NO_NAME_1, 10).
diff --git a/test/tools/basic.zeros-epsilons.lexc.flag.result b/test/tools/basic.zeros-epsilons.lexc.flag.result
deleted file mode 100644
index 423f28d..0000000
Binary files a/test/tools/basic.zeros-epsilons.lexc.flag.result and /dev/null differ
diff --git a/test/tools/basic.zeros-epsilons.lexc.flag.result.prolog b/test/tools/basic.zeros-epsilons.lexc.flag.result.prolog
new file mode 100644
index 0000000..3c05278
--- /dev/null
+++ b/test/tools/basic.zeros-epsilons.lexc.flag.result.prolog
@@ -0,0 +1,46 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "o":"0", 0.000000).
+arc(NO_NAME_1, 2, 4, "v":"0", 0.000000).
+arc(NO_NAME_1, 2, 5, "0":"o", 0.000000).
+arc(NO_NAME_1, 2, 6, "<":"v", 0.000000).
+arc(NO_NAME_1, 2, 7, "0":"v", 0.000000).
+arc(NO_NAME_1, 2, 8, "%0":"n", 0.000000).
+arc(NO_NAME_1, 2, 9, "o":"<", 0.000000).
+arc(NO_NAME_1, 2, 10, "n":"%0", 0.000000).
+arc(NO_NAME_1, 3, 11, "o":"0", 0.000000).
+arc(NO_NAME_1, 4, 12, "a":"0", 0.000000).
+arc(NO_NAME_1, 5, 13, "0":"i", 0.000000).
+arc(NO_NAME_1, 6, 14, ">":"e", 0.000000).
+arc(NO_NAME_1, 7, 15, "0":"e", 0.000000).
+arc(NO_NAME_1, 8, 16, "0":"o", 0.000000).
+arc(NO_NAME_1, 9, 17, "o":">", 0.000000).
+arc(NO_NAME_1, 10, 18, "o":"0", 0.000000).
+arc(NO_NAME_1, 11, 10, "n":"0", 0.000000).
+arc(NO_NAME_1, 12, 19, "s":"0", 0.000000).
+arc(NO_NAME_1, 13, 20, "0":"k", 0.000000).
+arc(NO_NAME_1, 14, 21, "0":"e", 0.000000).
+arc(NO_NAME_1, 15, 22, "0":"e", 0.000000).
+arc(NO_NAME_1, 16, 23, "0":"l", 0.000000).
+arc(NO_NAME_1, 17, 24, "s":"0", 0.000000).
+arc(NO_NAME_1, 18, 25, "l":"0", 0.000000).
+arc(NO_NAME_1, 19, 26, "e":"0", 0.000000).
+arc(NO_NAME_1, 20, 27, "0":"e", 0.000000).
+arc(NO_NAME_1, 21, 28, "0":"s", 0.000000).
+arc(NO_NAME_1, 22, 8, "0":"n", 0.000000).
+arc(NO_NAME_1, 23, 29, "0":"l", 0.000000).
+arc(NO_NAME_1, 24, 30, "f":"0", 0.000000).
+arc(NO_NAME_1, 25, 31, "l":"0", 0.000000).
+arc(NO_NAME_1, 26, 32, "n":"0", 0.000000).
+arc(NO_NAME_1, 27, 32, "0":"e", 0.000000).
+arc(NO_NAME_1, 28, 33, "0":"f", 0.000000).
+arc(NO_NAME_1, 29, 32, "0":"a", 0.000000).
+arc(NO_NAME_1, 30, 34, "s":"0", 0.000000).
+arc(NO_NAME_1, 31, 32, "a":"0", 0.000000).
+arc(NO_NAME_1, 32, 35, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 33, 36, "0":"s", 0.000000).
+arc(NO_NAME_1, 34, 32, "t":"0", 0.000000).
+arc(NO_NAME_1, 35, 37, "@R.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 36, 32, "0":"t", 0.000000).
+final(NO_NAME_1, 37, 0.000000).
diff --git a/test/tools/basic.zeros-epsilons.lexc.result b/test/tools/basic.zeros-epsilons.lexc.result
deleted file mode 100644
index 539411c..0000000
--- a/test/tools/basic.zeros-epsilons.lexc.result
+++ /dev/null
@@ -1,63 +0,0 @@
-##foma-net 1.0##
-##props##
-2 40 34 42 1 8 1 1 1 1 1 2 1FB91813
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 0
-4 <
-5 >
-6 a
-7 e
-8 f
-9 i
-10 k
-11 l
-12 n
-13 o
-14 s
-15 t
-16 v
-##states##
-0 12 3 18 0
-3 12 12
-13 4 28
-4 16 22
-13 0 16
-0 16 9
-16 0 5
-0 13 1
-1 0 9 2 0
-2 0 10 3 0
-3 0 7 4 0
-4 0 7 33 0
-5 6 0 6 0
-6 14 0 7 0
-7 7 0 8 0
-8 12 0 33 0
-9 0 7 10 0
-10 0 7 11 0
-11 0 12 12 0
-12 0 13 13 0
-13 0 11 14 0
-14 0 11 15 0
-15 0 6 33 0
-16 13 0 17 0
-17 12 0 18 0
-18 13 0 19 0
-19 11 0 20 0
-20 11 0 21 0
-21 6 0 33 0
-22 5 7 23 0
-23 0 7 24 0
-24 0 14 25 0
-25 0 8 26 0
-26 0 14 27 0
-27 0 15 33 0
-28 13 5 29 0
-29 14 0 30 0
-30 8 0 31 0
-31 14 0 32 0
-32 15 0 33 0
-33 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/basic.zeros-epsilons.lexc.result.prolog b/test/tools/basic.zeros-epsilons.lexc.result.prolog
new file mode 100644
index 0000000..d7e272f
--- /dev/null
+++ b/test/tools/basic.zeros-epsilons.lexc.result.prolog
@@ -0,0 +1,42 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 18, "n":"%0").
+arc(NO_NAME_1, 0, 12, "%0":"n").
+arc(NO_NAME_1, 0, 28, "o":"<").
+arc(NO_NAME_1, 0, 22, "<":"v").
+arc(NO_NAME_1, 0, 16, "o":"0").
+arc(NO_NAME_1, 0, 9, "0":"v").
+arc(NO_NAME_1, 0, 5, "v":"0").
+arc(NO_NAME_1, 0, 1, "0":"o").
+arc(NO_NAME_1, 1, 2, "0":"i").
+arc(NO_NAME_1, 2, 3, "0":"k").
+arc(NO_NAME_1, 3, 4, "0":"e").
+arc(NO_NAME_1, 4, 33, "0":"e").
+arc(NO_NAME_1, 5, 6, "a":"0").
+arc(NO_NAME_1, 6, 7, "s":"0").
+arc(NO_NAME_1, 7, 8, "e":"0").
+arc(NO_NAME_1, 8, 33, "n":"0").
+arc(NO_NAME_1, 9, 10, "0":"e").
+arc(NO_NAME_1, 10, 11, "0":"e").
+arc(NO_NAME_1, 11, 12, "0":"n").
+arc(NO_NAME_1, 12, 13, "0":"o").
+arc(NO_NAME_1, 13, 14, "0":"l").
+arc(NO_NAME_1, 14, 15, "0":"l").
+arc(NO_NAME_1, 15, 33, "0":"a").
+arc(NO_NAME_1, 16, 17, "o":"0").
+arc(NO_NAME_1, 17, 18, "n":"0").
+arc(NO_NAME_1, 18, 19, "o":"0").
+arc(NO_NAME_1, 19, 20, "l":"0").
+arc(NO_NAME_1, 20, 21, "l":"0").
+arc(NO_NAME_1, 21, 33, "a":"0").
+arc(NO_NAME_1, 22, 23, ">":"e").
+arc(NO_NAME_1, 23, 24, "0":"e").
+arc(NO_NAME_1, 24, 25, "0":"s").
+arc(NO_NAME_1, 25, 26, "0":"f").
+arc(NO_NAME_1, 26, 27, "0":"s").
+arc(NO_NAME_1, 27, 33, "0":"t").
+arc(NO_NAME_1, 28, 29, "o":">").
+arc(NO_NAME_1, 29, 30, "s":"0").
+arc(NO_NAME_1, 30, 31, "f":"0").
+arc(NO_NAME_1, 31, 32, "s":"0").
+arc(NO_NAME_1, 32, 33, "t":"0").
+final(NO_NAME_1, 33).
diff --git a/test/tools/cats_and_dogs_semicolon.xre b/test/tools/cats_and_dogs_semicolon.xre
index 1644d6f..75fb47c 100644
--- a/test/tools/cats_and_dogs_semicolon.xre
+++ b/test/tools/cats_and_dogs_semicolon.xre
@@ -2,4 +2,4 @@ c a t ;
c a t | d o g ;
c:d a:o t:g ;
c:d::1 a:o::2 g:t::3 ;
-c a t ; 3.141
+# c a t ; 3.141 commented because this hasn't been supported for a while
diff --git a/test/tools/compare-functionality.sh b/test/tools/compare-functionality.sh
index 8e2da0c..562a025 100755
--- a/test/tools/compare-functionality.sh
+++ b/test/tools/compare-functionality.sh
@@ -46,3 +46,5 @@ do
fi
fi
done
+
+rm -f tmp1 tmp2 tmp1.txt tmp2.txt
diff --git a/test/tools/fsmbook-tests/hfst-scripts/Lingala.hfst.script b/test/tools/fsmbook-tests/hfst-scripts/Lingala.hfst.script
index e322a1a..56182f0 100644
--- a/test/tools/fsmbook-tests/hfst-scripts/Lingala.hfst.script
+++ b/test/tools/fsmbook-tests/hfst-scripts/Lingala.hfst.script
@@ -58,7 +58,7 @@ echo '"<" @"Stems" "," @"Features" ">" ' | $2/hfst-regexp2fst -f $1 > VerbLex
echo '[[. .] -> {mo} || "<" _ [$[@"Agr" & $@"Person3" & $Sg & $4]]]' | $2/hfst-regexp2fst -f $1 > RAgr1
echo '[[. .] -> {li} || "<" _ [$[@"Agr" & $@"Person3" & $Sg & $5]]]' | $2/hfst-regexp2fst -f $1 > RAgr2
-echo '[[. .] -> e || "<" _ [$[@"Agr" & $@"Person3" & $Sg & $[9a . "10a"]]]]' | $2/hfst-regexp2fst -f $1 > RAgr3
+echo '[[. .] -> e || "<" _ [$[@"Agr" & $@"Person3" & $Sg & $[9a "." "10a"]]]]' | $2/hfst-regexp2fst -f $1 > RAgr3
echo '[[. .] -> {lo} || "<" _ [$[@"Agr" & $@"Person3" & $Sg & $["10"|11]]]]' | $2/hfst-regexp2fst -f $1 > RAgr4
echo '[[. .] -> {bo} || "<" _ [$[@"Agr" & $@"Person3" & $Sg & $14]]]' | $2/hfst-regexp2fst -f $1 > RAgr5
# Common plural agreement markers
diff --git a/test/tools/fsmbook-tests/test.sh b/test/tools/fsmbook-tests/test.sh
index 698c87d..24be9e1 100755
--- a/test/tools/fsmbook-tests/test.sh
+++ b/test/tools/fsmbook-tests/test.sh
@@ -111,6 +111,7 @@ do
echo "ERROR: in converting result from xfst/foma to hfst format"
exit 1;
fi
+ rm -f Result.prolog
# Also compile with hfst-xfst using all back-end formats..
if [ "$COMPILE_XFST_SCRIPT" == "true" ]; then
@@ -149,6 +150,7 @@ do
exit 1;
fi
fi
+ rm -f Result_from_hfst_xfst
done
fi
@@ -268,6 +270,9 @@ do
rm -f Result_from_hfst_script_$format
done
+rm -f tmpdir/NumbersToNumerals
+rmdir tmpdir
+
echo ""
echo "**********"
echo "All fsmbook tests that were performed passed."
diff --git a/test/tools/hfst-check-environment.sh b/test/tools/hfst-check-environment.sh
index 7d1c996..5aee4d5 100755
--- a/test/tools/hfst-check-environment.sh
+++ b/test/tools/hfst-check-environment.sh
@@ -17,5 +17,5 @@ for f in $TOOLDIR/hfst-{compare,compose,compose-intersect,concatenate,disjunct,m
fi
fi
done
-rm version.out 2>1 > /dev/null
+rm version.out 2> /dev/null > /dev/null
HFST_OPTIONS="$old_opts"
diff --git a/test/tools/hfst-check-help.sh b/test/tools/hfst-check-help.sh
index df248d4..19ba7b8 100755
--- a/test/tools/hfst-check-help.sh
+++ b/test/tools/hfst-check-help.sh
@@ -14,7 +14,13 @@ hfst-shuffle hfst-split hfst-strings2fst hfst-substitute \
hfst-subtract hfst-summarize hfst-tail hfst-traverse \
hfst-txt2fst"
-TWOLC_TOOLS="hfst-twolc/src/hfst-twolc-loc"
+TWOLC_TOOLS=
+if [ -e "$TOOLDIR/hfst-twolc/src/hfst-twolc-loc" ] ; then
+ TWOLC_TOOLS="hfst-twolc/src/hfst-twolc-loc"
+else
+ TWOLC_TOOLS="$TOOLDIR/hfst-twolc/src/hfst-twolc"
+fi
+
TAGGER_TOOLS="hfst-tagger/src/hfst-tag /hfst-tagger/src/hfst-train-tagger-loc"
PROC_TOOLS="hfst-proc/hfst-apertium-proc"
XFST_TOOLS="parsers/hfst-xfst"
@@ -22,12 +28,11 @@ XFST_TOOLS="parsers/hfst-xfst"
# Extension for executables
EXT=
# Check if we are in MinGW environment
-if (uname | egrep "MINGW|mingw" 2>1 > /dev/null); then
+if (uname | egrep "MINGW|mingw" 2> /dev/null > /dev/null); then
# Executables have an exe extension
EXT=".exe";
- # We use system call version of the shell script
- TWOLC_TOOLS=hfst-twolc/src/hfst-twolc-system
- # Tagger tool shell script not implemented for windows (MinGW)
+ # TODO: implement these tools for windows
+ TWOLC_TOOLS=
TAGGER_TOOLS=
fi
@@ -96,4 +101,4 @@ for f in $TWOLC_TOOLS $TAGGER_TOOLS; do
fi
done
-rm -f help.out1 help.out2
+rm -f help1.out help2.out
diff --git a/test/tools/hfst-check-version.sh b/test/tools/hfst-check-version.sh
index 57bdca2..04261ae 100755
--- a/test/tools/hfst-check-version.sh
+++ b/test/tools/hfst-check-version.sh
@@ -14,7 +14,13 @@ hfst-shuffle hfst-split hfst-strings2fst hfst-substitute \
hfst-subtract hfst-summarize hfst-tail hfst-traverse \
hfst-txt2fst"
-TWOLC_TOOLS="hfst-twolc/src/hfst-twolc-loc"
+TWOLC_TOOLS=
+if [ -e "$TOOLDIR/hfst-twolc/src/hfst-twolc-loc" ] ; then
+ TWOLC_TOOLS="hfst-twolc/src/hfst-twolc-loc"
+else
+ TWOLC_TOOLS="$TOOLDIR/hfst-twolc/src/hfst-twolc"
+fi
+
TAGGER_TOOLS="hfst-tagger/src/hfst-tag /hfst-tagger/src/hfst-train-tagger-loc"
PROC_TOOLS="hfst-proc/hfst-apertium-proc"
XFST_TOOLS="parsers/hfst-xfst"
@@ -22,13 +28,12 @@ XFST_TOOLS="parsers/hfst-xfst"
# Extension for executables
EXT=
# Check if we are in MinGW environment
-if (uname | egrep "MINGW|mingw" 2>1 > /dev/null); then
+if (uname | egrep "MINGW|mingw" 2> /dev/null > /dev/null); then
# Executables have an exe extension
EXT=".exe";
- # We use system call version of the shell script
- TWOLC_TOOLS=hfst-twolc/src/hfst-twolc-system
- # Tagger tool shell script not implemented for windows (MinGW)
+ # TODO: implement these for windows
TAGGER_TOOLS=
+ TWOLC_TOOLS=
fi
verbose="true"
@@ -99,4 +104,4 @@ for prog in $TWOLC_TOOLS $TAGGER_TOOLS; do
fi
done
-rm -f version1.out1 version2.out
+rm -f version1.out version2.out
diff --git a/test/tools/hfst.weights.lexc.flag.result b/test/tools/hfst.weights.lexc.flag.result
deleted file mode 100644
index a323ec2..0000000
Binary files a/test/tools/hfst.weights.lexc.flag.result and /dev/null differ
diff --git a/test/tools/hfst.weights.lexc.flag.result.prolog b/test/tools/hfst.weights.lexc.flag.result.prolog
new file mode 100644
index 0000000..c1492b4
--- /dev/null
+++ b/test/tools/hfst.weights.lexc.flag.result.prolog
@@ -0,0 +1,16 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 2.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "c", 0.000000).
+arc(NO_NAME_1, 2, 4, "d", 1.000000).
+arc(NO_NAME_1, 3, 5, "a", 0.000000).
+arc(NO_NAME_1, 4, 6, "o", 0.000000).
+arc(NO_NAME_1, 5, 7, "t", 0.000000).
+arc(NO_NAME_1, 6, 7, "g", 0.000000).
+arc(NO_NAME_1, 7, 8, "@P.LEXNAME.Num@", 0.000000).
+arc(NO_NAME_1, 8, 9, "@R.LEXNAME.Num@", 0.000000).
+arc(NO_NAME_1, 9, 10, "+Sg":"0", 0.000000).
+arc(NO_NAME_1, 9, 10, "+Pl":"s", 3.000000).
+arc(NO_NAME_1, 10, 11, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 11, 12, "@R.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 12, 0.000000).
diff --git a/test/tools/hfst.weights.lexc.result b/test/tools/hfst.weights.lexc.result
deleted file mode 100644
index 83038f0..0000000
Binary files a/test/tools/hfst.weights.lexc.result and /dev/null differ
diff --git a/test/tools/hfst.weights.lexc.result.prolog b/test/tools/hfst.weights.lexc.result.prolog
new file mode 100644
index 0000000..f3eb39b
--- /dev/null
+++ b/test/tools/hfst.weights.lexc.result.prolog
@@ -0,0 +1,10 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "c", 2.000000).
+arc(NO_NAME_1, 0, 2, "d", 3.000000).
+arc(NO_NAME_1, 1, 3, "a", 0.000000).
+arc(NO_NAME_1, 2, 4, "o", 0.000000).
+arc(NO_NAME_1, 3, 5, "t", 0.000000).
+arc(NO_NAME_1, 4, 5, "g", 0.000000).
+arc(NO_NAME_1, 5, 6, "+Sg":"0", 0.000000).
+arc(NO_NAME_1, 5, 6, "+Pl":"s", 3.000000).
+final(NO_NAME_1, 6, 0.000000).
diff --git a/test/tools/incompatible-formats.sh b/test/tools/incompatible-formats.sh
index a1f292d..7c62592 100755
--- a/test/tools/incompatible-formats.sh
+++ b/test/tools/incompatible-formats.sh
@@ -5,7 +5,7 @@ for f in "" .sfst .foma .openfst; do
if test cat$f != cat$g ; then
if test -f cat$f -a -f cat$g ; then
for p in compare compose concatenate conjunct disjunct subtract ; do
- if $TOOLDIR/hfst-$p cat$f cat$g 2>1 > /dev/null; then
+ if $TOOLDIR/hfst-$p cat$f cat$g 2> /dev/null > /dev/null; then
echo-$p does not fail cat$f cat$g
exit 0
fi
diff --git a/test/tools/lexc-compiler-functionality.sh b/test/tools/lexc-compiler-functionality.sh
index 2fe2351..788c0f2 100755
--- a/test/tools/lexc-compiler-functionality.sh
+++ b/test/tools/lexc-compiler-functionality.sh
@@ -15,6 +15,8 @@ LEXCTESTS="basic.cat-dog-bird.lexc basic.colons.lexc basic.comments.lexc
basic.almost-reserved-words.lexc
basic.regexps.lexc
hfst.weights.lexc
+ xre.more-than-twice.lexc
+ xre.less-than-twice.lexc
xre.automatic-multichar-symbols.lexc xre.basic.lexc
xre.definitions.lexc xre.months.lexc xre.nested-definitions.lexc
xre.numeric-star.lexc xre.sharp.lexc xre.quotations.lexc
@@ -87,7 +89,7 @@ for i in .sfst .ofst .foma ; do
exit 1
fi
- RESULT="$f.result"
+ RESULT="$f.result.prolog"
# create foma result
# RESULT_GZ="$RESULT.gz"
@@ -99,7 +101,7 @@ for i in .sfst .ofst .foma ; do
# mv test.foma $RESULT
# rm tmp-foma-script
- $TOOLDIR/hfst-fst2fst $FFLAG $RESULT -o $RESULT.tmp
+ $TOOLDIR/hfst-txt2fst --prolog $FFLAG -i $RESULT -o $RESULT.tmp
#echo "comparing file: $f"
if ! $TOOLDIR/hfst-compare -e -s $RESULT.tmp test ; then
@@ -111,7 +113,7 @@ for i in .sfst .ofst .foma ; do
# check flag results
- RESULT="$f.flag.result"
+ RESULT="$f.flag.result.prolog"
if ! $TOOLDIR/hfst-lexc -F $FFLAG $srcdir/$f -o test 2> /dev/null; then
@@ -119,7 +121,7 @@ for i in .sfst .ofst .foma ; do
exit 1
fi
- $TOOLDIR/hfst-fst2fst $FFLAG $RESULT -o $RESULT.tmp
+ $TOOLDIR/hfst-txt2fst --prolog $FFLAG -i $RESULT -o $RESULT.tmp
#echo "comparing flag file: $f"
if ! $TOOLDIR/hfst-compare -e -s $RESULT.tmp test ; then
diff --git a/test/tools/mismatching-input-streams.sh b/test/tools/mismatching-input-streams.sh
index 047fd2a..fd5acf5 100755
--- a/test/tools/mismatching-input-streams.sh
+++ b/test/tools/mismatching-input-streams.sh
@@ -77,5 +77,5 @@ do
done
done
-rm test
+rm -f test test1 test2
diff --git a/test/tools/no-newline-before-sublexicon.lexc.flag.result b/test/tools/no-newline-before-sublexicon.lexc.flag.result
deleted file mode 100644
index f4b5275..0000000
--- a/test/tools/no-newline-before-sublexicon.lexc.flag.result
+++ /dev/null
@@ -1,31 +0,0 @@
-##foma-net 1.0##
-##props##
-2 9 10 11 1 1 1 2 2 1 1 2
-##sigma##
-0 @_EPSILON_SYMBOL_@
-1 @_UNKNOWN_SYMBOL_@
-2 @_IDENTITY_SYMBOL_@
-3 @P.LEXNAME.#@
-4 @P.LEXNAME.BAZ@
-5 @P.LEXNAME.Root@
-6 @R.LEXNAME.#@
-7 @R.LEXNAME.BAZ@
-8 @R.LEXNAME.Root@
-9 a
-10 b
-11 f
-12 o
-13 r
-##states##
-0 5 1 0
-1 8 2 0
-2 11 10 3 0
-3 12 9 4 0
-4 12 13 5 0
-5 4 6 0
-6 7 7 0
-7 3 8 0
-8 6 9 0
-9 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/no-newline-before-sublexicon.lexc.flag.result.prolog b/test/tools/no-newline-before-sublexicon.lexc.flag.result.prolog
new file mode 100644
index 0000000..c34ad7a
--- /dev/null
+++ b/test/tools/no-newline-before-sublexicon.lexc.flag.result.prolog
@@ -0,0 +1,11 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@").
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@").
+arc(NO_NAME_1, 2, 3, "f":"b").
+arc(NO_NAME_1, 3, 4, "o":"a").
+arc(NO_NAME_1, 4, 5, "o":"r").
+arc(NO_NAME_1, 5, 6, "@P.LEXNAME.BAZ@").
+arc(NO_NAME_1, 6, 7, "@R.LEXNAME.BAZ@").
+arc(NO_NAME_1, 7, 8, "@P.LEXNAME.#@").
+arc(NO_NAME_1, 8, 9, "@R.LEXNAME.#@").
+final(NO_NAME_1, 9).
diff --git a/test/tools/no-newline-before-sublexicon.lexc.result b/test/tools/no-newline-before-sublexicon.lexc.result
deleted file mode 100644
index 5b6373a..0000000
--- a/test/tools/no-newline-before-sublexicon.lexc.result
+++ /dev/null
@@ -1,19 +0,0 @@
-##foma-net 1.0##
-##props##
-2 3 4 5 1 1 1 2 2 1 1 2
-##sigma##
-0 @_EPSILON_SYMBOL_@
-1 @_UNKNOWN_SYMBOL_@
-2 @_IDENTITY_SYMBOL_@
-3 a
-4 b
-5 f
-6 o
-7 r
-##states##
-0 5 4 1 0
-1 6 3 2 0
-2 6 7 3 0
-3 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/no-newline-before-sublexicon.lexc.result.prolog b/test/tools/no-newline-before-sublexicon.lexc.result.prolog
new file mode 100644
index 0000000..6aa38c4
--- /dev/null
+++ b/test/tools/no-newline-before-sublexicon.lexc.result.prolog
@@ -0,0 +1,5 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "f":"b").
+arc(NO_NAME_1, 1, 2, "o":"a").
+arc(NO_NAME_1, 2, 3, "o":"r").
+final(NO_NAME_1, 3).
diff --git a/test/tools/pmatch-functionality.sh b/test/tools/pmatch-functionality.sh
index b4d6e2e..039f19f 100755
--- a/test/tools/pmatch-functionality.sh
+++ b/test/tools/pmatch-functionality.sh
@@ -17,7 +17,7 @@ if ! echo "cat" | $TOOLDIR/hfst-pmatch pmatch_endtag.pmatch > test.pmatch;
exit 1
fi
-rm test.pmatch
+rm test.pmatch test.lookups
# Jyrki's suite
if ! $srcdir/pmatch-tests.sh --log none; then
diff --git a/test/tools/pmatch-tests.sh b/test/tools/pmatch-tests.sh
index a2aa380..1d067eb 100755
--- a/test/tools/pmatch-tests.sh
+++ b/test/tools/pmatch-tests.sh
@@ -8,6 +8,19 @@ fi
. $srcdir/pmatch-tester.sh
+# Tests that currently fail are skipped.
+# The last three tests pass in some environments.
+exclude_tests="(Converting tags with replacement)\
+|(Replace semantics)\
+|(Add characters with replacement or transduction)\
+|(Transductions and replacements with EndTag)\
+|(OptCap, ToUpper, ToLower)\
+|(Named expressions in OptCap, ToUpper)\
+|(Long input lines)\
+|(Any character except in a set)\
+|(Referring to a double quote in a regexp)\
+|(Character literal escapes)\
+|(Named expressions in replace)"
testset_begin "Testing Pmatch"
diff --git a/test/tools/proc-caps-gen.strings b/test/tools/proc-caps-gen.strings
index e5b439c..8e8deda 100644
--- a/test/tools/proc-caps-gen.strings
+++ b/test/tools/proc-caps-gen.strings
@@ -12,3 +12,4 @@
^Test+n$
^Test+nlp$
^Test+np$
+^HFST+n+gen$
diff --git a/test/tools/proc-caps-out1.strings b/test/tools/proc-caps-out1.strings
index 6d73284..d5b0d91 100644
--- a/test/tools/proc-caps-out1.strings
+++ b/test/tools/proc-caps-out1.strings
@@ -3,4 +3,4 @@
^TEST/TEST+np/TEST+n/TEST+nlp$
^TeSt/Test+np/Test+n/Test+nlp$
^tesT/test+n$
-^TEst/Test+np/Test+n/Test+nlp$
+^TEst/TEST+np/TEST+n/TEST+nlp$
diff --git a/test/tools/proc-caps-out2.strings b/test/tools/proc-caps-out2.strings
index edf7f92..6964272 100644
--- a/test/tools/proc-caps-out2.strings
+++ b/test/tools/proc-caps-out2.strings
@@ -12,3 +12,4 @@ test
Test
Test
Test
+HFST's
diff --git a/test/tools/proc-caps.txt b/test/tools/proc-caps.txt
index e65f61a..becb0eb 100644
--- a/test/tools/proc-caps.txt
+++ b/test/tools/proc-caps.txt
@@ -1,17 +1,22 @@
-0 1 t t
-0 5 T T
-0 9 T t
-1 2 e e
-2 3 s s
-3 4 t t
-4 13 @_EPSILON_SYMBOL_@ +n
-5 6 e e
-6 7 s s
-7 8 t t
-8 13 @_EPSILON_SYMBOL_@ +np
-9 10 e e
-10 11 s s
-11 12 t t
-12 13 @_EPSILON_SYMBOL_@ +nlp
-13
-
+0 1 H H
+0 2 T t
+0 3 T T
+0 4 t t
+1 5 F F
+2 6 e e
+3 7 e e
+4 8 e e
+5 9 S S
+6 10 s s
+7 11 s s
+8 12 s s
+9 13 T T
+10 14 t t
+11 15 t t
+12 16 t t
+13 17 ' +n
+14 18 @0@ +nlp
+15 18 @0@ +np
+16 18 @0@ +n
+17 18 s +gen
+18
diff --git a/test/tools/symbol-harmonization-functionality.sh b/test/tools/symbol-harmonization-functionality.sh
index ea36314..7750f1d 100755
--- a/test/tools/symbol-harmonization-functionality.sh
+++ b/test/tools/symbol-harmonization-functionality.sh
@@ -207,7 +207,7 @@ for ext in .sfst .ofst .foma; do
## freely insert aka ignore
echo "[?] / a" | $TOOLDIR/hfst-regexp2fst ${FFLAG} > tmp1;
- echo "[a* ? a*]" | $TOOLDIR/hfst-regexp2fst -H ${FFLAG} > tmp2;
+ echo "[a* [?|a] a*]" | $TOOLDIR/hfst-regexp2fst -H ${FFLAG} > tmp2;
if ! ($TOOLDIR/hfst-compare -H tmp1 tmp2 > /dev/null); then
echo "freely insert (aka ignore) test #1" ${FFLAG}
exit 1;
@@ -221,7 +221,7 @@ for ext in .sfst .ofst .foma; do
fi
echo "[?|a] / [a b]" | $TOOLDIR/hfst-regexp2fst ${FFLAG} > tmp1;
- echo "[[a b]* [?|a] [a b]*]" | $TOOLDIR/hfst-regexp2fst -H ${FFLAG} > tmp2;
+ echo "[[a b]* [?|a|b] [a b]*]" | $TOOLDIR/hfst-regexp2fst -H ${FFLAG} > tmp2;
if ! ($TOOLDIR/hfst-compare -H tmp1 tmp2 > /dev/null); then
echo "freely insert (aka ignore) test #3" ${FFLAG}
exit 1;
@@ -298,4 +298,4 @@ for ext in .sfst .ofst .foma; do
done
-exit 0;
\ No newline at end of file
+rm -f tmp1 tmp2 tmp3 result
diff --git a/test/tools/tokenize-backtrack-functionality.sh b/test/tools/tokenize-backtrack-functionality.sh
new file mode 100755
index 0000000..549a8e9
--- /dev/null
+++ b/test/tools/tokenize-backtrack-functionality.sh
@@ -0,0 +1,56 @@
+#!/bin/sh
+
+TOOLDIR=../../tools/src
+
+if [ "$srcdir" = "" ]; then
+ srcdir="./";
+fi
+
+# Prerequisites:
+if ! $TOOLDIR/hfst-lexc -q < $srcdir/tokenize-backtrack.lexc > $srcdir/tokenize-backtrack-gen.hfst; then
+ echo punct backtrack gen fail
+ exit 1
+fi
+if ! $TOOLDIR/hfst-invert < $srcdir/tokenize-backtrack-gen.hfst > $srcdir/tokenize-backtrack.hfst; then
+ echo invert backtrack fail
+ exit 1
+fi
+if ! $TOOLDIR/hfst-pmatch2fst < $srcdir/tokenize-backtrack.pmscript > $srcdir/tokenize-backtrack.pmhfst; then
+ echo pmatch2fst backtrack fail
+ exit 1
+fi
+
+# Only --giella-cg supports this:
+if ! echo "busse skuvla skuvla busse Jan." | $TOOLDIR/hfst-tokenize --giella-cg $srcdir/tokenize-backtrack.pmhfst > test.strings ; then
+ echo tokenize --giella-cg fail:
+ cat test.strings
+ exit 1
+fi
+if ! diff test.strings $srcdir/tokenize-backtrack-out-giella-cg.strings ; then
+ echo diff test.strings $srcdir/tokenize-backtrack-out-giella-cg.strings
+ exit 1
+fi
+
+if ! echo "su. su" | $TOOLDIR/hfst-tokenize --giella-cg $srcdir/tokenize-backtrack.pmhfst > test.strings ; then
+ echo tokenize --giella-cg contiguous fail:
+ cat test.strings
+ exit 1
+fi
+if ! diff test.strings $srcdir/tokenize-backtrack-out-giella-cg-contiguous.strings ; then
+ echo diff test.strings $srcdir/tokenize-backtrack-out-giella-cg-contiguous.strings
+ exit 1
+fi
+
+if ! echo "njeallje logi guokte" | $TOOLDIR/hfst-tokenize --giella-cg $srcdir/tokenize-backtrack.pmhfst > test.strings ; then
+ echo tokenize --giella-cg contiguous fail:
+ cat test.strings
+ exit 1
+fi
+if ! diff test.strings $srcdir/tokenize-backtrack-out-giella-cg-spaces.strings ; then
+ echo diff test.strings $srcdir/tokenize-backtrack-out-giella-cg-spaces.strings
+ exit 1
+fi
+
+
+rm test.strings tokenize-backtrack.pmhfst tokenize-backtrack.hfst tokenize-backtrack-gen.hfst
+exit 0
diff --git a/test/tools/tokenize-backtrack-out-giella-cg-contiguous.strings b/test/tools/tokenize-backtrack-out-giella-cg-contiguous.strings
new file mode 100644
index 0000000..486271d
--- /dev/null
+++ b/test/tools/tokenize-backtrack-out-giella-cg-contiguous.strings
@@ -0,0 +1,9 @@
+"<su.>"
+ "." PUNCT <W:0> "<.>"
+ "su" Adv Abbr <W:0> "<su>"
+ "." PUNCT <W:0> "<.>"
+ "su" Prn <W:0> "<su>"
+:
+"<su>"
+ "su" Prn <W:0>
+:\n
diff --git a/test/tools/tokenize-backtrack-out-giella-cg-spaces.strings b/test/tools/tokenize-backtrack-out-giella-cg-spaces.strings
new file mode 100644
index 0000000..75ed25d
--- /dev/null
+++ b/test/tools/tokenize-backtrack-out-giella-cg-spaces.strings
@@ -0,0 +1,14 @@
+"<njeallje logi guokte>"
+ "njealljelogiguokte" Num ErrSpace <W:0>
+ "guokte" Num <W:0> "<guokte>"
+ "logi" Num <W:0> "< logi >"
+ "njeallje" Num <W:0> "<njeallje >"
+ "guokte" Num <W:0> "<guokte>"
+ "gi" N <W:0> "<gi >"
+ "lo" Num <W:0> "< lo>"
+ "njeallje" Num <W:0> "<njeallje >"
+ "guokte" Num <W:0> "<guokte>"
+ "gi" N <W:0> "< logi >"
+ "lo" Cmp <W:0>
+ "njeallje" Num <W:0> "<njeallje >"
+:\n
diff --git a/test/tools/tokenize-backtrack-out-giella-cg.strings b/test/tools/tokenize-backtrack-out-giella-cg.strings
new file mode 100644
index 0000000..df34e77
--- /dev/null
+++ b/test/tools/tokenize-backtrack-out-giella-cg.strings
@@ -0,0 +1,21 @@
+"<busse>"
+ "busse" N <W:0>
+ "busset" V <W:0>
+:
+"<skuvla>"
+ "skuvla" N <W:0>
+:
+"<skuvla busse>"
+ "skuvlabusse" N ErrSpace <W:0>
+ "busset" V <W:0> "<busse>"
+ "skuvla" N <W:0> "<skuvla >"
+ "busse" N <W:0> "<busse>"
+ "skuvla" N <W:0> "<skuvla >"
+:
+"<Jan.>"
+ "Jan." N Abbr <W:0>
+ "." PUNCT <W:0> "<.>"
+ "Jan." N Abbr <W:0> "<Jan>"
+ "." PUNCT <W:0> "<.>"
+ "Jan" N Prop <W:0> "<Jan>"
+:\n
diff --git a/test/tools/tokenize-backtrack.lexc b/test/tools/tokenize-backtrack.lexc
new file mode 100644
index 0000000..623a7d9
--- /dev/null
+++ b/test/tools/tokenize-backtrack.lexc
@@ -0,0 +1,65 @@
+Multichar_Symbols
+
++N
++Prop
++V
++Abbr
++PUNCT
+ at PMATCH_INPUT_MARK@
+ at PMATCH_BACKTRACK@
++ErrSpace
+%#
++Adv
++Num
++Prn
++Cmp
+
+LEXICON Root
+
+skuvla+N:skuvla #;
+busse+N:busse #;
+busset+V:busse #;
+< {skuvla} "@PMATCH_BACKTRACK@":0 "@PMATCH_INPUT_MARK@":" " {busse} "+N":0 "+ErrSpace":0 > # ;
+
+logi+Num:logi #;
+guokte+Num:guokte #;
+njeallje+Num:njeallje #;
+lo+Cmp#:lo GI;
+lo+Num:lo UNSPACEGI;
+! Silly example, but just to test that spaces on both sides work, as well as multiple backtracking points:
+< {njeallje} "@PMATCH_BACKTRACK@":0 "@PMATCH_INPUT_MARK@":" " 0:" " 0:" " {logi} "@PMATCH_BACKTRACK@":0 "@PMATCH_INPUT_MARK@":" " {guokte} "+Num":0 "+ErrSpace":0 > # ;
+
+Jan+N+Prop:Jan #;
+.+PUNCT:. #;
+Jan DOTABBR;
+
+su+Prn:su #;
+su+Adv+Abbr:su DOTNOTAG;
+
+
+LEXICON DOTABBR
+! This one should create a new backtrack-point before the dot, but *no* sub-reading:
+< "@PMATCH_BACKTRACK@":0 "@PMATCH_INPUT_MARK@":0 {.} "+N":0 "+Abbr":0 > # ;
+! This one should create a new backtrack-point before the dot, *as well as* a sub-reading:
+< "@PMATCH_BACKTRACK@":0 "@PMATCH_INPUT_MARK@":0 {.}:0 "+N":0 "+Abbr":0 "@PMATCH_INPUT_MARK@":0 {.} "+PUNCT":0 > # ;
+
+! Want:
+!"<Jan.>"
+! "." PUNCT "<.>"
+! "Jan." N Abbr "<Jan>"
+! "Jan." N Abbr
+! "." PUNCT "<.>"
+! "Jan" N Prop "<Jan>"
+
+LEXICON DOTNOTAG
+! As the sub-reading-line of DOTABBR, but the two input-marks are right next to each other:
+< "@PMATCH_INPUT_MARK@":0 "@PMATCH_BACKTRACK@":0 "@PMATCH_INPUT_MARK@":0 {.} "+PUNCT":0 > # ;
+! TODO: This doesn't give the sub-reading, however; should it?
+!< "@PMATCH_BACKTRACK@":0 "@PMATCH_INPUT_MARK@":0 "@PMATCH_INPUT_MARK@":0 {.} "+PUNCT":0 > # ;
+
+
+LEXICON UNSPACEGI
+ at PMATCH_INPUT_MARK@:0 GI;
+
+LEXICON GI
+gi+N:gi #;
diff --git a/test/tools/tokenize-backtrack.pmscript b/test/tools/tokenize-backtrack.pmscript
new file mode 100644
index 0000000..eaac09b
--- /dev/null
+++ b/test/tools/tokenize-backtrack.pmscript
@@ -0,0 +1,16 @@
+Define morphology @bin"tokenize-backtrack.hfst" ;
+
+Define blank Whitespace | Punct ;
+
+! Define incondword morphology & [ Punct 0:?* ] ;
+Define morphoword morphology LC([blank | #]) RC([blank | # ]);
+
+Define alphabet "a-z" | {á}|{š}|{ž}|{č}|{đ}|{ŋ}|{æ}|{ø}|{å} | "A-Z" | {Á}|{Š}|{Ž}|{Č}|{Đ}|{Ŋ}|{Æ}|{Ø}|{Å} ;
+Define alphaword alphabet+;
+Define unknownform [ [alphaword].u - [morphology].u];
+Define unknownwordEmpty unknownform:0 LC([blank | #]) RC([[blank ] | # ]);
+
+! Define token [ morphoword | unknownwordEmpty | incondword ] EndTag(token);
+Define token [ morphoword | unknownwordEmpty ] EndTag(token);
+
+Define TOP token ;
diff --git a/test/tools/tokenize-dog-in.lexc b/test/tools/tokenize-dog-in.lexc
new file mode 100644
index 0000000..a68a368
--- /dev/null
+++ b/test/tools/tokenize-dog-in.lexc
@@ -0,0 +1,32 @@
+Multichar_Symbols
+
++N
++V
++Foc
++Cmp
+ at PMATCH_INPUT_MARK@
++Du3
++Sg3
++ErrSpace
+%#
+
+LEXICON Root
+DOG;
+cat+N+Cmp%#:cat DOG;
+be+V:be SPACEDOG;
+leat+V+Du3:lea UNSPACEDOG;
+leat+V+Sg3:leaba ENDLEX;
+bedog+V:be% dog ENDLEX;
+
+LEXICON ENDLEX
+#;
+
+LEXICON DOG
+dog+N:dog ENDLEX;
+ba+Foc:ba ENDLEX;
+
+LEXICON SPACEDOG
+ at PMATCH_INPUT_MARK@:% DOG;
+
+LEXICON UNSPACEDOG
+ at PMATCH_INPUT_MARK@:0 DOG;
diff --git a/test/tools/tokenize-dog-out-cg.strings b/test/tools/tokenize-dog-out-cg.strings
new file mode 100644
index 0000000..72e4d4e
--- /dev/null
+++ b/test/tools/tokenize-dog-out-cg.strings
@@ -0,0 +1,13 @@
+"<test>"
+
+
+"<dog>"
+ "dog"+N
+
+"<be dog>"
+ be+Vdog+N
+ bedog+V
+
+"<catdog>"
+ cat+N+Cmp#dog+N
+
diff --git a/test/tools/tokenize-dog-out-giella-cg-flushing.strings b/test/tools/tokenize-dog-out-giella-cg-flushing.strings
new file mode 100644
index 0000000..4bac074
--- /dev/null
+++ b/test/tools/tokenize-dog-out-giella-cg-flushing.strings
@@ -0,0 +1,13 @@
+"<dog>"
+ "dog" N <W:0>
+:[\\n<\\>]
+"<cat>"
+ "cat" ?
+: !
+"<and>"
+ "and" ?
+: \n
+"<dogs>"
+ "dogs" ?
+:[][\n]
+<STREAMCMD:FLUSH>
diff --git a/test/tools/tokenize-dog-out-giella-cg-superblank.strings b/test/tools/tokenize-dog-out-giella-cg-superblank.strings
new file mode 100644
index 0000000..7358c0e
--- /dev/null
+++ b/test/tools/tokenize-dog-out-giella-cg-superblank.strings
@@ -0,0 +1,12 @@
+"<dog>"
+ "dog" N <W:0>
+:[\\n<\\>]
+"<cat>"
+ "cat" ?
+: !
+"<and>"
+ "and" ?
+: \n
+"<dogs>"
+ "dogs" ?
+:[][\n]
diff --git a/test/tools/tokenize-dog-out-giella-cg.strings b/test/tools/tokenize-dog-out-giella-cg.strings
new file mode 100644
index 0000000..d10de62
--- /dev/null
+++ b/test/tools/tokenize-dog-out-giella-cg.strings
@@ -0,0 +1,15 @@
+"<test>"
+ "test" ?
+:
+"<dog>"
+ "dog" N <W:0>
+:
+"<be dog>"
+ "dog" N <W:0> "<dog>"
+ "be" V <W:0> "<be >"
+ "bedog" V <W:0>
+:
+"<catdog>"
+ "dog" N <W:0>
+ "cat" N Cmp <W:0>
+: собака\n
diff --git a/test/tools/tokenize-dog-out-xerox.strings b/test/tools/tokenize-dog-out-xerox.strings
new file mode 100644
index 0000000..21e2485
--- /dev/null
+++ b/test/tools/tokenize-dog-out-xerox.strings
@@ -0,0 +1,9 @@
+test
+
+dog dog+N
+
+be dog be+Vdog+N
+be dog bedog+V
+
+catdog cat+N+Cmp#dog+N
+
diff --git a/test/tools/tokenize-dog-out.strings b/test/tools/tokenize-dog-out.strings
new file mode 100644
index 0000000..3436988
--- /dev/null
+++ b/test/tools/tokenize-dog-out.strings
@@ -0,0 +1,4 @@
+test
+dog
+be dog
+catdog
diff --git a/test/tools/tokenize-dog.pmscript b/test/tools/tokenize-dog.pmscript
new file mode 100644
index 0000000..3032dce
--- /dev/null
+++ b/test/tools/tokenize-dog.pmscript
@@ -0,0 +1,16 @@
+Define morphology @bin"tokenize-dog.hfst" ;
+
+Define blank Whitespace | Punct ;
+
+! Define incondword morphology & [ Punct 0:?* ] ;
+Define morphoword morphology LC([blank | #]) RC([blank | # ]);
+
+Define alphabet "a-z" | {á}|{š}|{ž}|{č}|{đ}|{ŋ}|{æ}|{ø}|{å} | "A-Z" | {Á}|{Š}|{Ž}|{Č}|{Đ}|{Ŋ}|{Æ}|{Ø}|{Å} ;
+Define alphaword alphabet+;
+Define unknownform [ [alphaword].u - [morphology].u];
+Define unknownwordEmpty unknownform:0 LC([blank | #]) RC([[blank ] | # ]);
+
+! Define token [ morphoword | unknownwordEmpty | incondword ] EndTag(token);
+Define token [ morphoword | unknownwordEmpty ] EndTag(token);
+
+Define TOP token ;
diff --git a/test/tools/tokenize-flushing-functionality.sh b/test/tools/tokenize-flushing-functionality.sh
new file mode 100755
index 0000000..cd7c0bb
--- /dev/null
+++ b/test/tools/tokenize-flushing-functionality.sh
@@ -0,0 +1,71 @@
+#!/bin/sh
+
+TOOLDIR=../../tools/src
+
+if [ "$srcdir" = "" ]; then
+ srcdir="./";
+fi
+
+if ! command -V timeout >/dev/null 2>/dev/null; then
+ # require /usr/bin/timeout, since it could hang if there's a bug
+ exit 77
+fi
+
+# Prerequisites
+if ! $TOOLDIR/hfst-lexc -q < $srcdir/tokenize-dog-in.lexc > $srcdir/tokenize-dog-gen.hfst; then
+ echo lexc dog fail
+ exit 1
+fi
+if ! $TOOLDIR/hfst-invert < $srcdir/tokenize-dog-gen.hfst > $srcdir/tokenize-dog.hfst; then
+ echo invert dog fail
+ exit 1
+fi
+if ! $TOOLDIR/hfst-pmatch2fst < $srcdir/tokenize-dog.pmscript > $srcdir/tokenize-dog.pmhfst; then
+ echo pmatch2fst tokenize-dog fail
+ exit 1
+fi
+
+tmpd=$(mktemp -d -t hfst-tokenize-tests.XXXXXXXX)
+to="${tmpd}/to"
+from="${tmpd}/from"
+mkfifo "${to}" "${from}"
+
+# Server:
+$TOOLDIR/hfst-tokenize --giella-cg --superblanks $srcdir/tokenize-dog.pmhfst <"${to}" >"${from}" &
+pid=$!
+trap 'kill $pid; wait $pid 2>/dev/null; rm -rf "${tmpd}"' EXIT
+
+
+# Client:
+exec 3>"${to}"
+exec 4<"${from}"
+for _ in test a couple times; do
+ timeout 1 printf 'dog[\\\n<\\\\>]cat !and \ndogs[][\n]\0' >&3
+ case $? in
+ 0) : ;;
+ 124) echo tokenize --giella-cg flushing write timed out
+ exit 1
+ ;;
+ *) echo tokenize --giella-cg flushing write failed with "$?"
+ exit 1
+ ;;
+ esac
+ timeout 1 sed '/^<STREAMCMD:FLUSH>$/{q}' <&4 >test.strings
+ case $? in
+ 0) : ;;
+ 124) echo tokenize --giella-cg flushing read timed out
+ exit 1
+ ;;
+ *) echo tokenize --giella-cg flushing read failed with "$?", output is:
+ cat test.strings
+ exit 1
+ ;;
+ esac
+ if ! diff test.strings $srcdir/tokenize-dog-out-giella-cg-flushing.strings ; then
+ echo diff test.strings $srcdir/tokenize-dog-out-giella-cg-flushing.strings
+ exit 1
+ fi
+done
+
+rm test.strings tokenize-dog.pmhfst tokenize-dog.hfst tokenize-dog-gen.hfst
+exit 0
diff --git a/test/tools/tokenize-functionality.sh b/test/tools/tokenize-functionality.sh
new file mode 100755
index 0000000..2107dff
--- /dev/null
+++ b/test/tools/tokenize-functionality.sh
@@ -0,0 +1,80 @@
+#!/bin/sh
+
+TOOLDIR=../../tools/src
+
+if [ "$srcdir" = "" ]; then
+ srcdir="./";
+fi
+
+# Prerequisites
+if ! $TOOLDIR/hfst-lexc -q < $srcdir/tokenize-dog-in.lexc > $srcdir/tokenize-dog-gen.hfst; then
+ echo lexc dog fail
+ exit 1
+fi
+if ! $TOOLDIR/hfst-invert < $srcdir/tokenize-dog-gen.hfst > $srcdir/tokenize-dog.hfst; then
+ echo invert dog fail
+ exit 1
+fi
+if ! $TOOLDIR/hfst-pmatch2fst < $srcdir/tokenize-dog.pmscript > $srcdir/tokenize-dog.pmhfst; then
+ echo pmatch2fst tokenize-dog fail
+ exit 1
+fi
+
+# basic lookup
+if ! echo "test dog be dog catdog" | $TOOLDIR/hfst-tokenize $srcdir/tokenize-dog.pmhfst > test.strings ; then
+ echo tokenize fail:
+ cat test.strings
+ exit 1
+fi
+if ! diff test.strings $srcdir/tokenize-dog-out.strings ; then
+ echo diff test.strings $srcdir/tokenize-dog-out.strings
+ exit 1
+fi
+
+# --cg
+if ! echo "test dog be dog catdog" | $TOOLDIR/hfst-tokenize --cg $srcdir/tokenize-dog.pmhfst > test.strings ; then
+ echo tokenize --cg fail:
+ cat test.strings
+ exit 1
+fi
+if ! diff test.strings $srcdir/tokenize-dog-out-cg.strings ; then
+ echo diff test.strings $srcdir/tokenize-dog-out-cg.strings
+ exit 1
+fi
+
+# --giella-cg
+if ! echo "test dog be dog catdog собака" | $TOOLDIR/hfst-tokenize --giella-cg $srcdir/tokenize-dog.pmhfst > test.strings ; then
+ echo tokenize --giella-cg fail:
+ cat test.strings
+ exit 1
+fi
+if ! diff test.strings $srcdir/tokenize-dog-out-giella-cg.strings ; then
+ echo diff test.strings $srcdir/tokenize-dog-out-giella-cg.strings
+ exit 1
+fi
+
+# --xerox
+if ! echo "test dog be dog catdog" | $TOOLDIR/hfst-tokenize --xerox $srcdir/tokenize-dog.pmhfst > test.strings ; then
+ echo tokenize --xerox fail:
+ cat test.strings
+ exit 1
+fi
+if ! diff test.strings $srcdir/tokenize-dog-out-xerox.strings ; then
+ echo diff test.strings $srcdir/tokenize-dog-out-xerox.strings
+ exit 1
+fi
+
+
+# --giella-cg superblanks
+if ! printf 'dog[\\\n<\\\\>]cat !and \ndogs[][\n]' | $TOOLDIR/hfst-tokenize --giella-cg --superblanks $srcdir/tokenize-dog.pmhfst > test.strings ; then
+ echo tokenize --giella-cg --superblanks superblank fail:
+ cat test.strings
+ exit 1
+fi
+if ! diff test.strings $srcdir/tokenize-dog-out-giella-cg-superblank.strings ; then
+ echo diff test.strings $srcdir/tokenize-dog-out-giella-cg-superblank.strings
+ exit 1
+fi
+
+rm test.strings tokenize-dog.pmhfst tokenize-dog.hfst tokenize-dog-gen.hfst
+exit 0
diff --git a/test/tools/warn.one-sided-flags.lexc.flag.result b/test/tools/warn.one-sided-flags.lexc.flag.result
deleted file mode 100644
index fb4d840..0000000
Binary files a/test/tools/warn.one-sided-flags.lexc.flag.result and /dev/null differ
diff --git a/test/tools/warn.one-sided-flags.lexc.flag.result.prolog b/test/tools/warn.one-sided-flags.lexc.flag.result.prolog
new file mode 100644
index 0000000..2aeb3ca
--- /dev/null
+++ b/test/tools/warn.one-sided-flags.lexc.flag.result.prolog
@@ -0,0 +1,11 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@").
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@").
+arc(NO_NAME_1, 2, 7, "@P.LEXNAME.#@").
+arc(NO_NAME_1, 2, 3, "@U.FOO.ON@":"b").
+arc(NO_NAME_1, 3, 4, "f":"a").
+arc(NO_NAME_1, 4, 5, "o":"r").
+arc(NO_NAME_1, 5, 6, "o":"0").
+arc(NO_NAME_1, 6, 7, "@P.LEXNAME.#@").
+arc(NO_NAME_1, 7, 8, "@R.LEXNAME.#@").
+final(NO_NAME_1, 8).
diff --git a/test/tools/warn.one-sided-flags.lexc.result b/test/tools/warn.one-sided-flags.lexc.result
deleted file mode 100644
index 4250076..0000000
Binary files a/test/tools/warn.one-sided-flags.lexc.result and /dev/null differ
diff --git a/test/tools/warn.one-sided-flags.lexc.result.prolog b/test/tools/warn.one-sided-flags.lexc.result.prolog
new file mode 100644
index 0000000..640cf4a
--- /dev/null
+++ b/test/tools/warn.one-sided-flags.lexc.result.prolog
@@ -0,0 +1,7 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@U.FOO.ON@":"b").
+arc(NO_NAME_1, 1, 2, "f":"a").
+arc(NO_NAME_1, 2, 3, "o":"r").
+arc(NO_NAME_1, 3, 4, "o":"0").
+final(NO_NAME_1, 0).
+final(NO_NAME_1, 4).
diff --git a/test/tools/warn.sublexicon-mentioned-but-not-defined.lexc.flag.result b/test/tools/warn.sublexicon-mentioned-but-not-defined.lexc.flag.result
deleted file mode 100644
index 88501a5..0000000
--- a/test/tools/warn.sublexicon-mentioned-but-not-defined.lexc.flag.result
+++ /dev/null
@@ -1,19 +0,0 @@
-##foma-net 1.0##
-##props##
-1 4 5 6 1 1 1 2 2 1 1 2
-##sigma##
-0 @_EPSILON_SYMBOL_@
-1 @_UNKNOWN_SYMBOL_@
-2 @_IDENTITY_SYMBOL_@
-3 @P.LEXNAME.#@
-4 @P.LEXNAME.Root@
-5 @R.LEXNAME.#@
-6 @R.LEXNAME.Root@
-##states##
-0 4 1 0
-1 6 2 0
-2 3 3 0
-3 5 4 0
-4 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/warn.sublexicon-mentioned-but-not-defined.lexc.flag.result.prolog b/test/tools/warn.sublexicon-mentioned-but-not-defined.lexc.flag.result.prolog
new file mode 100644
index 0000000..e9fe2e1
--- /dev/null
+++ b/test/tools/warn.sublexicon-mentioned-but-not-defined.lexc.flag.result.prolog
@@ -0,0 +1,6 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@").
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@").
+arc(NO_NAME_1, 2, 3, "@P.LEXNAME.#@").
+arc(NO_NAME_1, 3, 4, "@R.LEXNAME.#@").
+final(NO_NAME_1, 4).
diff --git a/test/tools/warn.sublexicon-mentioned-but-not-defined.lexc.result b/test/tools/warn.sublexicon-mentioned-but-not-defined.lexc.result
deleted file mode 100644
index 4e502d0..0000000
--- a/test/tools/warn.sublexicon-mentioned-but-not-defined.lexc.result
+++ /dev/null
@@ -1,11 +0,0 @@
-##foma-net 1.0##
-##props##
-1 0 1 2 1 1 1 2 2 1 1 2
-##sigma##
-0 @_EPSILON_SYMBOL_@
-1 @_UNKNOWN_SYMBOL_@
-2 @_IDENTITY_SYMBOL_@
-##states##
-0 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/warn.sublexicon-mentioned-but-not-defined.lexc.result.prolog b/test/tools/warn.sublexicon-mentioned-but-not-defined.lexc.result.prolog
new file mode 100644
index 0000000..28de08b
--- /dev/null
+++ b/test/tools/warn.sublexicon-mentioned-but-not-defined.lexc.result.prolog
@@ -0,0 +1,2 @@
+network(NO_NAME_1).
+final(NO_NAME_1, 0).
diff --git a/test/tools/xre.any-variations.lexc.flag.result b/test/tools/xre.any-variations.lexc.flag.result
deleted file mode 100644
index d2761a6..0000000
Binary files a/test/tools/xre.any-variations.lexc.flag.result and /dev/null differ
diff --git a/test/tools/xre.any-variations.lexc.flag.result.prolog b/test/tools/xre.any-variations.lexc.flag.result.prolog
new file mode 100644
index 0000000..24f0267
--- /dev/null
+++ b/test/tools/xre.any-variations.lexc.flag.result.prolog
@@ -0,0 +1,87 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "c", 0.000000).
+arc(NO_NAME_1, 2, 4, "a", 0.000000).
+arc(NO_NAME_1, 2, 5, "x", 0.000000).
+arc(NO_NAME_1, 2, 6, "b", 0.000000).
+arc(NO_NAME_1, 2, 7, "z", 0.000000).
+arc(NO_NAME_1, 3, 8, "x", 0.000000).
+arc(NO_NAME_1, 3, 9, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 3, 8, "z":"x", 0.000000).
+arc(NO_NAME_1, 3, 8, "y":"x", 0.000000).
+arc(NO_NAME_1, 3, 8, "b":"x", 0.000000).
+arc(NO_NAME_1, 3, 8, "a":"x", 0.000000).
+arc(NO_NAME_1, 3, 8, "?":"x", 0.000000).
+arc(NO_NAME_1, 3, 8, "c":"x", 0.000000).
+arc(NO_NAME_1, 4, 8, "c", 0.000000).
+arc(NO_NAME_1, 4, 8, "a", 0.000000).
+arc(NO_NAME_1, 4, 8, "x", 0.000000).
+arc(NO_NAME_1, 4, 8, "b", 0.000000).
+arc(NO_NAME_1, 4, 8, "z", 0.000000).
+arc(NO_NAME_1, 4, 9, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 4, 8, "y", 0.000000).
+arc(NO_NAME_1, 4, 8, "?", 0.000000).
+arc(NO_NAME_1, 5, 10, "y", 0.000000).
+arc(NO_NAME_1, 6, 8, "x", 0.000000).
+arc(NO_NAME_1, 6, 9, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 6, 8, "x":"c", 0.000000).
+arc(NO_NAME_1, 6, 8, "x":"b", 0.000000).
+arc(NO_NAME_1, 6, 8, "x":"?", 0.000000).
+arc(NO_NAME_1, 6, 8, "x":"a", 0.000000).
+arc(NO_NAME_1, 6, 8, "x":"y", 0.000000).
+arc(NO_NAME_1, 6, 8, "x":"z", 0.000000).
+arc(NO_NAME_1, 7, 8, "c", 0.000000).
+arc(NO_NAME_1, 7, 8, "a", 0.000000).
+arc(NO_NAME_1, 7, 8, "x", 0.000000).
+arc(NO_NAME_1, 7, 8, "b", 0.000000).
+arc(NO_NAME_1, 7, 8, "z", 0.000000).
+arc(NO_NAME_1, 7, 8, "?":"z", 0.000000).
+arc(NO_NAME_1, 7, 8, "z":"?", 0.000000).
+arc(NO_NAME_1, 7, 8, "?":"?", 0.000000).
+arc(NO_NAME_1, 7, 8, "a":"z", 0.000000).
+arc(NO_NAME_1, 7, 8, "b":"z", 0.000000).
+arc(NO_NAME_1, 7, 8, "c":"z", 0.000000).
+arc(NO_NAME_1, 7, 8, "y":"z", 0.000000).
+arc(NO_NAME_1, 7, 8, "z":"a", 0.000000).
+arc(NO_NAME_1, 7, 8, "z":"b", 0.000000).
+arc(NO_NAME_1, 7, 8, "z":"c", 0.000000).
+arc(NO_NAME_1, 7, 8, "z":"y", 0.000000).
+arc(NO_NAME_1, 7, 8, "a":"?", 0.000000).
+arc(NO_NAME_1, 7, 8, "b":"?", 0.000000).
+arc(NO_NAME_1, 7, 8, "c":"?", 0.000000).
+arc(NO_NAME_1, 7, 8, "y":"?", 0.000000).
+arc(NO_NAME_1, 7, 8, "?":"a", 0.000000).
+arc(NO_NAME_1, 7, 8, "?":"b", 0.000000).
+arc(NO_NAME_1, 7, 8, "?":"c", 0.000000).
+arc(NO_NAME_1, 7, 8, "?":"y", 0.000000).
+arc(NO_NAME_1, 7, 8, "b":"a", 0.000000).
+arc(NO_NAME_1, 7, 8, "c":"a", 0.000000).
+arc(NO_NAME_1, 7, 8, "y":"a", 0.000000).
+arc(NO_NAME_1, 7, 8, "a":"b", 0.000000).
+arc(NO_NAME_1, 7, 8, "c":"b", 0.000000).
+arc(NO_NAME_1, 7, 8, "y":"b", 0.000000).
+arc(NO_NAME_1, 7, 8, "a":"c", 0.000000).
+arc(NO_NAME_1, 7, 8, "b":"c", 0.000000).
+arc(NO_NAME_1, 7, 8, "y":"c", 0.000000).
+arc(NO_NAME_1, 7, 8, "a":"y", 0.000000).
+arc(NO_NAME_1, 7, 8, "b":"y", 0.000000).
+arc(NO_NAME_1, 7, 8, "c":"y", 0.000000).
+arc(NO_NAME_1, 7, 8, "x":"c", 0.000000).
+arc(NO_NAME_1, 7, 8, "x":"b", 0.000000).
+arc(NO_NAME_1, 7, 8, "x":"?", 0.000000).
+arc(NO_NAME_1, 7, 8, "x":"a", 0.000000).
+arc(NO_NAME_1, 7, 8, "x":"y", 0.000000).
+arc(NO_NAME_1, 7, 8, "x":"z", 0.000000).
+arc(NO_NAME_1, 7, 8, "y", 0.000000).
+arc(NO_NAME_1, 7, 8, "?", 0.000000).
+arc(NO_NAME_1, 7, 8, "z":"x", 0.000000).
+arc(NO_NAME_1, 7, 8, "y":"x", 0.000000).
+arc(NO_NAME_1, 7, 8, "b":"x", 0.000000).
+arc(NO_NAME_1, 7, 8, "a":"x", 0.000000).
+arc(NO_NAME_1, 7, 8, "?":"x", 0.000000).
+arc(NO_NAME_1, 7, 8, "c":"x", 0.000000).
+arc(NO_NAME_1, 8, 9, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 9, 11, "@R.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 10, 8, "z", 0.000000).
+final(NO_NAME_1, 11, 0.000000).
diff --git a/test/tools/xre.any-variations.lexc.result b/test/tools/xre.any-variations.lexc.result
deleted file mode 100644
index 47eff0a..0000000
Binary files a/test/tools/xre.any-variations.lexc.result and /dev/null differ
diff --git a/test/tools/xre.any-variations.lexc.result.prolog b/test/tools/xre.any-variations.lexc.result.prolog
new file mode 100644
index 0000000..19b6078
--- /dev/null
+++ b/test/tools/xre.any-variations.lexc.result.prolog
@@ -0,0 +1,83 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "c", 0.000000).
+arc(NO_NAME_1, 0, 2, "a", 0.000000).
+arc(NO_NAME_1, 0, 3, "x", 0.000000).
+arc(NO_NAME_1, 0, 4, "b", 0.000000).
+arc(NO_NAME_1, 0, 5, "z", 0.000000).
+arc(NO_NAME_1, 1, 6, "x", 0.000000).
+arc(NO_NAME_1, 1, 6, "z":"x", 0.000000).
+arc(NO_NAME_1, 1, 6, "y":"x", 0.000000).
+arc(NO_NAME_1, 1, 6, "b":"x", 0.000000).
+arc(NO_NAME_1, 1, 6, "a":"x", 0.000000).
+arc(NO_NAME_1, 1, 6, "?":"x", 0.000000).
+arc(NO_NAME_1, 1, 6, "c":"x", 0.000000).
+arc(NO_NAME_1, 2, 6, "c", 0.000000).
+arc(NO_NAME_1, 2, 6, "a", 0.000000).
+arc(NO_NAME_1, 2, 6, "x", 0.000000).
+arc(NO_NAME_1, 2, 6, "b", 0.000000).
+arc(NO_NAME_1, 2, 6, "z", 0.000000).
+arc(NO_NAME_1, 2, 6, "y", 0.000000).
+arc(NO_NAME_1, 2, 6, "?", 0.000000).
+arc(NO_NAME_1, 3, 7, "y", 0.000000).
+arc(NO_NAME_1, 4, 6, "x", 0.000000).
+arc(NO_NAME_1, 4, 6, "x":"c", 0.000000).
+arc(NO_NAME_1, 4, 6, "x":"b", 0.000000).
+arc(NO_NAME_1, 4, 6, "x":"?", 0.000000).
+arc(NO_NAME_1, 4, 6, "x":"a", 0.000000).
+arc(NO_NAME_1, 4, 6, "x":"y", 0.000000).
+arc(NO_NAME_1, 4, 6, "x":"z", 0.000000).
+arc(NO_NAME_1, 5, 6, "c", 0.000000).
+arc(NO_NAME_1, 5, 6, "a", 0.000000).
+arc(NO_NAME_1, 5, 6, "x", 0.000000).
+arc(NO_NAME_1, 5, 6, "b", 0.000000).
+arc(NO_NAME_1, 5, 6, "z", 0.000000).
+arc(NO_NAME_1, 5, 6, "?":"z", 0.000000).
+arc(NO_NAME_1, 5, 6, "z":"?", 0.000000).
+arc(NO_NAME_1, 5, 6, "?":"?", 0.000000).
+arc(NO_NAME_1, 5, 6, "a":"z", 0.000000).
+arc(NO_NAME_1, 5, 6, "b":"z", 0.000000).
+arc(NO_NAME_1, 5, 6, "c":"z", 0.000000).
+arc(NO_NAME_1, 5, 6, "y":"z", 0.000000).
+arc(NO_NAME_1, 5, 6, "z":"a", 0.000000).
+arc(NO_NAME_1, 5, 6, "z":"b", 0.000000).
+arc(NO_NAME_1, 5, 6, "z":"c", 0.000000).
+arc(NO_NAME_1, 5, 6, "z":"y", 0.000000).
+arc(NO_NAME_1, 5, 6, "a":"?", 0.000000).
+arc(NO_NAME_1, 5, 6, "b":"?", 0.000000).
+arc(NO_NAME_1, 5, 6, "c":"?", 0.000000).
+arc(NO_NAME_1, 5, 6, "y":"?", 0.000000).
+arc(NO_NAME_1, 5, 6, "?":"a", 0.000000).
+arc(NO_NAME_1, 5, 6, "?":"b", 0.000000).
+arc(NO_NAME_1, 5, 6, "?":"c", 0.000000).
+arc(NO_NAME_1, 5, 6, "?":"y", 0.000000).
+arc(NO_NAME_1, 5, 6, "b":"a", 0.000000).
+arc(NO_NAME_1, 5, 6, "c":"a", 0.000000).
+arc(NO_NAME_1, 5, 6, "y":"a", 0.000000).
+arc(NO_NAME_1, 5, 6, "a":"b", 0.000000).
+arc(NO_NAME_1, 5, 6, "c":"b", 0.000000).
+arc(NO_NAME_1, 5, 6, "y":"b", 0.000000).
+arc(NO_NAME_1, 5, 6, "a":"c", 0.000000).
+arc(NO_NAME_1, 5, 6, "b":"c", 0.000000).
+arc(NO_NAME_1, 5, 6, "y":"c", 0.000000).
+arc(NO_NAME_1, 5, 6, "a":"y", 0.000000).
+arc(NO_NAME_1, 5, 6, "b":"y", 0.000000).
+arc(NO_NAME_1, 5, 6, "c":"y", 0.000000).
+arc(NO_NAME_1, 5, 6, "x":"c", 0.000000).
+arc(NO_NAME_1, 5, 6, "x":"b", 0.000000).
+arc(NO_NAME_1, 5, 6, "x":"?", 0.000000).
+arc(NO_NAME_1, 5, 6, "x":"a", 0.000000).
+arc(NO_NAME_1, 5, 6, "x":"y", 0.000000).
+arc(NO_NAME_1, 5, 6, "x":"z", 0.000000).
+arc(NO_NAME_1, 5, 6, "y", 0.000000).
+arc(NO_NAME_1, 5, 6, "?", 0.000000).
+arc(NO_NAME_1, 5, 6, "z":"x", 0.000000).
+arc(NO_NAME_1, 5, 6, "y":"x", 0.000000).
+arc(NO_NAME_1, 5, 6, "b":"x", 0.000000).
+arc(NO_NAME_1, 5, 6, "a":"x", 0.000000).
+arc(NO_NAME_1, 5, 6, "?":"x", 0.000000).
+arc(NO_NAME_1, 5, 6, "c":"x", 0.000000).
+arc(NO_NAME_1, 7, 6, "z", 0.000000).
+final(NO_NAME_1, 1, 0.000000).
+final(NO_NAME_1, 2, 0.000000).
+final(NO_NAME_1, 4, 0.000000).
+final(NO_NAME_1, 6, 0.000000).
diff --git a/test/tools/xre.automatic-multichar-symbols.lexc.flag.result b/test/tools/xre.automatic-multichar-symbols.lexc.flag.result
deleted file mode 100644
index 530eaa3..0000000
Binary files a/test/tools/xre.automatic-multichar-symbols.lexc.flag.result and /dev/null differ
diff --git a/test/tools/xre.automatic-multichar-symbols.lexc.flag.result.prolog b/test/tools/xre.automatic-multichar-symbols.lexc.flag.result.prolog
new file mode 100644
index 0000000..a14c77d
--- /dev/null
+++ b/test/tools/xre.automatic-multichar-symbols.lexc.flag.result.prolog
@@ -0,0 +1,41 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "1", 0.000000).
+arc(NO_NAME_1, 2, 3, "2", 0.000000).
+arc(NO_NAME_1, 2, 3, "3", 0.000000).
+arc(NO_NAME_1, 2, 3, "4", 0.000000).
+arc(NO_NAME_1, 2, 3, "5", 0.000000).
+arc(NO_NAME_1, 2, 3, "6", 0.000000).
+arc(NO_NAME_1, 2, 3, "7", 0.000000).
+arc(NO_NAME_1, 2, 3, "8", 0.000000).
+arc(NO_NAME_1, 2, 3, "9", 0.000000).
+arc(NO_NAME_1, 2, 3, "%0", 0.000000).
+arc(NO_NAME_1, 2, 4, "d", 0.000000).
+arc(NO_NAME_1, 2, 5, "c", 0.000000).
+arc(NO_NAME_1, 2, 5, "r", 0.000000).
+arc(NO_NAME_1, 3, 3, "1", 0.000000).
+arc(NO_NAME_1, 3, 3, "2", 0.000000).
+arc(NO_NAME_1, 3, 3, "3", 0.000000).
+arc(NO_NAME_1, 3, 3, "4", 0.000000).
+arc(NO_NAME_1, 3, 3, "5", 0.000000).
+arc(NO_NAME_1, 3, 3, "6", 0.000000).
+arc(NO_NAME_1, 3, 3, "7", 0.000000).
+arc(NO_NAME_1, 3, 3, "8", 0.000000).
+arc(NO_NAME_1, 3, 3, "9", 0.000000).
+arc(NO_NAME_1, 3, 3, "%0", 0.000000).
+arc(NO_NAME_1, 3, 16, "+Int":"0", 0.000000).
+arc(NO_NAME_1, 4, 6, "o", 0.000000).
+arc(NO_NAME_1, 5, 7, "a", 0.000000).
+arc(NO_NAME_1, 6, 9, "g", 0.000000).
+arc(NO_NAME_1, 7, 9, "t", 0.000000).
+arc(NO_NAME_1, 8, 10, "@R.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 9, 11, "@P.LEXNAME.N@", 0.000000).
+arc(NO_NAME_1, 11, 12, "@R.LEXNAME.N@", 0.000000).
+arc(NO_NAME_1, 12, 13, "+Noun":"0", 0.000000).
+arc(NO_NAME_1, 13, 14, "@P.LEXNAME.Num@", 0.000000).
+arc(NO_NAME_1, 14, 15, "@R.LEXNAME.Num@", 0.000000).
+arc(NO_NAME_1, 15, 16, "+Sg":"0", 0.000000).
+arc(NO_NAME_1, 15, 16, "+Pl":"s", 0.000000).
+arc(NO_NAME_1, 16, 8, "@P.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 10, 0.000000).
diff --git a/test/tools/xre.automatic-multichar-symbols.lexc.result b/test/tools/xre.automatic-multichar-symbols.lexc.result
deleted file mode 100644
index 52114c0..0000000
--- a/test/tools/xre.automatic-multichar-symbols.lexc.result
+++ /dev/null
@@ -1,62 +0,0 @@
-##foma-net 1.0##
-##props##
-2 31 9 33 1 -1 1 1 1 1 0 2 1FB91813
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 +Int
-4 +Noun
-5 +Pl
-6 +Sg
-7 0
-8 1
-9 2
-10 3
-11 4
-12 5
-13 6
-14 7
-15 8
-16 9
-17 a
-18 c
-19 d
-20 g
-21 o
-22 r
-23 s
-24 t
-##states##
-0 22 7 0
-18 7
-19 3
-16 1
-15 1
-14 1
-13 1
-12 1
-11 1
-10 1
-9 1
-8 1
-7 1
-1 3 0 2 0
-16 1
-15 1
-14 1
-13 1
-12 1
-11 1
-10 1
-9 1
-8 1
-7 1
-2 -1 -1 1
-3 21 4 0
-4 20 5 0
-5 4 0 6 0
-6 5 23 2 0
-6 0 2
-7 17 8 0
-8 24 5 0
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/xre.automatic-multichar-symbols.lexc.result.prolog b/test/tools/xre.automatic-multichar-symbols.lexc.result.prolog
new file mode 100644
index 0000000..dc7c92c
--- /dev/null
+++ b/test/tools/xre.automatic-multichar-symbols.lexc.result.prolog
@@ -0,0 +1,33 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 7, "r").
+arc(NO_NAME_1, 0, 7, "c").
+arc(NO_NAME_1, 0, 3, "d").
+arc(NO_NAME_1, 0, 1, "9").
+arc(NO_NAME_1, 0, 1, "8").
+arc(NO_NAME_1, 0, 1, "7").
+arc(NO_NAME_1, 0, 1, "6").
+arc(NO_NAME_1, 0, 1, "5").
+arc(NO_NAME_1, 0, 1, "4").
+arc(NO_NAME_1, 0, 1, "3").
+arc(NO_NAME_1, 0, 1, "2").
+arc(NO_NAME_1, 0, 1, "1").
+arc(NO_NAME_1, 0, 1, "%0").
+arc(NO_NAME_1, 1, 2, "+Int":"0").
+arc(NO_NAME_1, 1, 1, "9").
+arc(NO_NAME_1, 1, 1, "8").
+arc(NO_NAME_1, 1, 1, "7").
+arc(NO_NAME_1, 1, 1, "6").
+arc(NO_NAME_1, 1, 1, "5").
+arc(NO_NAME_1, 1, 1, "4").
+arc(NO_NAME_1, 1, 1, "3").
+arc(NO_NAME_1, 1, 1, "2").
+arc(NO_NAME_1, 1, 1, "1").
+arc(NO_NAME_1, 1, 1, "%0").
+arc(NO_NAME_1, 3, 4, "o").
+arc(NO_NAME_1, 4, 5, "g").
+arc(NO_NAME_1, 5, 6, "+Noun":"0").
+arc(NO_NAME_1, 6, 2, "+Pl":"s").
+arc(NO_NAME_1, 6, 2, "+Sg":"0").
+arc(NO_NAME_1, 7, 8, "a").
+arc(NO_NAME_1, 8, 5, "t").
+final(NO_NAME_1, 2).
diff --git a/test/tools/xre.basic.lexc.flag.result b/test/tools/xre.basic.lexc.flag.result
deleted file mode 100644
index 9e0f7db..0000000
Binary files a/test/tools/xre.basic.lexc.flag.result and /dev/null differ
diff --git a/test/tools/xre.basic.lexc.flag.result.prolog b/test/tools/xre.basic.lexc.flag.result.prolog
new file mode 100644
index 0000000..15a4b41
--- /dev/null
+++ b/test/tools/xre.basic.lexc.flag.result.prolog
@@ -0,0 +1,22 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "q", 0.000000).
+arc(NO_NAME_1, 2, 4, "b", 0.000000).
+arc(NO_NAME_1, 2, 5, "f", 0.000000).
+arc(NO_NAME_1, 3, 6, "u", 0.000000).
+arc(NO_NAME_1, 4, 7, "a", 0.000000).
+arc(NO_NAME_1, 5, 8, "i":"o", 0.000000).
+arc(NO_NAME_1, 5, 9, "o", 0.000000).
+arc(NO_NAME_1, 6, 10, "u", 0.000000).
+arc(NO_NAME_1, 7, 11, "z", 0.000000).
+arc(NO_NAME_1, 7, 11, "r", 0.000000).
+arc(NO_NAME_1, 8, 12, "0":"u", 0.000000).
+arc(NO_NAME_1, 9, 11, "o", 0.000000).
+arc(NO_NAME_1, 10, 11, "x", 0.000000).
+arc(NO_NAME_1, 11, 13, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 12, 14, "g", 0.000000).
+arc(NO_NAME_1, 13, 15, "@R.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 14, 16, "h", 0.000000).
+arc(NO_NAME_1, 16, 11, "t", 0.000000).
+final(NO_NAME_1, 15, 0.000000).
diff --git a/test/tools/xre.basic.lexc.result b/test/tools/xre.basic.lexc.result
deleted file mode 100644
index b56e7e3..0000000
--- a/test/tools/xre.basic.lexc.result
+++ /dev/null
@@ -1,38 +0,0 @@
-##foma-net 1.0##
-##props##
-2 16 13 18 1 5 1 1 1 1 1 2 1FB91813
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 a
-4 b
-5 f
-6 g
-7 h
-8 i
-9 o
-10 q
-11 r
-12 t
-13 u
-14 x
-15 z
-##states##
-0 5 6 0
-10 3
-4 1
-1 3 2 0
-2 15 12 0
-11 12
-3 13 4 0
-4 13 5 0
-5 14 12 0
-6 8 9 8 0
-9 7
-7 9 12 0
-8 0 13 9 0
-9 6 10 0
-10 7 11 0
-11 12 12 0
-12 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/xre.basic.lexc.result.prolog b/test/tools/xre.basic.lexc.result.prolog
new file mode 100644
index 0000000..0e04f17
--- /dev/null
+++ b/test/tools/xre.basic.lexc.result.prolog
@@ -0,0 +1,18 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 6, "f").
+arc(NO_NAME_1, 0, 3, "q").
+arc(NO_NAME_1, 0, 1, "b").
+arc(NO_NAME_1, 1, 2, "a").
+arc(NO_NAME_1, 2, 12, "z").
+arc(NO_NAME_1, 2, 12, "r").
+arc(NO_NAME_1, 3, 4, "u").
+arc(NO_NAME_1, 4, 5, "u").
+arc(NO_NAME_1, 5, 12, "x").
+arc(NO_NAME_1, 6, 8, "i":"o").
+arc(NO_NAME_1, 6, 7, "o").
+arc(NO_NAME_1, 7, 12, "o").
+arc(NO_NAME_1, 8, 9, "0":"u").
+arc(NO_NAME_1, 9, 10, "g").
+arc(NO_NAME_1, 10, 11, "h").
+arc(NO_NAME_1, 11, 12, "t").
+final(NO_NAME_1, 12).
diff --git a/test/tools/xre.definitions.lexc.flag.result b/test/tools/xre.definitions.lexc.flag.result
deleted file mode 100644
index bf6e287..0000000
Binary files a/test/tools/xre.definitions.lexc.flag.result and /dev/null differ
diff --git a/test/tools/xre.definitions.lexc.flag.result.prolog b/test/tools/xre.definitions.lexc.flag.result.prolog
new file mode 100644
index 0000000..4986423
--- /dev/null
+++ b/test/tools/xre.definitions.lexc.flag.result.prolog
@@ -0,0 +1,67 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "c", 0.000000).
+arc(NO_NAME_1, 2, 3, "d", 0.000000).
+arc(NO_NAME_1, 2, 3, "f", 0.000000).
+arc(NO_NAME_1, 2, 3, "g", 0.000000).
+arc(NO_NAME_1, 2, 3, "h", 0.000000).
+arc(NO_NAME_1, 2, 3, "j", 0.000000).
+arc(NO_NAME_1, 2, 3, "k", 0.000000).
+arc(NO_NAME_1, 2, 3, "l", 0.000000).
+arc(NO_NAME_1, 2, 3, "m", 0.000000).
+arc(NO_NAME_1, 2, 3, "n", 0.000000).
+arc(NO_NAME_1, 2, 3, "p", 0.000000).
+arc(NO_NAME_1, 2, 3, "r", 0.000000).
+arc(NO_NAME_1, 2, 3, "s", 0.000000).
+arc(NO_NAME_1, 2, 3, "š", 0.000000).
+arc(NO_NAME_1, 2, 3, "t", 0.000000).
+arc(NO_NAME_1, 2, 3, "v", 0.000000).
+arc(NO_NAME_1, 2, 3, "w", 0.000000).
+arc(NO_NAME_1, 2, 3, "x", 0.000000).
+arc(NO_NAME_1, 2, 3, "y", 0.000000).
+arc(NO_NAME_1, 2, 3, "z", 0.000000).
+arc(NO_NAME_1, 2, 3, "ž", 0.000000).
+arc(NO_NAME_1, 2, 3, "b", 0.000000).
+arc(NO_NAME_1, 3, 4, "y", 0.000000).
+arc(NO_NAME_1, 3, 4, "a", 0.000000).
+arc(NO_NAME_1, 3, 4, "ö", 0.000000).
+arc(NO_NAME_1, 3, 4, "ä", 0.000000).
+arc(NO_NAME_1, 3, 4, "u", 0.000000).
+arc(NO_NAME_1, 3, 4, "o", 0.000000).
+arc(NO_NAME_1, 3, 4, "i", 0.000000).
+arc(NO_NAME_1, 3, 4, "e", 0.000000).
+arc(NO_NAME_1, 4, 5, "c", 0.000000).
+arc(NO_NAME_1, 4, 5, "d", 0.000000).
+arc(NO_NAME_1, 4, 5, "f", 0.000000).
+arc(NO_NAME_1, 4, 5, "g", 0.000000).
+arc(NO_NAME_1, 4, 5, "h", 0.000000).
+arc(NO_NAME_1, 4, 5, "j", 0.000000).
+arc(NO_NAME_1, 4, 5, "k", 0.000000).
+arc(NO_NAME_1, 4, 5, "l", 0.000000).
+arc(NO_NAME_1, 4, 5, "m", 0.000000).
+arc(NO_NAME_1, 4, 5, "n", 0.000000).
+arc(NO_NAME_1, 4, 5, "p", 0.000000).
+arc(NO_NAME_1, 4, 5, "r", 0.000000).
+arc(NO_NAME_1, 4, 5, "s", 0.000000).
+arc(NO_NAME_1, 4, 5, "š", 0.000000).
+arc(NO_NAME_1, 4, 5, "t", 0.000000).
+arc(NO_NAME_1, 4, 5, "v", 0.000000).
+arc(NO_NAME_1, 4, 5, "w", 0.000000).
+arc(NO_NAME_1, 4, 5, "x", 0.000000).
+arc(NO_NAME_1, 4, 5, "y", 0.000000).
+arc(NO_NAME_1, 4, 5, "z", 0.000000).
+arc(NO_NAME_1, 4, 5, "ž", 0.000000).
+arc(NO_NAME_1, 4, 5, "b", 0.000000).
+arc(NO_NAME_1, 4, 6, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 5, 7, "y", 0.000000).
+arc(NO_NAME_1, 5, 7, "a", 0.000000).
+arc(NO_NAME_1, 5, 7, "ö", 0.000000).
+arc(NO_NAME_1, 5, 7, "ä", 0.000000).
+arc(NO_NAME_1, 5, 7, "u", 0.000000).
+arc(NO_NAME_1, 5, 7, "o", 0.000000).
+arc(NO_NAME_1, 5, 7, "i", 0.000000).
+arc(NO_NAME_1, 5, 7, "e", 0.000000).
+arc(NO_NAME_1, 6, 8, "@R.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 7, 6, "@P.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 8, 0.000000).
diff --git a/test/tools/xre.definitions.lexc.result b/test/tools/xre.definitions.lexc.result
deleted file mode 100644
index c63c589..0000000
--- a/test/tools/xre.definitions.lexc.result
+++ /dev/null
@@ -1,98 +0,0 @@
-##foma-net 1.0##
-##props##
-1 60 5 62 2 31152 1 1 1 1 1 2 1FB91813
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 a
-4 b
-5 c
-6 d
-7 e
-8 f
-9 g
-10 h
-11 i
-12 j
-13 k
-14 l
-15 m
-16 n
-17 o
-18 p
-19 r
-20 s
-21 t
-22 u
-23 v
-24 w
-25 x
-26 y
-27 z
-28 ä
-29 ö
-30 š
-31 ž
-##states##
-0 26 1 0
-31 1
-27 1
-25 1
-24 1
-23 1
-21 1
-30 1
-20 1
-19 1
-18 1
-16 1
-15 1
-14 1
-13 1
-12 1
-10 1
-9 1
-8 1
-6 1
-5 1
-4 1
-1 29 2 0
-28 2
-22 2
-17 2
-11 2
-7 2
-3 2
-26 2
-2 26 3 1
-31 3
-27 3
-25 3
-24 3
-23 3
-21 3
-30 3
-20 3
-19 3
-18 3
-16 3
-15 3
-14 3
-13 3
-12 3
-10 3
-9 3
-8 3
-6 3
-5 3
-4 3
-3 29 4 0
-28 4
-22 4
-17 4
-11 4
-7 4
-3 4
-26 4
-4 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/xre.definitions.lexc.result.prolog b/test/tools/xre.definitions.lexc.result.prolog
new file mode 100644
index 0000000..3d8941c
--- /dev/null
+++ b/test/tools/xre.definitions.lexc.result.prolog
@@ -0,0 +1,63 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "y").
+arc(NO_NAME_1, 0, 1, "ž").
+arc(NO_NAME_1, 0, 1, "z").
+arc(NO_NAME_1, 0, 1, "x").
+arc(NO_NAME_1, 0, 1, "w").
+arc(NO_NAME_1, 0, 1, "v").
+arc(NO_NAME_1, 0, 1, "t").
+arc(NO_NAME_1, 0, 1, "š").
+arc(NO_NAME_1, 0, 1, "s").
+arc(NO_NAME_1, 0, 1, "r").
+arc(NO_NAME_1, 0, 1, "p").
+arc(NO_NAME_1, 0, 1, "n").
+arc(NO_NAME_1, 0, 1, "m").
+arc(NO_NAME_1, 0, 1, "l").
+arc(NO_NAME_1, 0, 1, "k").
+arc(NO_NAME_1, 0, 1, "j").
+arc(NO_NAME_1, 0, 1, "h").
+arc(NO_NAME_1, 0, 1, "g").
+arc(NO_NAME_1, 0, 1, "f").
+arc(NO_NAME_1, 0, 1, "d").
+arc(NO_NAME_1, 0, 1, "c").
+arc(NO_NAME_1, 0, 1, "b").
+arc(NO_NAME_1, 1, 2, "ö").
+arc(NO_NAME_1, 1, 2, "ä").
+arc(NO_NAME_1, 1, 2, "u").
+arc(NO_NAME_1, 1, 2, "o").
+arc(NO_NAME_1, 1, 2, "i").
+arc(NO_NAME_1, 1, 2, "e").
+arc(NO_NAME_1, 1, 2, "a").
+arc(NO_NAME_1, 1, 2, "y").
+arc(NO_NAME_1, 2, 3, "y").
+arc(NO_NAME_1, 2, 3, "ž").
+arc(NO_NAME_1, 2, 3, "z").
+arc(NO_NAME_1, 2, 3, "x").
+arc(NO_NAME_1, 2, 3, "w").
+arc(NO_NAME_1, 2, 3, "v").
+arc(NO_NAME_1, 2, 3, "t").
+arc(NO_NAME_1, 2, 3, "š").
+arc(NO_NAME_1, 2, 3, "s").
+arc(NO_NAME_1, 2, 3, "r").
+arc(NO_NAME_1, 2, 3, "p").
+arc(NO_NAME_1, 2, 3, "n").
+arc(NO_NAME_1, 2, 3, "m").
+arc(NO_NAME_1, 2, 3, "l").
+arc(NO_NAME_1, 2, 3, "k").
+arc(NO_NAME_1, 2, 3, "j").
+arc(NO_NAME_1, 2, 3, "h").
+arc(NO_NAME_1, 2, 3, "g").
+arc(NO_NAME_1, 2, 3, "f").
+arc(NO_NAME_1, 2, 3, "d").
+arc(NO_NAME_1, 2, 3, "c").
+arc(NO_NAME_1, 2, 3, "b").
+arc(NO_NAME_1, 3, 4, "ö").
+arc(NO_NAME_1, 3, 4, "ä").
+arc(NO_NAME_1, 3, 4, "u").
+arc(NO_NAME_1, 3, 4, "o").
+arc(NO_NAME_1, 3, 4, "i").
+arc(NO_NAME_1, 3, 4, "e").
+arc(NO_NAME_1, 3, 4, "a").
+arc(NO_NAME_1, 3, 4, "y").
+final(NO_NAME_1, 2).
+final(NO_NAME_1, 4).
diff --git a/test/tools/xre.less-than-twice.lexc b/test/tools/xre.less-than-twice.lexc
new file mode 100644
index 0000000..f226a58
--- /dev/null
+++ b/test/tools/xre.less-than-twice.lexc
@@ -0,0 +1 @@
+LEXICON Root < [a|b]^<2 > # ;
diff --git a/test/tools/xre.less-than-twice.lexc.flag.result.prolog b/test/tools/xre.less-than-twice.lexc.flag.result.prolog
new file mode 100644
index 0000000..fb0008a
--- /dev/null
+++ b/test/tools/xre.less-than-twice.lexc.flag.result.prolog
@@ -0,0 +1,9 @@
+network(NO_NAME_1).
+symbol(NO_NAME_1, "@R.LEXNAME.#@").
+symbol(NO_NAME_1, "@R.LEXNAME.Root@").
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "a", 0.000000).
+arc(NO_NAME_1, 1, 2, "b", 0.000000).
+arc(NO_NAME_1, 2, 3, "@P.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 1, 0.000000).
+final(NO_NAME_1, 3, 0.000000).
diff --git a/test/tools/xre.less-than-twice.lexc.result.prolog b/test/tools/xre.less-than-twice.lexc.result.prolog
new file mode 100644
index 0000000..4d411f0
--- /dev/null
+++ b/test/tools/xre.less-than-twice.lexc.result.prolog
@@ -0,0 +1,5 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "a", 0.000000).
+arc(NO_NAME_1, 0, 1, "b", 0.000000).
+final(NO_NAME_1, 0, 0.000000).
+final(NO_NAME_1, 1, 0.000000).
diff --git a/test/tools/xre.months.lexc.flag.result b/test/tools/xre.months.lexc.flag.result
deleted file mode 100644
index dca3f4e..0000000
Binary files a/test/tools/xre.months.lexc.flag.result and /dev/null differ
diff --git a/test/tools/xre.months.lexc.flag.result.prolog b/test/tools/xre.months.lexc.flag.result.prolog
new file mode 100644
index 0000000..4ce8d04
--- /dev/null
+++ b/test/tools/xre.months.lexc.flag.result.prolog
@@ -0,0 +1,52 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "M", 0.000000).
+arc(NO_NAME_1, 2, 4, "A", 0.000000).
+arc(NO_NAME_1, 2, 5, "J", 0.000000).
+arc(NO_NAME_1, 2, 6, "D", 0.000000).
+arc(NO_NAME_1, 2, 7, "N", 0.000000).
+arc(NO_NAME_1, 2, 8, "O", 0.000000).
+arc(NO_NAME_1, 2, 9, "F", 0.000000).
+arc(NO_NAME_1, 3, 10, "a", 0.000000).
+arc(NO_NAME_1, 4, 11, "u", 0.000000).
+arc(NO_NAME_1, 4, 12, "p", 0.000000).
+arc(NO_NAME_1, 5, 13, "u", 0.000000).
+arc(NO_NAME_1, 5, 14, "a", 0.000000).
+arc(NO_NAME_1, 6, 15, "e", 0.000000).
+arc(NO_NAME_1, 7, 16, "o", 0.000000).
+arc(NO_NAME_1, 8, 17, "c", 0.000000).
+arc(NO_NAME_1, 9, 18, "e", 0.000000).
+arc(NO_NAME_1, 10, 19, "r", 0.000000).
+arc(NO_NAME_1, 10, 20, "y", 0.000000).
+arc(NO_NAME_1, 11, 21, "g", 0.000000).
+arc(NO_NAME_1, 12, 22, "r", 0.000000).
+arc(NO_NAME_1, 13, 23, "n", 0.000000).
+arc(NO_NAME_1, 13, 24, "l", 0.000000).
+arc(NO_NAME_1, 14, 25, "n", 0.000000).
+arc(NO_NAME_1, 15, 26, "c", 0.000000).
+arc(NO_NAME_1, 16, 26, "v", 0.000000).
+arc(NO_NAME_1, 17, 27, "t", 0.000000).
+arc(NO_NAME_1, 18, 28, "b", 0.000000).
+arc(NO_NAME_1, 19, 29, "c", 0.000000).
+arc(NO_NAME_1, 20, 30, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 21, 31, "u", 0.000000).
+arc(NO_NAME_1, 22, 32, "i", 0.000000).
+arc(NO_NAME_1, 23, 20, "e", 0.000000).
+arc(NO_NAME_1, 24, 20, "y", 0.000000).
+arc(NO_NAME_1, 25, 33, "u", 0.000000).
+arc(NO_NAME_1, 26, 34, "e", 0.000000).
+arc(NO_NAME_1, 27, 35, "o", 0.000000).
+arc(NO_NAME_1, 28, 25, "r", 0.000000).
+arc(NO_NAME_1, 29, 20, "h", 0.000000).
+arc(NO_NAME_1, 30, 36, "@R.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 31, 37, "s", 0.000000).
+arc(NO_NAME_1, 32, 20, "l", 0.000000).
+arc(NO_NAME_1, 33, 38, "a", 0.000000).
+arc(NO_NAME_1, 34, 35, "m", 0.000000).
+arc(NO_NAME_1, 35, 39, "b", 0.000000).
+arc(NO_NAME_1, 37, 20, "t", 0.000000).
+arc(NO_NAME_1, 38, 24, "r", 0.000000).
+arc(NO_NAME_1, 39, 40, "e", 0.000000).
+arc(NO_NAME_1, 40, 20, "r", 0.000000).
+final(NO_NAME_1, 36, 0.000000).
diff --git a/test/tools/xre.months.lexc.result b/test/tools/xre.months.lexc.result
deleted file mode 100644
index b8c6fc2..0000000
--- a/test/tools/xre.months.lexc.result
+++ /dev/null
@@ -1,80 +0,0 @@
-##foma-net 1.0##
-##props##
-1 46 37 48 1 11 1 1 1 1 1 2 1FB91813
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 A
-4 D
-5 F
-6 J
-7 M
-8 N
-9 O
-10 a
-11 b
-12 c
-13 e
-14 g
-15 h
-16 i
-17 l
-18 m
-19 n
-20 o
-21 p
-22 r
-23 s
-24 t
-25 u
-26 v
-27 y
-##states##
-0 4 29 0
-8 27
-9 24
-3 16
-6 8
-7 4
-5 1
-1 13 2 0
-2 11 3 0
-3 22 10 0
-4 10 5 0
-5 22 6 0
-27 36
-6 12 7 0
-7 15 36 0
-8 25 13 0
-10 9
-9 19 10 0
-10 25 11 0
-11 10 12 0
-12 22 15 0
-13 17 15 0
-19 14
-14 13 36 0
-15 27 36 0
-16 25 20 0
-21 17
-17 22 18 0
-18 16 19 0
-19 17 36 0
-20 14 21 0
-21 25 22 0
-22 23 23 0
-23 24 36 0
-24 12 25 0
-25 24 26 0
-26 20 33 0
-27 20 28 0
-28 26 31 0
-29 13 30 0
-30 12 31 0
-31 13 32 0
-32 18 33 0
-33 11 34 0
-34 13 35 0
-35 22 36 0
-36 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/xre.months.lexc.result.prolog b/test/tools/xre.months.lexc.result.prolog
new file mode 100644
index 0000000..1a30365
--- /dev/null
+++ b/test/tools/xre.months.lexc.result.prolog
@@ -0,0 +1,48 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 29, "D").
+arc(NO_NAME_1, 0, 27, "N").
+arc(NO_NAME_1, 0, 24, "O").
+arc(NO_NAME_1, 0, 16, "A").
+arc(NO_NAME_1, 0, 8, "J").
+arc(NO_NAME_1, 0, 4, "M").
+arc(NO_NAME_1, 0, 1, "F").
+arc(NO_NAME_1, 1, 2, "e").
+arc(NO_NAME_1, 2, 3, "b").
+arc(NO_NAME_1, 3, 10, "r").
+arc(NO_NAME_1, 4, 5, "a").
+arc(NO_NAME_1, 5, 6, "r").
+arc(NO_NAME_1, 5, 36, "y").
+arc(NO_NAME_1, 6, 7, "c").
+arc(NO_NAME_1, 7, 36, "h").
+arc(NO_NAME_1, 8, 13, "u").
+arc(NO_NAME_1, 8, 9, "a").
+arc(NO_NAME_1, 9, 10, "n").
+arc(NO_NAME_1, 10, 11, "u").
+arc(NO_NAME_1, 11, 12, "a").
+arc(NO_NAME_1, 12, 15, "r").
+arc(NO_NAME_1, 13, 15, "l").
+arc(NO_NAME_1, 13, 14, "n").
+arc(NO_NAME_1, 14, 36, "e").
+arc(NO_NAME_1, 15, 36, "y").
+arc(NO_NAME_1, 16, 20, "u").
+arc(NO_NAME_1, 16, 17, "p").
+arc(NO_NAME_1, 17, 18, "r").
+arc(NO_NAME_1, 18, 19, "i").
+arc(NO_NAME_1, 19, 36, "l").
+arc(NO_NAME_1, 20, 21, "g").
+arc(NO_NAME_1, 21, 22, "u").
+arc(NO_NAME_1, 22, 23, "s").
+arc(NO_NAME_1, 23, 36, "t").
+arc(NO_NAME_1, 24, 25, "c").
+arc(NO_NAME_1, 25, 26, "t").
+arc(NO_NAME_1, 26, 33, "o").
+arc(NO_NAME_1, 27, 28, "o").
+arc(NO_NAME_1, 28, 31, "v").
+arc(NO_NAME_1, 29, 30, "e").
+arc(NO_NAME_1, 30, 31, "c").
+arc(NO_NAME_1, 31, 32, "e").
+arc(NO_NAME_1, 32, 33, "m").
+arc(NO_NAME_1, 33, 34, "b").
+arc(NO_NAME_1, 34, 35, "e").
+arc(NO_NAME_1, 35, 36, "r").
+final(NO_NAME_1, 36).
diff --git a/test/tools/xre.more-than-twice.lexc b/test/tools/xre.more-than-twice.lexc
new file mode 100644
index 0000000..b05c475
--- /dev/null
+++ b/test/tools/xre.more-than-twice.lexc
@@ -0,0 +1 @@
+LEXICON Root < [a|b]^>2 > # ;
diff --git a/test/tools/xre.more-than-twice.lexc.flag.result.prolog b/test/tools/xre.more-than-twice.lexc.flag.result.prolog
new file mode 100644
index 0000000..389a391
--- /dev/null
+++ b/test/tools/xre.more-than-twice.lexc.flag.result.prolog
@@ -0,0 +1,14 @@
+network(NO_NAME_1).
+symbol(NO_NAME_1, "@R.LEXNAME.#@").
+symbol(NO_NAME_1, "@R.LEXNAME.Root@").
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "a", 0.000000).
+arc(NO_NAME_1, 1, 2, "b", 0.000000).
+arc(NO_NAME_1, 2, 3, "a", 0.000000).
+arc(NO_NAME_1, 2, 3, "b", 0.000000).
+arc(NO_NAME_1, 3, 4, "a", 0.000000).
+arc(NO_NAME_1, 3, 4, "b", 0.000000).
+arc(NO_NAME_1, 4, 4, "a", 0.000000).
+arc(NO_NAME_1, 4, 4, "b", 0.000000).
+arc(NO_NAME_1, 4, 5, "@P.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 5, 0.000000).
diff --git a/test/tools/xre.more-than-twice.lexc.result.prolog b/test/tools/xre.more-than-twice.lexc.result.prolog
new file mode 100644
index 0000000..45db674
--- /dev/null
+++ b/test/tools/xre.more-than-twice.lexc.result.prolog
@@ -0,0 +1,10 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "a", 0.000000).
+arc(NO_NAME_1, 0, 1, "b", 0.000000).
+arc(NO_NAME_1, 1, 2, "a", 0.000000).
+arc(NO_NAME_1, 1, 2, "b", 0.000000).
+arc(NO_NAME_1, 2, 3, "a", 0.000000).
+arc(NO_NAME_1, 2, 3, "b", 0.000000).
+arc(NO_NAME_1, 3, 3, "a", 0.000000).
+arc(NO_NAME_1, 3, 3, "b", 0.000000).
+final(NO_NAME_1, 3, 0.000000).
diff --git a/test/tools/xre.nested-definitions.lexc.flag.result b/test/tools/xre.nested-definitions.lexc.flag.result
deleted file mode 100644
index 318ecfd..0000000
Binary files a/test/tools/xre.nested-definitions.lexc.flag.result and /dev/null differ
diff --git a/test/tools/xre.nested-definitions.lexc.flag.result.prolog b/test/tools/xre.nested-definitions.lexc.flag.result.prolog
new file mode 100644
index 0000000..4330fd3
--- /dev/null
+++ b/test/tools/xre.nested-definitions.lexc.flag.result.prolog
@@ -0,0 +1,39 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "p", 0.000000).
+arc(NO_NAME_1, 2, 3, "t", 0.000000).
+arc(NO_NAME_1, 2, 3, "g", 0.000000).
+arc(NO_NAME_1, 2, 3, "b", 0.000000).
+arc(NO_NAME_1, 2, 3, "d", 0.000000).
+arc(NO_NAME_1, 2, 3, "k", 0.000000).
+arc(NO_NAME_1, 3, 4, "e", 0.000000).
+arc(NO_NAME_1, 3, 4, "i", 0.000000).
+arc(NO_NAME_1, 3, 4, "o", 0.000000).
+arc(NO_NAME_1, 3, 4, "u", 0.000000).
+arc(NO_NAME_1, 3, 4, "a", 0.000000).
+arc(NO_NAME_1, 4, 5, "p", 0.000000).
+arc(NO_NAME_1, 4, 5, "t", 0.000000).
+arc(NO_NAME_1, 4, 5, "g", 0.000000).
+arc(NO_NAME_1, 4, 5, "b", 0.000000).
+arc(NO_NAME_1, 4, 5, "d", 0.000000).
+arc(NO_NAME_1, 4, 5, "k", 0.000000).
+arc(NO_NAME_1, 4, 5, "e", 0.000000).
+arc(NO_NAME_1, 4, 5, "i", 0.000000).
+arc(NO_NAME_1, 4, 5, "o", 0.000000).
+arc(NO_NAME_1, 4, 5, "u", 0.000000).
+arc(NO_NAME_1, 4, 5, "a", 0.000000).
+arc(NO_NAME_1, 5, 6, "p", 0.000000).
+arc(NO_NAME_1, 5, 6, "t", 0.000000).
+arc(NO_NAME_1, 5, 6, "g", 0.000000).
+arc(NO_NAME_1, 5, 6, "b", 0.000000).
+arc(NO_NAME_1, 5, 6, "d", 0.000000).
+arc(NO_NAME_1, 5, 6, "k", 0.000000).
+arc(NO_NAME_1, 6, 7, "e", 0.000000).
+arc(NO_NAME_1, 6, 7, "i", 0.000000).
+arc(NO_NAME_1, 6, 7, "o", 0.000000).
+arc(NO_NAME_1, 6, 7, "u", 0.000000).
+arc(NO_NAME_1, 6, 7, "a", 0.000000).
+arc(NO_NAME_1, 7, 8, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 8, 9, "@R.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 9, 0.000000).
diff --git a/test/tools/xre.nested-definitions.lexc.result b/test/tools/xre.nested-definitions.lexc.result
deleted file mode 100644
index 51e5724..0000000
--- a/test/tools/xre.nested-definitions.lexc.result
+++ /dev/null
@@ -1,53 +0,0 @@
-##foma-net 1.0##
-##props##
-1 33 6 35 1 9900 1 1 1 1 1 2 1FB91813
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 a
-4 b
-5 d
-6 e
-7 g
-8 i
-9 k
-10 o
-11 p
-12 t
-13 u
-##states##
-0 5 1 0
-4 1
-7 1
-12 1
-11 1
-9 1
-1 13 2 0
-10 2
-8 2
-6 2
-3 2
-2 13 3 0
-10 3
-8 3
-6 3
-3 3
-5 3
-4 3
-7 3
-12 3
-11 3
-9 3
-3 5 4 0
-4 4
-7 4
-12 4
-11 4
-9 4
-4 13 5 0
-10 5
-8 5
-6 5
-3 5
-5 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/xre.nested-definitions.lexc.result.prolog b/test/tools/xre.nested-definitions.lexc.result.prolog
new file mode 100644
index 0000000..d4a5eaf
--- /dev/null
+++ b/test/tools/xre.nested-definitions.lexc.result.prolog
@@ -0,0 +1,35 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "d").
+arc(NO_NAME_1, 0, 1, "b").
+arc(NO_NAME_1, 0, 1, "g").
+arc(NO_NAME_1, 0, 1, "t").
+arc(NO_NAME_1, 0, 1, "p").
+arc(NO_NAME_1, 0, 1, "k").
+arc(NO_NAME_1, 1, 2, "u").
+arc(NO_NAME_1, 1, 2, "o").
+arc(NO_NAME_1, 1, 2, "i").
+arc(NO_NAME_1, 1, 2, "e").
+arc(NO_NAME_1, 1, 2, "a").
+arc(NO_NAME_1, 2, 3, "u").
+arc(NO_NAME_1, 2, 3, "o").
+arc(NO_NAME_1, 2, 3, "i").
+arc(NO_NAME_1, 2, 3, "e").
+arc(NO_NAME_1, 2, 3, "a").
+arc(NO_NAME_1, 2, 3, "d").
+arc(NO_NAME_1, 2, 3, "b").
+arc(NO_NAME_1, 2, 3, "g").
+arc(NO_NAME_1, 2, 3, "t").
+arc(NO_NAME_1, 2, 3, "p").
+arc(NO_NAME_1, 2, 3, "k").
+arc(NO_NAME_1, 3, 4, "d").
+arc(NO_NAME_1, 3, 4, "b").
+arc(NO_NAME_1, 3, 4, "g").
+arc(NO_NAME_1, 3, 4, "t").
+arc(NO_NAME_1, 3, 4, "p").
+arc(NO_NAME_1, 3, 4, "k").
+arc(NO_NAME_1, 4, 5, "u").
+arc(NO_NAME_1, 4, 5, "o").
+arc(NO_NAME_1, 4, 5, "i").
+arc(NO_NAME_1, 4, 5, "e").
+arc(NO_NAME_1, 4, 5, "a").
+final(NO_NAME_1, 5).
diff --git a/test/tools/xre.numeric-star.lexc.flag.result b/test/tools/xre.numeric-star.lexc.flag.result
deleted file mode 100644
index af6f213..0000000
Binary files a/test/tools/xre.numeric-star.lexc.flag.result and /dev/null differ
diff --git a/test/tools/xre.numeric-star.lexc.flag.result.prolog b/test/tools/xre.numeric-star.lexc.flag.result.prolog
new file mode 100644
index 0000000..ed2ffe5
--- /dev/null
+++ b/test/tools/xre.numeric-star.lexc.flag.result.prolog
@@ -0,0 +1,16 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 2, "1", 0.000000).
+arc(NO_NAME_1, 2, 2, "2", 0.000000).
+arc(NO_NAME_1, 2, 2, "3", 0.000000).
+arc(NO_NAME_1, 2, 2, "4", 0.000000).
+arc(NO_NAME_1, 2, 2, "5", 0.000000).
+arc(NO_NAME_1, 2, 2, "6", 0.000000).
+arc(NO_NAME_1, 2, 2, "7", 0.000000).
+arc(NO_NAME_1, 2, 2, "8", 0.000000).
+arc(NO_NAME_1, 2, 2, "9", 0.000000).
+arc(NO_NAME_1, 2, 2, "%0", 0.000000).
+arc(NO_NAME_1, 2, 3, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 3, 4, "@R.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 4, 0.000000).
diff --git a/test/tools/xre.numeric-star.lexc.result b/test/tools/xre.numeric-star.lexc.result
deleted file mode 100644
index a5ec121..0000000
--- a/test/tools/xre.numeric-star.lexc.result
+++ /dev/null
@@ -1,28 +0,0 @@
-##foma-net 1.0##
-##props##
-1 10 1 11 1 -1 1 1 1 1 0 2 4DAF709F
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 0
-4 1
-5 2
-6 3
-7 4
-8 5
-9 6
-10 7
-11 8
-12 9
-##states##
-0 12 0 1
-11 0
-10 0
-9 0
-8 0
-7 0
-6 0
-5 0
-4 0
-3 0
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/xre.numeric-star.lexc.result.prolog b/test/tools/xre.numeric-star.lexc.result.prolog
new file mode 100644
index 0000000..2ddf8f9
--- /dev/null
+++ b/test/tools/xre.numeric-star.lexc.result.prolog
@@ -0,0 +1,12 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 0, "9").
+arc(NO_NAME_1, 0, 0, "8").
+arc(NO_NAME_1, 0, 0, "7").
+arc(NO_NAME_1, 0, 0, "6").
+arc(NO_NAME_1, 0, 0, "5").
+arc(NO_NAME_1, 0, 0, "4").
+arc(NO_NAME_1, 0, 0, "3").
+arc(NO_NAME_1, 0, 0, "2").
+arc(NO_NAME_1, 0, 0, "1").
+arc(NO_NAME_1, 0, 0, "%0").
+final(NO_NAME_1, 0).
diff --git a/test/tools/xre.quotations.lexc.flag.result b/test/tools/xre.quotations.lexc.flag.result
deleted file mode 100644
index c1098d0..0000000
Binary files a/test/tools/xre.quotations.lexc.flag.result and /dev/null differ
diff --git a/test/tools/xre.quotations.lexc.flag.result.prolog b/test/tools/xre.quotations.lexc.flag.result.prolog
new file mode 100644
index 0000000..23009ed
--- /dev/null
+++ b/test/tools/xre.quotations.lexc.flag.result.prolog
@@ -0,0 +1,15 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "а", 0.000000).
+arc(NO_NAME_1, 3, 4, "в", 0.000000).
+arc(NO_NAME_1, 4, 5, "о", 0.000000).
+arc(NO_NAME_1, 5, 6, "л", 0.000000).
+arc(NO_NAME_1, 6, 7, "ь", 0.000000).
+arc(NO_NAME_1, 7, 8, "+Verb":"0", 0.000000).
+arc(NO_NAME_1, 8, 9, "+Neg":"0", 0.000000).
+arc(NO_NAME_1, 9, 10, "+Cond":"0", 0.000000).
+arc(NO_NAME_1, 10, 11, "@P.CONJ.ObjAll@", 0.000000).
+arc(NO_NAME_1, 11, 12, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 12, 13, "@R.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 13, 0.000000).
diff --git a/test/tools/xre.quotations.lexc.result b/test/tools/xre.quotations.lexc.result
deleted file mode 100644
index ee8d2a0..0000000
--- a/test/tools/xre.quotations.lexc.result
+++ /dev/null
@@ -1,27 +0,0 @@
-##foma-net 1.0##
-##props##
-2 9 10 11 1 1 1 1 1 1 1 2 4DAF709F
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 +Cond
-4 +Neg
-5 +Verb
-6 @P.CONJ.ObjAll@
-7 а
-8 в
-9 л
-10 о
-11 ь
-##states##
-0 7 1 0
-1 8 2 0
-2 10 3 0
-3 9 4 0
-4 11 5 0
-5 5 0 6 0
-6 4 0 7 0
-7 3 0 8 0
-8 6 9 0
-9 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/xre.quotations.lexc.result.prolog b/test/tools/xre.quotations.lexc.result.prolog
new file mode 100644
index 0000000..be0cc45
--- /dev/null
+++ b/test/tools/xre.quotations.lexc.result.prolog
@@ -0,0 +1,11 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "а").
+arc(NO_NAME_1, 1, 2, "в").
+arc(NO_NAME_1, 2, 3, "о").
+arc(NO_NAME_1, 3, 4, "л").
+arc(NO_NAME_1, 4, 5, "ь").
+arc(NO_NAME_1, 5, 6, "+Verb":"0").
+arc(NO_NAME_1, 6, 7, "+Neg":"0").
+arc(NO_NAME_1, 7, 8, "+Cond":"0").
+arc(NO_NAME_1, 8, 9, "@P.CONJ.ObjAll@").
+final(NO_NAME_1, 9).
diff --git a/test/tools/xre.sharp.lexc.flag.result b/test/tools/xre.sharp.lexc.flag.result
deleted file mode 100644
index df91139..0000000
Binary files a/test/tools/xre.sharp.lexc.flag.result and /dev/null differ
diff --git a/test/tools/xre.sharp.lexc.flag.result.prolog b/test/tools/xre.sharp.lexc.flag.result.prolog
new file mode 100644
index 0000000..6d512eb
--- /dev/null
+++ b/test/tools/xre.sharp.lexc.flag.result.prolog
@@ -0,0 +1,35 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 2, "b", 0.000000).
+arc(NO_NAME_1, 2, 2, "c", 0.000000).
+arc(NO_NAME_1, 2, 2, "d", 0.000000).
+arc(NO_NAME_1, 2, 2, "e", 0.000000).
+arc(NO_NAME_1, 2, 2, "f", 0.000000).
+arc(NO_NAME_1, 2, 2, "g", 0.000000).
+arc(NO_NAME_1, 2, 2, "h", 0.000000).
+arc(NO_NAME_1, 2, 2, "i", 0.000000).
+arc(NO_NAME_1, 2, 2, "j", 0.000000).
+arc(NO_NAME_1, 2, 2, "k", 0.000000).
+arc(NO_NAME_1, 2, 2, "l", 0.000000).
+arc(NO_NAME_1, 2, 2, "m", 0.000000).
+arc(NO_NAME_1, 2, 2, "n", 0.000000).
+arc(NO_NAME_1, 2, 2, "o", 0.000000).
+arc(NO_NAME_1, 2, 2, "p", 0.000000).
+arc(NO_NAME_1, 2, 2, "q", 0.000000).
+arc(NO_NAME_1, 2, 2, "r", 0.000000).
+arc(NO_NAME_1, 2, 2, "s", 0.000000).
+arc(NO_NAME_1, 2, 2, "t", 0.000000).
+arc(NO_NAME_1, 2, 2, "u", 0.000000).
+arc(NO_NAME_1, 2, 2, "v", 0.000000).
+arc(NO_NAME_1, 2, 2, "x", 0.000000).
+arc(NO_NAME_1, 2, 2, "y", 0.000000).
+arc(NO_NAME_1, 2, 2, "z", 0.000000).
+arc(NO_NAME_1, 2, 2, "å", 0.000000).
+arc(NO_NAME_1, 2, 2, "ä", 0.000000).
+arc(NO_NAME_1, 2, 2, "ö", 0.000000).
+arc(NO_NAME_1, 2, 2, "a", 0.000000).
+arc(NO_NAME_1, 2, 3, "#", 0.000000).
+arc(NO_NAME_1, 3, 4, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 4, 5, "@R.LEXNAME.#@", 0.000000).
+final(NO_NAME_1, 5, 0.000000).
diff --git a/test/tools/xre.sharp.lexc.result b/test/tools/xre.sharp.lexc.result
deleted file mode 100644
index ab2774d..0000000
--- a/test/tools/xre.sharp.lexc.result
+++ /dev/null
@@ -1,67 +0,0 @@
-##foma-net 1.0##
-##props##
-1 29 2 31 1 -1 1 1 1 1 0 2 4DAF709F
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 #
-4 a
-5 b
-6 c
-7 d
-8 e
-9 f
-10 g
-11 h
-12 i
-13 j
-14 k
-15 l
-16 m
-17 n
-18 o
-19 p
-20 q
-21 r
-22 s
-23 t
-24 u
-25 v
-26 x
-27 y
-28 z
-29 ä
-30 å
-31 ö
-##states##
-0 3 1 0
-31 0
-29 0
-30 0
-28 0
-27 0
-26 0
-25 0
-24 0
-23 0
-22 0
-21 0
-20 0
-19 0
-18 0
-17 0
-16 0
-15 0
-14 0
-13 0
-12 0
-11 0
-10 0
-9 0
-8 0
-7 0
-6 0
-5 0
-4 0
-1 -1 -1 1
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/xre.sharp.lexc.result.prolog b/test/tools/xre.sharp.lexc.result.prolog
new file mode 100644
index 0000000..58f16bb
--- /dev/null
+++ b/test/tools/xre.sharp.lexc.result.prolog
@@ -0,0 +1,31 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "#").
+arc(NO_NAME_1, 0, 0, "ö").
+arc(NO_NAME_1, 0, 0, "ä").
+arc(NO_NAME_1, 0, 0, "å").
+arc(NO_NAME_1, 0, 0, "z").
+arc(NO_NAME_1, 0, 0, "y").
+arc(NO_NAME_1, 0, 0, "x").
+arc(NO_NAME_1, 0, 0, "v").
+arc(NO_NAME_1, 0, 0, "u").
+arc(NO_NAME_1, 0, 0, "t").
+arc(NO_NAME_1, 0, 0, "s").
+arc(NO_NAME_1, 0, 0, "r").
+arc(NO_NAME_1, 0, 0, "q").
+arc(NO_NAME_1, 0, 0, "p").
+arc(NO_NAME_1, 0, 0, "o").
+arc(NO_NAME_1, 0, 0, "n").
+arc(NO_NAME_1, 0, 0, "m").
+arc(NO_NAME_1, 0, 0, "l").
+arc(NO_NAME_1, 0, 0, "k").
+arc(NO_NAME_1, 0, 0, "j").
+arc(NO_NAME_1, 0, 0, "i").
+arc(NO_NAME_1, 0, 0, "h").
+arc(NO_NAME_1, 0, 0, "g").
+arc(NO_NAME_1, 0, 0, "f").
+arc(NO_NAME_1, 0, 0, "e").
+arc(NO_NAME_1, 0, 0, "d").
+arc(NO_NAME_1, 0, 0, "c").
+arc(NO_NAME_1, 0, 0, "b").
+arc(NO_NAME_1, 0, 0, "a").
+final(NO_NAME_1, 1).
diff --git a/test/tools/xre.star-plus-optional.lexc.flag.result b/test/tools/xre.star-plus-optional.lexc.flag.result
deleted file mode 100644
index 1a5e117..0000000
Binary files a/test/tools/xre.star-plus-optional.lexc.flag.result and /dev/null differ
diff --git a/test/tools/xre.star-plus-optional.lexc.flag.result.prolog b/test/tools/xre.star-plus-optional.lexc.flag.result.prolog
new file mode 100644
index 0000000..b012e9f
--- /dev/null
+++ b/test/tools/xre.star-plus-optional.lexc.flag.result.prolog
@@ -0,0 +1,23 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 1, "@P.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 1, 2, "@R.LEXNAME.Root@", 0.000000).
+arc(NO_NAME_1, 2, 3, "d", 0.000000).
+arc(NO_NAME_1, 2, 4, "a", 0.000000).
+arc(NO_NAME_1, 2, 5, "g", 0.000000).
+arc(NO_NAME_1, 2, 6, "g":"w", 0.000000).
+arc(NO_NAME_1, 3, 7, "o":"i", 0.000000).
+arc(NO_NAME_1, 4, 8, "b", 0.000000).
+arc(NO_NAME_1, 5, 10, "o", 0.000000).
+arc(NO_NAME_1, 6, 9, "o":"e", 0.000000).
+arc(NO_NAME_1, 7, 10, "0":"d", 0.000000).
+arc(NO_NAME_1, 8, 11, "d", 0.000000).
+arc(NO_NAME_1, 8, 8, "c", 0.000000).
+arc(NO_NAME_1, 8, 12, "e", 0.000000).
+arc(NO_NAME_1, 9, 14, "0":"n", 0.000000).
+arc(NO_NAME_1, 10, 13, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 11, 12, "e", 0.000000).
+arc(NO_NAME_1, 12, 13, "@P.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 12, 12, "e", 0.000000).
+arc(NO_NAME_1, 13, 15, "@R.LEXNAME.#@", 0.000000).
+arc(NO_NAME_1, 14, 10, "0":"t", 0.000000).
+final(NO_NAME_1, 15, 0.000000).
diff --git a/test/tools/xre.star-plus-optional.lexc.result b/test/tools/xre.star-plus-optional.lexc.result
deleted file mode 100644
index a5090af..0000000
--- a/test/tools/xre.star-plus-optional.lexc.result
+++ /dev/null
@@ -1,36 +0,0 @@
-##foma-net 1.0##
-##props##
-2 16 12 18 2 -1 1 1 1 1 0 2 4DAF709F
-##sigma##
-0 @_EPSILON_SYMBOL_@
-3 a
-4 b
-5 c
-6 d
-7 e
-8 g
-9 i
-10 n
-11 o
-12 t
-13 w
-##states##
-0 8 13 9 0
-8 8
-6 5
-3 1
-1 4 2 0
-2 7 4 0
-6 3
-5 2
-3 7 4 0
-4 7 4 1
-5 11 9 6 0
-6 0 6 7 0
-7 -1 -1 1
-8 11 7 0
-9 11 7 10 0
-10 0 10 11 0
-11 0 12 7 0
--1 -1 -1 -1 -1
-##end##
diff --git a/test/tools/xre.star-plus-optional.lexc.result.prolog b/test/tools/xre.star-plus-optional.lexc.result.prolog
new file mode 100644
index 0000000..067d89c
--- /dev/null
+++ b/test/tools/xre.star-plus-optional.lexc.result.prolog
@@ -0,0 +1,19 @@
+network(NO_NAME_1).
+arc(NO_NAME_1, 0, 9, "g":"w").
+arc(NO_NAME_1, 0, 8, "g").
+arc(NO_NAME_1, 0, 5, "d").
+arc(NO_NAME_1, 0, 1, "a").
+arc(NO_NAME_1, 1, 2, "b").
+arc(NO_NAME_1, 2, 4, "e").
+arc(NO_NAME_1, 2, 3, "d").
+arc(NO_NAME_1, 2, 2, "c").
+arc(NO_NAME_1, 3, 4, "e").
+arc(NO_NAME_1, 4, 4, "e").
+arc(NO_NAME_1, 5, 6, "o":"i").
+arc(NO_NAME_1, 6, 7, "0":"d").
+arc(NO_NAME_1, 8, 7, "o").
+arc(NO_NAME_1, 9, 10, "o":"e").
+arc(NO_NAME_1, 10, 11, "0":"n").
+arc(NO_NAME_1, 11, 7, "0":"t").
+final(NO_NAME_1, 4).
+final(NO_NAME_1, 7).
diff --git a/tools/src/FunctionalTransducer.cc b/tools/src/FunctionalTransducer.cc
index 9881cd9..60fe0de 100644
--- a/tools/src/FunctionalTransducer.cc
+++ b/tools/src/FunctionalTransducer.cc
@@ -10,7 +10,7 @@ FunctionalTransducer::FunctionalTransducer(const HfstBasicTransducer &fst):
StringSet input_symbols;
StringSet output_symbols;
verbose_print("Collecting input and output symbols for tokenization.");
- collect_symbols_from_fst(input_symbols,output_symbols,fst);
+ collect_symbols_from_fst(input_symbols,output_symbols,fst);
verbose_print("Defining input symbols.");
define_multichar_symbols(input_symbols,input_tokenizer);
@@ -23,7 +23,7 @@ void FunctionalTransducer::set_verbose_print(std::ostream &out)
{ verbose_out = &out; }
void FunctionalTransducer::verbose_print(const std::string &msg)
-{
+{
if (verbose_out != NULL)
{ std::cout << msg << std::endl; }
}
@@ -99,7 +99,7 @@ StringVector FunctionalTransducer::tokenize
++it)
{
if (not it->empty())
- {
+ {
if (*it == ZERO)
{ *it = EPSILON_SYMBOL; }
if (*it->begin() == '\\')
@@ -128,7 +128,7 @@ std::string FunctionalTransducer::join(const StringVector & v,
return joined;
}
-StringVector
+StringVector
FunctionalTransducer::apply(const StringVector &v,bool input) const
{
StringVectorVector result;
@@ -156,4 +156,4 @@ int main(void)
std::cerr << output << std::endl;
std::string input = func_cats.apply_on_output("cat+N+Nom+Pl0\\0");
std::cerr << input << std::endl;
-}
+}
diff --git a/tools/src/FunctionalTransducer.h b/tools/src/FunctionalTransducer.h
index 0e01133..378a474 100644
--- a/tools/src/FunctionalTransducer.h
+++ b/tools/src/FunctionalTransducer.h
@@ -65,7 +65,7 @@ class FunctionalTransducer
void follow_epsilon_transitions
(StringVector::iterator it,StringVector::iterator end,
HfstState,bool,StringStringVector &,StateSet &);
- void follow_transitions
+ void follow_transitions
(StringVector::iterator it,StringVector::iterator end,
HfstState,bool,StringStringVector &);
void join_and_uniquify(StringVectorVector &);
diff --git a/tools/src/Makefile.am b/tools/src/Makefile.am
index 696af74..c159ea7 100644
--- a/tools/src/Makefile.am
+++ b/tools/src/Makefile.am
@@ -21,16 +21,6 @@ LDADD = $(top_builddir)/libhfst/src/libhfst.la
AM_CPPFLAGS = -I${top_srcdir}/libhfst/src -I${top_srcdir}/libhfst/src/parsers -I${top_srcdir}/tools/src/parsers -Wno-sign-compare
AM_CXXFLAGS = -Wno-deprecated -Wno-sign-compare
-AM_CPPFLAGS+= -I${top_srcdir}/back-ends/foma/ \
- -I${top_srcdir}/back-ends/
-
-if WANT_MINGW
- AM_CPPFLAGS+= -I${top_srcdir}/back-ends/openfstwin/src/include \
- -I${top_srcdir}/back-ends/dlfcn -DWINDOWS
-else
- AM_CPPFLAGS+= -I${top_srcdir}/back-ends/openfst/src/include
-endif
-
# sort alphabetically:
if WANT_AFFIX_GUESSIFY
MAYBE_AFFIX_GUESSIFY=hfst-affix-guessify$(EXEEXT)
@@ -194,7 +184,6 @@ endif
noinst_HEADERS= \
calculate_functions.h hfst-commandline.h hfst-program-options.h \
- HfstCompiler.h HfstBasic.h HfstAlphabet.h HfstUtf8.h \
inc/check-params-binary.h inc/check-params-common.h \
inc/check-params-unary.h inc/getopt-cases-binary.h \
inc/getopt-cases-common.h inc/getopt-cases-error.h \
@@ -202,7 +191,7 @@ noinst_HEADERS= \
inc/globals-common.h inc/globals-unary.h \
hfst-file-to-mem.h \
hfst-tool-metadata.h hfst-optimized-lookup.h \
- guessify_fst.h generate_model_forms.h hfst-compiler.$(HEADER)
+ guessify_fst.h generate_model_forms.h
# parsers/XreCompiler.h parsers/xre_utils.h
# sort alphabetically
@@ -242,7 +231,7 @@ HFST_COMMON_SRC=hfst-program-options.cc hfst-commandline.cc \
hfst-tool-metadata.cc
# sort alphabetically
hfst_affix_guessify_SOURCES=hfst-affix-guessify.cc $(HFST_COMMON_SRC)
-hfst_calculate_SOURCES=hfst-compiler.yy hfst-scanner.ll HfstCompiler.cc HfstBasic.cc HfstAlphabet.cc HfstUtf8.cc $(HFST_COMMON_SRC)
+hfst_calculate_SOURCES=sfst-main.cc $(HFST_COMMON_SRC)
hfst_compare_SOURCES=hfst-compare.cc $(HFST_COMMON_SRC)
hfst_compose_intersect_SOURCES=hfst-compose-intersect.cc $(HFST_COMMON_SRC)
hfst_compose_SOURCES=hfst-compose.cc $(HFST_COMMON_SRC)
@@ -275,14 +264,14 @@ hfst_lexc_SOURCES=hfst-lexc-compiler.cc $(HFST_COMMON_SRC)
#hfst_lexc2fst_BUILT=parsers/lexc-parser.h parsers/xre_parse.h
hfst_lookup_SOURCES=hfst-lookup.cc $(HFST_COMMON_SRC)
hfst_flookup_SOURCES=hfst-flookup.cc $(HFST_COMMON_SRC)
-hfst_pair_test_SOURCES=hfst-pair-test.cc $(HFST_COMMON_SRC)
+hfst_pair_test_SOURCES=hfst-pair-test.cc $(HFST_COMMON_SRC)
hfst_minimize_SOURCES=hfst-minimize.cc $(HFST_COMMON_SRC)
hfst_name_SOURCES=hfst-name.cc $(HFST_COMMON_SRC)
hfst_optimized_lookup_SOURCES=hfst-optimized-lookup.cc
hfst_pmatch_SOURCES=hfst-pmatch.cc $(HFST_COMMON_SRC)
#hfst_preprocess_for_optimized_lookup_format_SOURCES=hfst-preprocess-for-optimized-lookup-format.cc
# $(HFST_COMMON_SRC)
-hfst_tokenize_SOURCES=hfst-tokenize.cc $(HFST_COMMON_SRC)
+hfst_tokenize_SOURCES=hfst-tokenize.cc $(HFST_COMMON_SRC)
hfst_project_SOURCES=hfst-project.cc $(HFST_COMMON_SRC)
hfst_prune_alphabet_SOURCES=hfst-prune-alphabet.cc $(HFST_COMMON_SRC)
hfst_push_weights_SOURCES=hfst-push-weights.cc $(HFST_COMMON_SRC)
@@ -334,18 +323,14 @@ uninstall-local:
rm hfst-tokenise && \
rm hfst-proc2
-BUILT_SOURCES=$(hfst_lexc_BUILT)
+BUILT_SOURCES=$(hfst_lexc_BUILT)
-CLEANFILES=$(hfst_compiler_header_file) hfst-compiler.cc $(hfst_lexc_BUILT)
+CLEANFILES=$(hfst_lexc_BUILT)
hfst_calculate_CXXFLAGS= -Wall -Wno-error -Wno-sign-compare -Wno-deprecated
## YACC and FLEX with autotools argh
AM_YFLAGS=-d
-AM_LFLAGS=--outfile=$(LEX_OUTPUT_ROOT).c
-
-hfst-scanner.ll: hfst-compiler.$(HEADER)
+AM_LFLAGS=--outfile=$(LEX_OUTPUT_ROOT).c
-CLEANFILES+=hfst-compiler.$(HEADER)
-EXTRA_DIST=hfst-scanner.cc hfst-compiler.cc hfst-compiler.$(HEADER)
diff --git a/tools/src/generate_model_forms.cc b/tools/src/generate_model_forms.cc
index 9a49d4e..7a37c6f 100644
--- a/tools/src/generate_model_forms.cc
+++ b/tools/src/generate_model_forms.cc
@@ -21,7 +21,7 @@ std::ostream &operator<<(std::ostream &out,const StringVector &v)
for (StringVector::const_iterator it = v.begin();
it != v.end();
++it)
- {
+ {
out << *it;
}
@@ -60,9 +60,9 @@ StringVector get_analysis_prefix(const StringVector &reversed_analysis)
prefix.push_back(*it);
if (is_cathegory_symbol(*it))
- {
+ {
// When we return, we have to reverse the result.
- return StringVector(prefix.rbegin(), prefix.rend());
+ return StringVector(prefix.rbegin(), prefix.rend());
}
}
@@ -84,8 +84,8 @@ bool contains_analysis_symbols(const StringVector &word_form)
++it)
{
const std::string &symbol = *it;
- if (symbol.size() > 1 and
- symbol[0] == '[' and
+ if (symbol.size() > 1 and
+ symbol[0] == '[' and
symbol[symbol.size() - 1] == ']')
{ return true; }
}
@@ -126,7 +126,7 @@ StringVector generate_word_forms(const StringVector &analysis,
// The word form is reversed, so we start from the end and
// iterate to the beginning.
- result_set.insert(StringVector(word_form.rbegin(),
+ result_set.insert(StringVector(word_form.rbegin(),
word_form.rend()));
++num;
@@ -162,7 +162,7 @@ StringVectorVector get_model_forms(const StringVector &reversed_analysis,
size_t max_generated_forms,
float generate_threshold)
{
- StringVector reversed_analysis_prefix =
+ StringVector reversed_analysis_prefix =
get_analysis_prefix(reversed_analysis);
StringVectorVector results;
@@ -242,10 +242,10 @@ StringVectorVector get_guesses(const std::string &word_form,
for (HfstOneLevelPaths::const_iterator it = paths->begin();
it != paths->end();
++it)
- {
+ {
if (num > number_of_guesses)
{ break; }
- results.push_back(it->second);
+ results.push_back(it->second);
++num;
}
@@ -266,7 +266,7 @@ StringVectorVector get_paradigms(const std::string &word_form,
for (StringVectorVector::const_iterator it = guesses.begin();
it != guesses.end();
++it)
- {
+ {
StringVector analysis_guess = *it;
StringVectorVector results = get_model_forms(analysis_guess,
@@ -281,8 +281,8 @@ StringVectorVector get_paradigms(const std::string &word_form,
StringVector rev_analysis_guess(analysis_guess);
std::reverse(rev_analysis_guess.begin(), rev_analysis_guess.end());
- paradigm.insert(paradigm.end(),
- rev_analysis_guess.begin(),
+ paradigm.insert(paradigm.end(),
+ rev_analysis_guess.begin(),
rev_analysis_guess.end());
for (StringVectorVector::const_iterator jt = results.begin();
@@ -292,8 +292,8 @@ StringVectorVector get_paradigms(const std::string &word_form,
const StringVector &model_form = *jt;
paradigm.push_back("\t");
- paradigm.insert(paradigm.end(),
- model_form.begin(),
+ paradigm.insert(paradigm.end(),
+ model_form.begin(),
model_form.end());
}
diff --git a/tools/src/guessify_fst.cc b/tools/src/guessify_fst.cc
index 7bc5aeb..bd96dcc 100644
--- a/tools/src/guessify_fst.cc
+++ b/tools/src/guessify_fst.cc
@@ -30,7 +30,7 @@ using hfst::implementations::HfstBasicTransducer;
using hfst::implementations::HfstState;
using hfst::implementations::HfstBasicTransition;
-typedef HfstBasicTransducer::HfstTransitions HfstBasicTransitions;
+typedef hfst::implementations::HfstBasicTransitions HfstBasicTransitions;
using hfst::internal_default;
using hfst::internal_epsilon;
@@ -50,7 +50,7 @@ using hfst::HfstOutputStream;
std::string my_default = "$_DEFAULT_SYMBOL_$";
-void remove_flag_diacritics(HfstTransducer &morphological_analyzer,
+void remove_flag_diacritics(HfstTransducer &morphological_analyzer,
const StringSet &alphabet)
{
HfstSymbolSubstitutions flag_diacritic_epsilon_pairs;
@@ -124,7 +124,7 @@ HfstTransducer get_prefix_remover(const StringSet &alphabet)
remove_suffix.concatenate(remove_symbol);
remove_suffix.optionalize().minimize();
- identity_except_cathegory.concatenate(remove_suffix).minimize();
+ identity_except_cathegory.concatenate(remove_suffix).minimize();
return identity_except_cathegory;
}
@@ -206,7 +206,7 @@ HfstTransducer guessify_analyzer(HfstTransducer morphological_analyzer,
morphological_analyzer.minimize();
// Remove the parts of analyses that precede the last cathegory
- // tag. After the last cathegory tag all input should be echoed as is to
+ // tag. After the last cathegory tag all input should be echoed as is to
// the output.
/*
HfstTransducer analysis_prefix_remover = get_prefix_remover(alphabet);
@@ -225,13 +225,13 @@ HfstTransducer guessify_analyzer(HfstTransducer morphological_analyzer,
HfstState sink_state = basic_guesser.add_state();
for (HfstState s = 0; s <= basic_guesser.get_max_state(); ++s)
- {
- basic_guesser.set_final_weight(s,0.0);
+ {
+ basic_guesser.set_final_weight(s,0.0);
}
for (HfstState s = 0; s <= basic_guesser.get_max_state(); ++s)
- {
+ {
basic_guesser.add_transition
(s,HfstBasicTransition(sink_state,
my_default,
@@ -259,7 +259,7 @@ HfstTransducer guessify_analyzer(HfstTransducer morphological_analyzer,
guesser.compose(invalid_form_filterer).minimize();
- guesser.set_name(std::string("guessified(") +
+ guesser.set_name(std::string("guessified(") +
morphological_analyzer_name +
")");
diff --git a/tools/src/guessify_fst.h b/tools/src/guessify_fst.h
index ae1d5c0..8aa5e2e 100644
--- a/tools/src/guessify_fst.h
+++ b/tools/src/guessify_fst.h
@@ -34,7 +34,7 @@ using hfst::StringSet;
#define CATEGORY_SYMBOL_PREFIX "[GUESS_CATEGORY="
// Replace every flag diacritic symbol with an epsilon.
-void remove_flag_diacritics(HfstTransducer &morphological_analyzer,
+void remove_flag_diacritics(HfstTransducer &morphological_analyzer,
const StringSet &alphabet);
// Return the set of all symbols in alphabet, which start "[GUESS_CATEGORY=".
diff --git a/tools/src/hfst-affix-guessify.cc b/tools/src/hfst-affix-guessify.cc
index 7aceeba..2557fdd 100644
--- a/tools/src/hfst-affix-guessify.cc
+++ b/tools/src/hfst-affix-guessify.cc
@@ -34,7 +34,7 @@
#include "HfstTransducer.h"
#include "HfstInputStream.h"
#include "HfstOutputStream.h"
-#include "implementations/HfstTransitionGraph.h"
+#include "implementations/HfstBasicTransducer.h"
using std::map;
using std::string;
@@ -104,14 +104,14 @@ parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
- // add tool-specific options here
+ // add tool-specific options here
{"weight", required_argument, 0, 'w'},
{"direction", required_argument, 0, 'D'},
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT
"w:D:",
long_options, &option_index);
@@ -229,7 +229,7 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
weight);
repl->add_transition(guess_state, x_arc);
}
- for (HfstBasicTransducer::HfstTransitions::const_iterator arc =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator arc =
i->begin();
arc != i->end();
++arc)
@@ -294,7 +294,7 @@ int main( int argc, char **argv ) {
{
fclose(inputfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-binary-tool.cc b/tools/src/hfst-binary-tool.cc
index 1689ad0..28dcbcb 100644
--- a/tools/src/hfst-binary-tool.cc
+++ b/tools/src/hfst-binary-tool.cc
@@ -78,7 +78,7 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_BINARY_SHORT,
long_options, &option_index);
if (-1 == c)
@@ -105,7 +105,7 @@ binaryoperate_streams(HfstInputStream& firststream, HfstInputStream& secondstrea
firststream.open();
secondstream.open();
outstream.open();
- // should be is_good?
+ // should be is_good?
bool bothInputs = firststream.is_good() && secondstream.is_good();
if (firststream.get_type() != secondstream.get_type())
{
@@ -118,7 +118,7 @@ binaryoperate_streams(HfstInputStream& firststream, HfstInputStream& secondstrea
transducer_n++;
if (transducer_n == 1)
{
- verbose_printf("Doing things with %s and %s...\n", firstfilename,
+ verbose_printf("Doing things with %s and %s...\n", firstfilename,
secondfilename);
}
else
@@ -169,7 +169,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s and %s, writing to %s\n",
+ verbose_printf("Reading from %s and %s, writing to %s\n",
firstfilename, secondfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* firststream = NULL;
diff --git a/tools/src/hfst-commandline.cc b/tools/src/hfst-commandline.cc
index abdf7ec..1c1e1b5 100644
--- a/tools/src/hfst-commandline.cc
+++ b/tools/src/hfst-commandline.cc
@@ -68,7 +68,7 @@
#endif
#ifndef HAVE_ERROR_AT_LINE
-void error_at_line(int status, int errnum, const char* filename,
+void error_at_line(int status, int errnum, const char* filename,
unsigned int linenum, const char* fmt, ...)
{
fprintf(stderr, "%s.%u: ", filename, linenum);
@@ -141,8 +141,9 @@ debug_save_transducer(hfst::HfstTransducer& t, const char* name)
{
if (debug)
{
- char* debug_name = static_cast<char*>(malloc(sizeof(char)*(strlen("DEBUG: " + strlen(name) + 1 ))));
- if ((sprintf(debug_name, "DEBUG %s", name)) > 0 )
+ char* debug_name = static_cast<char*>
+ ( malloc( sizeof(char)*(strlen("DEBUG: ") + strlen(name) + 1) ) );
+ if ((sprintf(debug_name, "DEBUG %s", name)) > 0 )
{
t.set_name(debug_name);
}
@@ -209,13 +210,13 @@ convert_transducers(hfst::HfstTransducer & first, hfst::HfstTransducer & second)
return;
else if (ct == 1)
{
- verbose_printf("warning: transducers have different types, converting to format %s\n",
+ verbose_printf("warning: transducers have different types, converting to format %s\n",
hfst_strformat(type1));
second.convert(type1);
}
else if (ct == 2)
{
- verbose_printf("warning: transducers have different types, converting to format %s\n",
+ verbose_printf("warning: transducers have different types, converting to format %s\n",
hfst_strformat(type2));
first.convert(type2);
}
@@ -234,7 +235,7 @@ convert_transducers(hfst::HfstTransducer & first, hfst::HfstTransducer & second)
bool
is_input_stream_in_ol_format(const hfst::HfstInputStream * is, const char * program)
{
- if ( is->get_type() == hfst::HFST_OL_TYPE ||
+ if ( is->get_type() == hfst::HFST_OL_TYPE ||
is->get_type() == hfst::HFST_OLW_TYPE )
{
fprintf(stderr, "Error: %s cannot process transducers that are in optimized lookup format.\n", program);
@@ -267,7 +268,7 @@ hfst_strtonumber(const char *s, bool *infinite)
*infinite = false;
errno = 0;
char *endptr;
- double rv = strtod(s, &endptr);
+ double rv = strtod(s, &endptr);
if (*endptr == '\0')
{
#ifndef _MSC_VER
@@ -467,7 +468,7 @@ hfst_fseek(FILE* stream, long offset, int whence)
}
}
-unsigned long
+unsigned long
hfst_ftell(FILE* stream)
{
errno = 0;
@@ -575,10 +576,10 @@ int
hfst_mkstemp(char* templ)
{
#ifdef _WIN32
- error(EXIT_FAILURE, errno,
+ error(EXIT_FAILURE, errno,
"'int hfst_mkstemp(char * temp1)' not implemented for windows");
return 1; // keep compiler happy
-#else
+#else
errno = 0;
int rv = mkstemp(templ);
if (rv == -1)
@@ -866,7 +867,7 @@ print_version()
{
// c.f. http://www.gnu.org/prep/standards/standards.html#g_t_002d_002dversion
fprintf(message_out, "%s %s (" PACKAGE_STRING ")\n"
- "Copyright (C) 2016 University of Helsinki,\n"
+ "Copyright (C) 2017 University of Helsinki,\n"
"License GPLv3: GNU GPL version 3 "
"<http://gnu.org/licenses/gpl.html>\n"
"This is free software: you are free to change and "
@@ -903,7 +904,7 @@ extend_options_getenv(int* argc, char*** argv)
p++;
}
// we cannot realloc argv since it's magic
- char** new_argv = static_cast<char**>(hfst_malloc(sizeof(char*) *
+ char** new_argv = static_cast<char**>(hfst_malloc(sizeof(char*) *
(*argc + spaces + 1)));
new_argv = static_cast<char**>(memcpy(new_argv, *argv, sizeof(char*)**argc));
// there's this magic stuff with *argv that we shouldn't free it still
diff --git a/tools/src/hfst-commandline.h b/tools/src/hfst-commandline.h
index 1ff2ef1..1aff359 100644
--- a/tools/src/hfst-commandline.h
+++ b/tools/src/hfst-commandline.h
@@ -47,7 +47,7 @@ namespace hfst { class HfstInputStream; } ;
/* These variables should be used in all command line programs.
* In some cases they may be nonsensical; just define something then.
*/
-/**
+/**
* @brief set @c verbose when program should print before and after every
* non-trivial step it takes.
*/
@@ -60,17 +60,17 @@ extern bool silent;
extern bool debug;
/** @brief set @c message_out to stream that is usable for non-error message
* print-outs.
- * This @e should be stdout in all cases, except when transducer
- * binaries are being transmitted through @c stdout. Some programs
+ * This @e should be stdout in all cases, except when transducer
+ * binaries are being transmitted through @c stdout. Some programs
* @e may have option to log these messages to a file instead.
*/
extern FILE* message_out;
-/**
+/**
* @brief set @c hfst_tool_version to version specific to the tool.
* @sa hfst_set_program_name
*/
extern const char* hfst_tool_version;
-/**
+/**
* @brief set @c hfst_tool_wikiname to name of the kitwiki page for this tool.
*/
extern const char* hfst_tool_wikiname;
@@ -86,7 +86,7 @@ void debug_printf(const char* format, ...);
/** print message @c s with parameters @c __VA_ARGS__ if debug is @a true. */
void verbose_printf(const char* format, ...);
-/**
+/**
* @brief set program's name and other infos for reusable messages defined
* below. This function must be called in beginning of main as the values are
* used in all error messages as well.
@@ -109,7 +109,7 @@ int conversion_type(hfst::ImplementationType type1, hfst::ImplementationType typ
/* Convert transducers into common format, if needed. */
void convert_transducers(hfst::HfstTransducer & first, hfst::HfstTransducer & second);
-/**
+/**
* @brief set @c program_name to program's executable name for error messages.
*/
extern const char* program_name;
@@ -124,9 +124,9 @@ void warning(int status, int error, const char* format , ...);
#endif
#ifndef HAVE_ERROR_AT_LINE
-/**
+/**
* @brief print standard formatted error message when parsing a file and exit
- * if needed.
+ * if needed.
* The use of this function is especially important since error
* highlighting of vim and emacs depends on this error format.
*/
@@ -152,7 +152,7 @@ void print_version();
/**
* @brief print standard short help message.
- *
+ *
* @sa http://www.gnu.org/software/womb/gnits/Help-Output.html#Help-Output
*/
void print_short_help();
@@ -263,7 +263,7 @@ size_t hfst_fread(void* ptr, size_t size, size_t nmemb, FILE* stream);
* error message end exit.
*/
size_t hfst_fwrite(void* ptr, size_t size, size_t nmemb, FILE* stream);
-/**
+/**
* @brief create temporary file or print informative error message and exit
* on failure.
*/
@@ -334,7 +334,7 @@ char* hfst_setlocale();
/**
* @brief determine if next transducers of given streams are compatible.
- *
+ *
* @deprecated all formats are compatible in HFST3
*/
int get_compatible_fst_format(std::istream& is1, std::istream& is2);
diff --git a/tools/src/hfst-compare.cc b/tools/src/hfst-compare.cc
index 71d7d8b..eb727f4 100644
--- a/tools/src/hfst-compare.cc
+++ b/tools/src/hfst-compare.cc
@@ -77,9 +77,9 @@ print_usage()
"\n"
"Examples:\n"
" $ %s cat.hfst dog.hfst\n"
- " cat.hfst[1] != dog.hfst[1]\n"
+ " cat.hfst[1] != dog.hfst[1]\n"
" $ %s cat.hfst cat.hfst\n"
- " cat.hfst[1] == cat.hfst[1]\n"
+ " cat.hfst[1] == cat.hfst[1]\n"
"\n",
program_name, program_name );
print_report_bugs();
@@ -103,7 +103,7 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_BINARY_SHORT "He",
long_options, &option_index);
if (-1 == c)
@@ -161,10 +161,10 @@ compare_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
{
free(secondname);
secondname = strdup(secondfilename);
- }
+ }
if (transducer_n_first == 1)
{
- verbose_printf("Comparing %s and %s...\n", firstname,
+ verbose_printf("Comparing %s and %s...\n", firstname,
secondname);
}
else
@@ -206,7 +206,7 @@ compare_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
{
if (! silent)
fprintf(outfile, "%s[" SIZE_T_SPECIFIER "] != %s[" SIZE_T_SPECIFIER "]\n",
- firstname, transducer_n_first,
+ firstname, transducer_n_first,
secondname, transducer_n_second);
}
mismatches++;
@@ -222,7 +222,7 @@ compare_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
hfst_strformat(secondstream.get_type()));
}
- continueReading = firststream.is_good() &&
+ continueReading = firststream.is_good() &&
(secondstream.is_good() || transducer_n_second == 1);
delete first;
@@ -242,7 +242,7 @@ compare_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
if (firststream.is_good())
{
error(EXIT_FAILURE, 0, "second input '%s' contains fewer transducers than first input '%s'; "
- "this is only possible if the second input contains exactly one transducer",
+ "this is only possible if the second input contains exactly one transducer",
secondfilename, firstfilename);
}
else if (secondstream.is_good())
@@ -285,7 +285,7 @@ int main( int argc, char **argv ) {
{
fclose(secondfile);
}
- verbose_printf("Reading from %s and %s, writing log to %s\n",
+ verbose_printf("Reading from %s and %s, writing log to %s\n",
firstfilename, secondfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* firststream = NULL;
diff --git a/tools/src/hfst-compiler.yy b/tools/src/hfst-compiler.yy
deleted file mode 100644
index 2b97ae2..0000000
--- a/tools/src/hfst-compiler.yy
+++ /dev/null
@@ -1,494 +0,0 @@
-%{
-/*******************************************************************/
-/* */
-/* FILE hfst-compiler.yy */
-/* MODULE hfst-compiler */
-/* PROGRAM HFST */
-/* */
-/*******************************************************************/
-
-#ifdef WINDOWS
-#include <io.h>
-#endif
-
-#include <stdio.h>
-#include "HfstCompiler.h"
-#include "HfstTransducer.h"
-
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <getopt.h>
-#include <errno.h>
-
-#include "hfst-commandline.h"
-#include "hfst-program-options.h"
-
-#include "inc/globals-common.h"
-#include "inc/globals-unary.h"
-
-extern char* FileName;
-extern bool Verbose;
-
-char * folder = NULL;
-
-using std::cerr;
-using namespace hfst;
-
-extern int yylineno;
-extern char *yytext;
-
-void yyerror(char *text);
-void warn(char *text);
-void warn2(const char *text, char *text2);
-int yylex( void );
-int yyparse( void );
-
-static int Switch=0;
-HfstCompiler * compiler;
-HfstTransducer * Result;
-
-static hfst::ImplementationType output_format = hfst::ERROR_TYPE; // = hfst::UNSPECIFIED_TYPE;
-bool DEBUG=false;
-
-
-%}
-
-/* Slight Hfst addition SFST::... */
-%union {
- int number;
- hfst::Twol_Type type;
- hfst::Repl_Type rtype;
- char *name;
- char *value;
- unsigned char uchar;
- unsigned int longchar;
- hfst::Character character;
- hfst::HfstTransducer *expression;
- hfst::Range *range;
- hfst::Ranges *ranges;
- hfst::Contexts *contexts;
-}
-
-%token <number> NEWLINE ALPHA COMPOSE PRINT POS INSERT SUBSTITUTE SWITCH
-%token <type> ARROW
-%token <rtype> REPLACE
-%token <name> SYMBOL VAR SVAR RVAR RSVAR
-%token <value> STRING STRING2 UTF8CHAR
-%token <uchar> CHARACTER
-
-%type <uchar> SCHAR
-%type <longchar> LCHAR
-%type <character> CODE
-%type <expression> RE
-%type <range> RANGE VALUE VALUES
-%type <ranges> RANGES
-%type <contexts> CONTEXT CONTEXT2 CONTEXTS CONTEXTS2
-
-%left PRINT INSERT SUBSTITUTE
-%left ARROW REPLACE
-%left COMPOSE
-%left '|'
-%left '-'
-%left '&'
-%left SEQ
-%left '!' '^' '_'
-%left '*' '+'
-%%
-
-ALL: ASSIGNMENTS RE NEWLINES { Result=compiler->result($2, Switch); }
- ;
-
-ASSIGNMENTS: ASSIGNMENTS ASSIGNMENT {}
- | ASSIGNMENTS NEWLINE {}
- | /* nothing */ {}
- ;
-
-ASSIGNMENT: VAR '=' RE { if (DEBUG) { printf("defining transducer variable \"%s\"..\n", $1); }; if (compiler->def_var($1,$3)) warn2("assignment of empty transducer to",$1); if(DEBUG) printf("done\n"); }
- | RVAR '=' RE { if (DEBUG) { printf("defining agreement transducer variable \"%s\"..\n", $1); }; if (compiler->def_rvar($1,$3)) warn2("assignment of empty transducer to",$1); }
- | SVAR '=' VALUES { if (DEBUG) { printf("defining range variable \"%s\"..\n", $1); }; if (compiler->def_svar($1,$3)) warn2("assignment of empty symbol range to",$1); }
- | RSVAR '=' VALUES { if (DEBUG) { printf("defining agreement range variable \"%s\"..\n", $1); }; if (compiler->def_svar($1,$3)) warn2("assignment of empty symbol range to",$1); }
- | RE PRINT STRING { compiler->write_to_file($1, folder, $3); }
- | ALPHA RE { if (DEBUG) { printf("defining alphabet..\n"); }; compiler->def_alphabet($2); delete $2; }
- ;
-
-RE: RE ARROW CONTEXTS2 { $$ = compiler->restriction($1,$2,$3,0); }
- | RE '^' ARROW CONTEXTS2 { $$ = compiler->restriction($1,$3,$4,1); }
- | RE '_' ARROW CONTEXTS2 { $$ = compiler->restriction($1,$3,$4,-1); }
- | RE REPLACE CONTEXT2 { $1 = compiler->explode($1); $1->minimize(); $$ = compiler->replace_in_context($1, $2, $3, false); }
- | RE REPLACE '?' CONTEXT2 { $1 = compiler->explode($1); $1->minimize(); $$ = compiler->replace_in_context($1, $2, $4, true); }
- | RE REPLACE '(' ')' { $1 = compiler->explode($1); $1->minimize(); $$ = compiler->replace($1, $2, false); }
- | RE REPLACE '?' '(' ')' { $1 = compiler->explode($1); $1->minimize(); $$ = compiler->replace($1, $2, true); }
- | RE RANGE ARROW RANGE RE { $$ = compiler->make_rule($1,$2,$3,$4,$5, output_format); }
- | RE RANGE ARROW RANGE { $$ = compiler->make_rule($1,$2,$3,$4,NULL, output_format); }
- | RANGE ARROW RANGE RE { $$ = compiler->make_rule(NULL,$1,$2,$3,$4, output_format); }
- | RANGE ARROW RANGE { $$ = compiler->make_rule(NULL,$1,$2,$3,NULL, output_format); }
- | RE COMPOSE RE { $1->compose(*$3); delete $3; $$ = $1; }
- | '{' RANGES '}' ':' '{' RANGES '}' { $$ = compiler->make_mapping($2,$6,output_format); }
- | RANGE ':' '{' RANGES '}' { $$ = compiler->make_mapping(compiler->add_range($1,NULL),$4,output_format); }
- | '{' RANGES '}' ':' RANGE { $$ = compiler->make_mapping($2,compiler->add_range($5,NULL),output_format); }
- | RE INSERT CODE ':' CODE { $$ = compiler->insert_freely($1,$3,$5); }
- | RE INSERT CODE { $$ = compiler->insert_freely($1,$3,$3); }
- | RE SUBSTITUTE CODE ':' CODE { $$ = compiler->substitute($1,$3,$5); }
- | RE SUBSTITUTE CODE ':' CODE ':' CODE ':' CODE { $$ = compiler->substitute($1,$3,$5,$7,$9); }
- | RE SUBSTITUTE CODE ':' CODE '(' RE ')' { $$ = compiler->substitute($1,$3,$5,$7); }
- | RANGE ':' RANGE { $$ = compiler->new_transducer($1,$3,output_format); }
- | RANGE { $$ = compiler->new_transducer($1,$1,output_format); }
- | VAR { if (DEBUG) { printf("calling transducer variable \"%s\"\n", $1); }; $$ = compiler->var_value($1); }
- | RVAR { if (DEBUG) { printf("calling agreement transducer variable \"%s\"\n", $1); }; $$ = compiler->rvar_value($1,output_format); }
- | RE '*' { $1->repeat_star(); $$ = $1; }
- | RE '+' { $1->repeat_plus(); $$ = $1; }
- | RE '?' { $1->optionalize(); $$ = $1; }
- | RE RE %prec SEQ { $1->concatenate(*$2); delete $2; $$ = $1; }
- | '!' RE { $$ = compiler->negation($2); }
- | SWITCH RE { $2->invert(); $$ = $2; }
- | '^' RE { $2->output_project(); $$ = $2; }
- | '_' RE { $2->input_project(); $$ = $2; }
- | RE '&' RE { $1->intersect(*$3); delete $3; $$ = $1; }
- | RE '-' RE { $1->subtract(*$3); delete $3; $$ = $1; }
- | RE '|' RE { $1->disjunct(*$3); delete $3; $$ = $1; }
- | '(' RE ')' { $$ = $2; }
- | STRING { $$ = compiler->read_words(folder, $1, output_format); }
- | STRING2 { try { $$ = compiler->read_transducer(folder, $1, output_format); } catch (HfstException e) { printf("\nAn error happened when reading file \"%s\"\n", $1); exit(1); } }
- ;
-
-RANGES: RANGE RANGES { $$ = compiler->add_range($1,$2); }
- | { $$ = NULL; }
- ;
-
-RANGE: '[' VALUES ']' { $$=$2; }
- | '[' '^' VALUES ']' { $$=compiler->complement_range($3); }
- | '[' RSVAR ']' { if (DEBUG) { printf("calling agreement range variable \"%s\"\n", $2); }; $$=compiler->rsvar_value($2); }
- | '.' { $$=NULL; }
- | CODE { $$=compiler->add_value($1,NULL); }
- ;
-
-CONTEXTS2: CONTEXTS { $$ = $1; }
- | '(' CONTEXTS ')' { $$ = $2; }
- ;
-
-CONTEXTS: CONTEXT ',' CONTEXTS { $$ = compiler->add_context($1,$3); }
- | CONTEXT { $$ = $1; }
- ;
-
-CONTEXT2: CONTEXT { $$ = $1; }
- | '(' CONTEXT ')' { $$ = $2; }
- ;
-
-CONTEXT : RE POS RE { $$ = compiler->make_context($1, $3); }
- | POS RE { $$ = compiler->make_context(NULL, $2); }
- | RE POS { $$ = compiler->make_context($1, NULL); }
- ;
-
-VALUES: VALUE VALUES { $$=compiler->append_values($1,$2); }
- | VALUE { $$ = $1; }
- ;
-
-VALUE: LCHAR '-' LCHAR { $$=compiler->add_values($1,$3,NULL); }
- | SVAR { if (DEBUG) { printf("calling range variable \"%s\"", $1); }; $$=compiler->svar_value($1); }
- | LCHAR { $$=compiler->add_value(compiler->character_code($1),NULL); }
- | CODE { $$=compiler->add_value($1,NULL); }
- | SCHAR { $$=compiler->add_value($1,NULL); }
- ;
-
-LCHAR: CHARACTER { $$=$1; }
- | UTF8CHAR { $$=compiler->utf8toint($1); free($1); }
- | SCHAR { $$=$1; }
- ;
-
-CODE: CHARACTER { $$=compiler->character_code($1); }
- | UTF8CHAR { $$=compiler->symbol_code($1); }
- | SYMBOL { $$=compiler->symbol_code($1); }
- ;
-
-SCHAR: '.' { $$=(unsigned char)compiler->character_code('.'); }
- | '!' { $$=(unsigned char)compiler->character_code('!'); }
- | '?' { $$=(unsigned char)compiler->character_code('?'); }
- | '{' { $$=(unsigned char)compiler->character_code('{'); }
- | '}' { $$=(unsigned char)compiler->character_code('}'); }
- | ')' { $$=(unsigned char)compiler->character_code(')'); }
- | '(' { $$=(unsigned char)compiler->character_code('('); }
- | '&' { $$=(unsigned char)compiler->character_code('&'); }
- | '|' { $$=(unsigned char)compiler->character_code('|'); }
- | '*' { $$=(unsigned char)compiler->character_code('*'); }
- | '+' { $$=(unsigned char)compiler->character_code('+'); }
- | ':' { $$=(unsigned char)compiler->character_code(':'); }
- | ',' { $$=(unsigned char)compiler->character_code(','); }
- | '=' { $$=(unsigned char)compiler->character_code('='); }
- | '_' { $$=(unsigned char)compiler->character_code('_'); }
- | '^' { $$=(unsigned char)compiler->character_code('^'); }
- | '-' { $$=(unsigned char)compiler->character_code('-'); }
- ;
-
-NEWLINES: NEWLINE NEWLINES {}
- | /* nothing */ {}
- ;
-
-%%
-
-extern FILE *yyin;
-static int Compact=0;
-static int LowMem=0;
-
-/*******************************************************************/
-/* */
-/* yyerror */
-/* */
-/*******************************************************************/
-
-void yyerror(char *text)
-
-{
- cerr << "\n" << FileName << ":" << yylineno << ": " << text << " at: ";
- cerr << yytext << "\naborted.\n";
- exit(1);
-}
-
-
-/*******************************************************************/
-/* */
-/* warn */
-/* */
-/*******************************************************************/
-
-void warn(char *text)
-
-{
- cerr << "\n" << FileName << ":" << yylineno << ": warning: " << text << "!\n";
-}
-
-
-/*******************************************************************/
-/* */
-/* warn2 */
-/* */
-/*******************************************************************/
-
-void warn2(const char *text, char *text2) // HFST: added const
-
-{
- cerr << "\n" << FileName << ":" << yylineno << ": warning: " << text << ": ";
- cerr << text2 << "\n";
-}
-
-
-/* print_usage */
-
-void
-print_usage()
-{
- // c.f. http://www.gnu.org/prep/standards/standards.html#g_t_002d_002dhelp
- fprintf(message_out, "Usage: %s %s [OPTIONS...] [INFILE]\n"
- "Compile a file written with SFST programming language into a transducer.\n"
- "\n", program_name, "(alias hfst-calculate)");
- print_common_program_options(message_out);
- fprintf(message_out, "Input/Output options:\n"
- " -i, --input=INFILE Read input from INFILE\n"
- " -o, --output=OUTFILE Write output transducer to OUTFILE\n");
- fprintf(message_out, "String and format options:\n"
- " -f, --format=FMT Write result in FMT format\n");
- fprintf(message_out, "\n");
-
- fprintf(message_out,
- "If OUTFILE or INFILE is missing or -, standard streams will be used.\n"
- "FMT must be one of the following: "
- "{foma, sfst, openfst-tropical, openfst-log}\n"
- );
- fprintf(message_out, "\n");
- fprintf(message_out, "Examples:\n"
- " echo \"[a-z]*\" | %s -f foma > az.foma\n"
- " create a foma transducer accepting any number of consecutive\n"
- " characters between a and z, inclusive\n"
- "\n"
- " echo \"a:b (a:<>)+\" > ab.sfst-pl ; %s -f openfst-tropical \\\n"
- " -i ab.sfst-pl -o ab.tropical\n"
- " create a tropical OpenFst transducer that accepts two or more\n"
- " consecutive 'a's and maps them into string \"b\"\n"
- "\n", program_name, program_name);
- print_report_bugs();
- print_more_info();
- fprintf(message_out, "\n");
-}
-
-
-int
-parse_options(int argc, char** argv)
-{
- // use of this function requires options are settable on global scope
- while (true)
- {
- static const struct option long_options[] =
- {
- HFST_GETOPT_COMMON_LONG,
- HFST_GETOPT_UNARY_LONG,
- {"format", required_argument, 0, 'f'},
- {"harmonize-smaller", no_argument, 0, 'H'},
- {"weighted", no_argument, 0, 'w'},
- {0,0,0,0}
- };
- int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
- HFST_GETOPT_UNARY_SHORT "f:Hw",
- long_options, &option_index);
- if (-1 == c)
- {
- break;
- }
-
- switch (c)
- {
-#include "inc/getopt-cases-common.h"
-#include "inc/getopt-cases-unary.h"
- case 'f':
- output_format = hfst_parse_format_name(optarg);
- break;
- case 'H':
- hfst::set_harmonize_smaller(true);
- break;
- case'w':
- verbose_printf(
- "Interpreting deprecated -w switch as openfst tropical format\n");
- output_format = hfst::TROPICAL_OPENFST_TYPE;
- break;
-
-#include "inc/getopt-cases-error.h"
- }
- }
-
-#include "inc/check-params-common.h"
-#include "inc/check-params-unary.h"
- Verbose = verbose;
- if (output_format == hfst::ERROR_TYPE)
- {
- verbose_printf("Output format not specified, "
- "defaulting to openfst tropical\n");
- output_format = hfst::TROPICAL_OPENFST_TYPE;
- }
- FileName = strdup(inputfilename);
-
- if (NULL != inputfilename && strcmp(inputfilename, "<stdin>") != 0)
- {
- std::string str(inputfilename);
- size_t found = str.find_last_of("/\\");
- if (found != std::string::npos)
- folder = strdup(str.substr(0,found).c_str());
- }
-
- return EXIT_CONTINUE;
-}
-
-
-/*******************************************************************/
-/* */
-/* get_flags */
-/* */
-/*******************************************************************/
-
-void get_flags( int *argc, char **argv )
-
-{
- for( int i=1; i<*argc; i++ ) {
- if (strcmp(argv[i],"-c") == 0) {
- Compact = 1;
- argv[i] = NULL;
- }
- else if (strcmp(argv[i],"-l") == 0) {
- LowMem = 1;
- argv[i] = NULL;
- }
- else if (strcmp(argv[i],"-q") == 0) {
- Verbose = 0;
- argv[i] = NULL;
- }
- else if (strcmp(argv[i],"-h") == 0 || strcmp(argv[i],"--help") == 0) {
- print_usage();
- exit(0);
- }
- else if (strcmp(argv[i],"-v") == 0 || strcmp(argv[i],"--version") == 0) {
- fprintf(stdout, "hfst-sfstpl2fst 0.1 (hfst 3.0)\n");
- exit(0);
- }
- else if (strcmp(argv[i],"-s") == 0) {
- Switch = 1;
- argv[i] = NULL;
- }
- // hfst addition
- else if (strcmp(argv[i],"-sfst") == 0) {
- output_format = SFST_TYPE;
- argv[i] = NULL;
- }
- else if (strcmp(argv[i],"-tropical") == 0) {
- output_format = TROPICAL_OPENFST_TYPE;
- argv[i] = NULL;
- }
- else if (strcmp(argv[i],"-log") == 0) {
- output_format = LOG_OPENFST_TYPE;
- argv[i] = NULL;
- }
- else if (strcmp(argv[i],"-foma") == 0) {
- output_format = FOMA_TYPE;
- argv[i] = NULL;
- }
- }
- // remove flags from the argument list
- int k;
- for( int i=k=1; i<*argc; i++)
- if (argv[i] != NULL)
- argv[k++] = argv[i];
- *argc = k;
-}
-
-
-/*******************************************************************/
-/* */
-/* main */
-/* */
-/*******************************************************************/
-
-int main( int argc, char *argv[] )
-
-{
-#ifdef WINDOWS
- _setmode(1, _O_BINARY);
-#endif
-
- //FILE *file;
-
- hfst_set_program_name(argv[0], "0.1", "HfstSfstPl2Fst");
- int retval = parse_options(argc, argv);
- if (retval != EXIT_CONTINUE)
- return retval;
-
- yyin = inputfile;
- if (strcmp(outfilename,"<stdout>") != 0)
- fclose(outfile); // stream is used when writing the result
-
- // Unknown symbols cannot be used in SFST-PL syntax.
- // If the HFST library is aware of this, some optimization can be done.
- hfst::set_unknown_symbols_in_use(false);
- compiler = new HfstCompiler(output_format, Verbose);
-
- try {
- yyparse();
- fclose(inputfile);
- try {
- if (strcmp(outfilename,"<stdout>") == 0)
- compiler->write_to_file(Result, NULL, "");
- else
- compiler->write_to_file(Result, NULL, strdup(outfilename));
- } catch (HfstException e) {
- printf("\nAn error happened when writing to file \"%s\"\n", outfilename); }
- //printf("type is: %i\n", Result->get_type());
- delete Result;
- // fprintf(stderr, "seconds spent in SFST::harmonize: %f\n", HfstTransducer::get_profile_seconds(output_format));
- // delete compiler;
- }
- catch(const char* p) {
- cerr << "\n" << p << "\n\n";
- exit(1);
- }
- if (NULL != folder)
- free(folder);
-
- exit(0);
-}
diff --git a/tools/src/hfst-compose-intersect.cc b/tools/src/hfst-compose-intersect.cc
index 7595eaf..166cec4 100644
--- a/tools/src/hfst-compose-intersect.cc
+++ b/tools/src/hfst-compose-intersect.cc
@@ -83,11 +83,11 @@ print_usage()
);
//print_common_binary_program_parameter_instructions(message_out);
fprintf(message_out,
-"\nIf OUTFILE, or either INFILE1 or INFILE2 is missing or -, standard\n"
+"\nIf OUTFILE, or either INFILE1 or INFILE2 is missing or -, standard\n"
"streams will be used. INFILE1, INFILE2, or both, must be specified\n"
-"The format of INFILE1 and INFILE2 must be the same; the result will\n"
+"The format of INFILE1 and INFILE2 must be the same; the result will\n"
"have the same format as these.\n"
-"INFILE1 (the lexicon) must contain exactly one transducer.\n"
+"INFILE1 (the lexicon) must contain exactly one transducer.\n"
"INFILE2 (rule file) may contain several transducers.\n");
fprintf(message_out,
"\n"
@@ -119,7 +119,7 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_BINARY_SHORT "FIeHfa",
long_options, &option_index);
if (-1 == c)
@@ -169,13 +169,13 @@ std::string check_all_symbols(const HfstTransducer &lexicon,
for (HfstState s = 0; s <= rule_b.get_max_state(); ++s)
{
- for (HfstBasicTransducer::HfstTransitions::const_iterator it =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator it =
rule_b[s].begin();
it != rule_b[s].end();
++it)
- {
+ {
const std::string &input_symbol = it->get_input_symbol();
- rule_input_symbols.insert(input_symbol);
+ rule_input_symbols.insert(input_symbol);
}
}
@@ -186,11 +186,11 @@ std::string check_all_symbols(const HfstTransducer &lexicon,
for (HfstState s = 0; s <= lexicon_b.get_max_state(); ++s)
{
- for (HfstBasicTransducer::HfstTransitions::const_iterator it =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator it =
lexicon_b[s].begin();
it != lexicon_b[s].end();
++it)
- {
+ {
const std::string &output_symbol = it->get_output_symbol();
if (rule_input_symbols.count(output_symbol) == 0)
@@ -213,23 +213,23 @@ std::string check_multi_char_symbols
for (HfstState s = 0; s <= rule_b.get_max_state(); ++s)
{
- for (HfstBasicTransducer::HfstTransitions::const_iterator it =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator it =
rule_b[s].begin();
it != rule_b[s].end();
++it)
- {
+ {
const std::string &input_symbol = it->get_input_symbol();
- rule_input_symbols.insert(input_symbol);
+ rule_input_symbols.insert(input_symbol);
}
}
for (HfstState s = 0; s <= lexicon_b.get_max_state(); ++s)
{
- for (HfstBasicTransducer::HfstTransitions::const_iterator it =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator it =
lexicon_b[s].begin();
it != lexicon_b[s].end();
++it)
- {
+ {
const std::string &output_symbol = it->get_output_symbol();
if (rule_input_symbols.count(output_symbol) == 0)
@@ -305,7 +305,7 @@ compose_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
}
HfstTransducerVector rules;
- size_t rule_n = 1;
+ size_t rule_n = 1;
while (secondstream.is_good()) {
HfstTransducer rule(secondstream);
@@ -332,11 +332,11 @@ compose_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
hfst::set_encode_weights(enc);
}
- rules.push_back(rule);
+ rules.push_back(rule);
rule_n++;
}
- while (firststream.is_good())
+ while (firststream.is_good())
{
verbose_printf("Reading lexicon...");
HfstTransducer lexicon(firststream);
@@ -351,7 +351,7 @@ compose_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
std::string symbol;
if ((symbol = check_all_symbols(lexicon,rules[0])) != "")
{
- warning(0, 0,
+ warning(0, 0,
"\nFound output symbols (e.g. \"%s\") in transducer in\n"
"file %s which will be filtered out because they are\n"
"not found on the input tapes of transducers in file\n"
@@ -359,8 +359,8 @@ compose_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
symbol.c_str(), firstfilename, secondfilename);
}
else if ((symbol = check_multi_char_symbols(lexicon,rules[0])) != "")
- {
- warning(0, 0,
+ {
+ warning(0, 0,
"\nFound output multi-char symbols (\"%s\") in \n"
"transducer in file %s which are not found on the\n"
"input tapes of transducers in file %s.",
@@ -401,12 +401,12 @@ compose_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
lexicon.compose_intersect(rules,invert);
}
- char* composed_name = static_cast<char*>(malloc(sizeof(char) *
+ char* composed_name = static_cast<char*>(malloc(sizeof(char) *
(strlen(lexiconname) +
strlen(secondfilename) +
- strlen("compose(%s, interserct(%s))"))
+ strlen("compose(%s, interserct(%s))"))
+ 1));
- if (sprintf(composed_name, "compose(%s, intersect(%s))",
+ if (sprintf(composed_name, "compose(%s, intersect(%s))",
lexiconname, secondfilename) > 0)
{
lexicon.set_name(composed_name);
@@ -448,7 +448,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s and %s, writing to %s\n",
+ verbose_printf("Reading from %s and %s, writing to %s\n",
firstfilename, secondfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* firststream = NULL;
diff --git a/tools/src/hfst-compose.cc b/tools/src/hfst-compose.cc
index d801c83..9101abd 100644
--- a/tools/src/hfst-compose.cc
+++ b/tools/src/hfst-compose.cc
@@ -105,7 +105,7 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_BINARY_SHORT "FHx:X:",
long_options, &option_index);
if (-1 == c)
@@ -228,7 +228,7 @@ compose_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
char* secondname = hfst_get_name(*second, secondfilename);
if (transducer_n_first == 1)
{
- verbose_printf("Composing %s and %s...\n", firstname,
+ verbose_printf("Composing %s and %s...\n", firstname,
secondname);
}
else
@@ -237,20 +237,20 @@ compose_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
firstname, secondname, transducer_n_first);
}
- if (first->has_flag_diacritics() or second->has_flag_diacritics())
+ if (first->has_flag_diacritics() or second->has_flag_diacritics())
{
if (not harmonize_flags)
{
- if (not silent)
+ if (not silent)
{
warning(0, 0, "At least one of the arguments contains "
- "flag diacritics. Use -F to harmonize them.",
+ "flag diacritics. Use -F to harmonize them.",
secondname, firstname);
}
}
else
{
- try
+ try
{
first->harmonize_flag_diacritics(*second);
}
@@ -299,7 +299,7 @@ compose_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
outstream << *first;
- continueReading =
+ continueReading =
(firststream.is_good() && secondstream.is_good()) ||
(firststream.is_good() && (transducer_n_second == 1)) ||
((transducer_n_first == 1) && secondstream.is_good());
@@ -321,13 +321,13 @@ compose_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
}
}
- //continueReading = firststream.is_good() &&
+ //continueReading = firststream.is_good() &&
// (secondstream.is_good() || transducer_n_second == 1);
//delete first;
//first=0;
// delete the transducer of second stream, unless we continue reading
- // the first stream and there is only one transducer in the second
+ // the first stream and there is only one transducer in the second
// stream
//if ((continueReading && secondstream.is_good()) || not continueReading)
// {
@@ -341,19 +341,19 @@ compose_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
if (firststream.is_good())
{
- error(EXIT_FAILURE, 0,
+ error(EXIT_FAILURE, 0,
"second input '%s' contains fewer transducers than first input"
" '%s'; this is only possible if the second input contains"
- " exactly one transducer",
+ " exactly one transducer",
secondfilename, firstfilename);
}
if (secondstream.is_good())
{
- error(EXIT_FAILURE, 0,
+ error(EXIT_FAILURE, 0,
"first input '%s' contains fewer transducers than second input"
" '%s'; this is only possible if the first input contains"
- " exactly one transducer",
+ " exactly one transducer",
firstfilename, secondfilename);
}
@@ -390,7 +390,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s and %s, writing to %s\n",
+ verbose_printf("Reading from %s and %s, writing to %s\n",
firstfilename, secondfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* firststream = NULL;
@@ -413,7 +413,7 @@ int main( int argc, char **argv ) {
new HfstOutputStream(outfilename, firststream->get_type()) :
new HfstOutputStream(firststream->get_type());
- if ( is_input_stream_in_ol_format(firststream, "hfst-compose") ||
+ if ( is_input_stream_in_ol_format(firststream, "hfst-compose") ||
is_input_stream_in_ol_format(secondstream, "hfst-compose") )
{
return EXIT_FAILURE;
diff --git a/tools/src/hfst-concatenate.cc b/tools/src/hfst-concatenate.cc
index 4130ba0..0e80e18 100644
--- a/tools/src/hfst-concatenate.cc
+++ b/tools/src/hfst-concatenate.cc
@@ -96,7 +96,7 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_BINARY_SHORT "FH",
long_options, &option_index);
if (-1 == c)
@@ -181,7 +181,7 @@ concatenate_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
char* secondname = hfst_get_name(*second, secondfilename);
if (transducer_n_first == 1)
{
- verbose_printf("Concatenating %s and %s...\n", firstname,
+ verbose_printf("Concatenating %s and %s...\n", firstname,
secondname);
}
else
@@ -189,14 +189,14 @@ concatenate_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
verbose_printf("Concatenating %s and %s... " SIZE_T_SPECIFIER "\n", firstname,
secondname, transducer_n_first);
}
- if (first->has_flag_diacritics() and second->has_flag_diacritics())
+ if (first->has_flag_diacritics() and second->has_flag_diacritics())
{
if (not harmonize_flags)
{
- if (not silent)
+ if (not silent)
{
warning(0, 0, "The argumentes contain "
- "flag diacritics. Use -F to harmonize them.",
+ "flag diacritics. Use -F to harmonize them.",
secondname, firstname);
}
}
@@ -229,13 +229,13 @@ concatenate_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
hfst_set_formula(*first, *first, *second, "⋅");
outstream << *first;
- continueReading = firststream.is_good() &&
+ continueReading = firststream.is_good() &&
(secondstream.is_good() || transducer_n_second == 1);
delete first;
first=0;
// delete the transducer of second stream, unless we continue reading
- // the first stream and there is only one transducer in the second
+ // the first stream and there is only one transducer in the second
// stream
if ((continueReading && secondstream.is_good()) || not continueReading)
{
@@ -249,10 +249,10 @@ concatenate_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
if (firststream.is_good())
{
- error(EXIT_FAILURE, 0,
+ error(EXIT_FAILURE, 0,
"second input '%s' contains fewer transducers than first input"
" '%s'; this is only possible if the second input contains"
- " exactly one transducer",
+ " exactly one transducer",
secondfilename, firstfilename);
}
@@ -295,7 +295,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s and %s, writing to %s\n",
+ verbose_printf("Reading from %s and %s, writing to %s\n",
firstfilename, secondfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* firststream = NULL;
diff --git a/tools/src/hfst-conjunct.cc b/tools/src/hfst-conjunct.cc
index f3b524c..7d025fb 100644
--- a/tools/src/hfst-conjunct.cc
+++ b/tools/src/hfst-conjunct.cc
@@ -95,7 +95,7 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_BINARY_SHORT "FH",
long_options, &option_index);
if (-1 == c)
@@ -178,7 +178,7 @@ conjunct_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
char* secondname = hfst_get_name(*second, secondfilename);
if (transducer_n_first == 1)
{
- verbose_printf("Intersecting %s and %s...\n", firstname,
+ verbose_printf("Intersecting %s and %s...\n", firstname,
secondname);
}
else
@@ -187,14 +187,14 @@ conjunct_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
firstname, secondname, transducer_n_first);
}
- if (first->has_flag_diacritics() or second->has_flag_diacritics())
+ if (first->has_flag_diacritics() or second->has_flag_diacritics())
{
if (not harmonize_flags)
{
- if (not silent)
+ if (not silent)
{
warning(0, 0, "At least one of the argumentes contains "
- "flag diacritics. Use -F to harmonize them.",
+ "flag diacritics. Use -F to harmonize them.",
secondname, firstname);
}
}
@@ -227,13 +227,13 @@ conjunct_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
hfst_set_formula(*first, *first, *second, "∩");
outstream << *first;
- continueReading = firststream.is_good() &&
+ continueReading = firststream.is_good() &&
(secondstream.is_good() || transducer_n_second == 1);
delete first;
first=0;
// delete the transducer of second stream, unless we continue reading
- // the first stream and there is only one transducer in the second
+ // the first stream and there is only one transducer in the second
// stream
if ((continueReading && secondstream.is_good()) || not continueReading)
{
@@ -247,10 +247,10 @@ conjunct_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
if (firststream.is_good())
{
- error(EXIT_FAILURE, 0,
+ error(EXIT_FAILURE, 0,
"second input '%s' contains fewer transducers than first input"
" '%s'; this is only possible if the second input contains"
- " exactly one transducer",
+ " exactly one transducer",
secondfilename, firstfilename);
}
@@ -292,7 +292,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s and %s, writing to %s\n",
+ verbose_printf("Reading from %s and %s, writing to %s\n",
firstfilename, secondfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* firststream = NULL;
diff --git a/tools/src/hfst-determinize.cc b/tools/src/hfst-determinize.cc
index 5c5010d..c5f2870 100644
--- a/tools/src/hfst-determinize.cc
+++ b/tools/src/hfst-determinize.cc
@@ -87,12 +87,12 @@ parse_options(int argc, char** argv)
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
// add tool-specific options here
- {"encode-weights", no_argument, 0, 'E'},
+ {"encode-weights", no_argument, 0, 'E'},
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "E",
long_options, &option_index);
if (-1 == c)
@@ -131,12 +131,12 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
char* inputname = hfst_get_name(trans, inputfilename);
if (transducer_n==1)
{
- verbose_printf("Determinizing %s...\n", inputname);
+ verbose_printf("Determinizing %s...\n", inputname);
}
else
{
verbose_printf("Determinizing %s..." SIZE_T_SPECIFIER "\n", inputname,
- transducer_n);
+ transducer_n);
}
trans.determinize();
hfst_set_name(trans, trans, "determinize");
@@ -177,7 +177,7 @@ int main( int argc, char **argv ) {
hfst::set_encode_weights(true);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
diff --git a/tools/src/hfst-disjunct.cc b/tools/src/hfst-disjunct.cc
index 1c395e8..65a53d3 100644
--- a/tools/src/hfst-disjunct.cc
+++ b/tools/src/hfst-disjunct.cc
@@ -94,7 +94,7 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_BINARY_SHORT "H",
long_options, &option_index);
if (-1 == c)
@@ -175,7 +175,7 @@ disjunct_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
char* secondname = hfst_get_name(*second, secondfilename);
if (transducer_n_first == 1)
{
- verbose_printf("Disjuncting %s and %s...\n", firstname,
+ verbose_printf("Disjuncting %s and %s...\n", firstname,
secondname);
}
else
@@ -207,13 +207,13 @@ disjunct_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
hfst_set_formula(*first, *first, *second, "∪");
outstream << *first;
- continueReading = firststream.is_good() &&
+ continueReading = firststream.is_good() &&
(secondstream.is_good() || transducer_n_second == 1);
delete first;
first=0;
// delete the transducer of second stream, unless we continue reading
- // the first stream and there is only one transducer in the second
+ // the first stream and there is only one transducer in the second
// stream
if ((continueReading && secondstream.is_good()) || not continueReading)
{
@@ -228,10 +228,10 @@ disjunct_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
if (firststream.is_good())
{
- error(EXIT_FAILURE, 0,
+ error(EXIT_FAILURE, 0,
"second input '%s' contains fewer transducers than first input"
" '%s'; this is only possible if the second input contains"
- " exactly one transducer",
+ " exactly one transducer",
secondfilename, firstfilename);
}
@@ -273,7 +273,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s and %s, writing to %s\n",
+ verbose_printf("Reading from %s and %s, writing to %s\n",
firstfilename, secondfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* firststream = NULL;
diff --git a/tools/src/hfst-edit-metadata.cc b/tools/src/hfst-edit-metadata.cc
index 9e44ccb..5dc3a26 100644
--- a/tools/src/hfst-edit-metadata.cc
+++ b/tools/src/hfst-edit-metadata.cc
@@ -105,8 +105,8 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "a:p::t:",
long_options, &option_index);
if (-1 == c)
@@ -166,19 +166,19 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
{
transducer_n++;
- if ((transducer_n > 1) && (print_all_properties ||
+ if ((transducer_n > 1) && (print_all_properties ||
(print_property != NULL))) {
std::cerr << "--- \n";
}
if (transducer_n==1)
{
- verbose_printf("Metadata %s...\n", inputfilename);
+ verbose_printf("Metadata %s...\n", inputfilename);
}
else
{
- verbose_printf("Metadata %s..." SIZE_T_SPECIFIER "\n",
- inputfilename, transducer_n);
+ verbose_printf("Metadata %s..." SIZE_T_SPECIFIER "\n",
+ inputfilename, transducer_n);
}
HfstTransducer trans(instream);
@@ -200,7 +200,7 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
"semantics for header;\n"
"use with caution");
}
- else if (prop->first == "character-encoding" &&
+ else if (prop->first == "character-encoding" &&
!((prop->second == "utf-8") || (prop->second == "UTF-8")))
{
error(EXIT_FAILURE, 0, "Cannot set `character-encoding' "
@@ -274,7 +274,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-expand-equivalences.cc b/tools/src/hfst-expand-equivalences.cc
index 809d076..6756213 100644
--- a/tools/src/hfst-expand-equivalences.cc
+++ b/tools/src/hfst-expand-equivalences.cc
@@ -119,7 +119,7 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "f:t:a:T:l:",
long_options, &option_index);
if (-1 == c)
@@ -160,9 +160,9 @@ parse_options(int argc, char** argv)
{
level = FSA_LEVEL_BOTH;
}
- else
+ else
{
- error(EXIT_FAILURE, 0,
+ error(EXIT_FAILURE, 0,
"The option for level parameter must"
" be one of:\n"
"upper, first, input; second, lower, output; both, "
@@ -365,7 +365,7 @@ process_stream(HfstInputStream* instream, HfstOutputStream* outstream)
{
xmlChar* to = xmlGetProp(equivNode,
reinterpret_cast<const xmlChar*>("value"));
- add_extension(extensions,
+ add_extension(extensions,
reinterpret_cast<const char*>(from),
reinterpret_cast<const char*>(to));
}
@@ -415,7 +415,7 @@ process_stream(HfstInputStream* instream, HfstOutputStream* outstream)
}
-int main( int argc, char **argv )
+int main( int argc, char **argv )
{
#ifdef WINDOWS
_setmode(0, _O_BINARY);
@@ -438,7 +438,7 @@ int main( int argc, char **argv )
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
@@ -457,7 +457,7 @@ int main( int argc, char **argv )
if ( is_input_stream_in_ol_format(instream, "hfst-expand-equivalences"))
{
return EXIT_FAILURE;
- }
+ }
process_stream(instream, outstream);
delete instream;
diff --git a/tools/src/hfst-file-to-mem.cc b/tools/src/hfst-file-to-mem.cc
index f7fa0d0..08da8c1 100644
--- a/tools/src/hfst-file-to-mem.cc
+++ b/tools/src/hfst-file-to-mem.cc
@@ -18,8 +18,8 @@ char * hfst_stdin_to_mem()
return NULL;
}
- while(true)
- {
+ while(true)
+ {
*(buffer + numbytes) = fgetc(stdin);
if (feof(stdin))
{
@@ -36,10 +36,10 @@ char * hfst_stdin_to_mem()
return buffer;
}
-// Based on a function in foma written by Mans Hulden.
-// Read the file 'filename' to memory and return a pointer to it.
-// Filename "<stdin>" uses stdin for reading.
-// Returns NULL if file cannot be opened or read or memory cannot be allocated.
+// Based on a function in foma written by Mans Hulden.
+// Read the file 'filename' to memory and return a pointer to it.
+// Filename "<stdin>" uses stdin for reading.
+// Returns NULL if file cannot be opened or read or memory cannot be allocated.
char * hfst_file_to_mem(const char *filename) {
diff --git a/tools/src/hfst-flookup.cc b/tools/src/hfst-flookup.cc
index 165c829..53100ad 100644
--- a/tools/src/hfst-flookup.cc
+++ b/tools/src/hfst-flookup.cc
@@ -54,7 +54,7 @@ using hfst::hfst_fprintf_console;
#include "HfstTransducer.h"
#include "HfstInputStream.h"
#include "HfstOutputStream.h"
-#include "implementations/HfstTransitionGraph.h"
+#include "implementations/HfstBasicTransducer.h"
#include "inc/globals-common.h"
#include "inc/globals-unary.h"
@@ -171,7 +171,7 @@ static const char* XEROX_END_SETF = "%n";
static const char* XEROX_EMPTY_BEGIN_SETF = "";
static const char* XEROX_EMPTY_LOOKUPF = "%i\t%i+?\t%w%n";
static const char* XEROX_EMPTY_END_SETF = "%n";
-// ¶ ¶+?
+// ¶ ¶+?
static const char* XEROX_UNKNOWN_BEGIN_SETF = "";
static const char* XEROX_UNKNOWN_LOOKUPF = "%i\t%i+?\t%w%n";
static const char* XEROX_UNKNOWN_END_SETF = "%n";
@@ -237,7 +237,7 @@ print_usage()
"\n", program_name);
print_common_program_options(message_out);
- fprintf(message_out,
+ fprintf(message_out,
"Input/Output options:\n"
" -i, --input=INFILE Read input transducer from INFILE\n"
" -o, --output=OUTFILE Write output to OUTFILE\n"
@@ -260,7 +260,7 @@ print_usage()
" -f, --force-ol Force lookup of optimized lookup transducers (slow)\n");
fprintf(message_out, "\n");
print_common_unary_program_parameter_instructions(message_out);
- fprintf(message_out,
+ fprintf(message_out,
"OFORMAT is one of {xerox,cg,apertium}, xerox being default\n"
"IFORMAT is one of {text,spaced,apertium}, default being text,\n"
"unless OFORMAT is apertium\n"
@@ -287,7 +287,7 @@ print_usage()
#endif
fprintf(message_out, "\n");
-/* fprintf(message_out,
+/* fprintf(message_out,
"Todo:\n"
" For optimized lookup format, only strings that pass "
"flag diacritic checks\n"
@@ -335,8 +335,8 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "I:O:F:xc:X:e:E:b:t:p::PRf",
long_options, &option_index);
if (-1 == c)
@@ -446,7 +446,7 @@ parse_options(int argc, char** argv)
{
obey_flags = false;
}
- else
+ else
{
error(EXIT_FAILURE, 0, "Xfst variable %s unrecognised",
optarg);
@@ -804,7 +804,7 @@ lookup_printf(const char* format, const HfstOneLevelPath* input,
// give up trying
anal_start = lookupform;
}
- b = static_cast<char*>(calloc(sizeof(char),
+ b = static_cast<char*>(calloc(sizeof(char),
anal_start - lookupform + 1));
b = static_cast<char*>(memcpy(b, lookupform, anal_start - lookupform));
a = static_cast<char*>(calloc(sizeof(char),
@@ -873,7 +873,7 @@ lookup_printf(const char* format, const HfstOneLevelPath* input,
src++;
}
else if (*src == 'w')
- {
+ {
int skip = 0;
#ifdef _MSC_VER
if (w == std::numeric_limits<float>::infinity())
@@ -1025,7 +1025,7 @@ line_to_lookup_path(char** s, hfst::HfstStrings2FstTokenizer& tok,
std::string S = escape_special_characters(*s);
- StringPairVector spv
+ StringPairVector spv
= tok.tokenize_string_pair(S, true);
for (StringPairVector::const_iterator it = spv.begin();
@@ -1045,7 +1045,7 @@ line_to_lookup_path(char** s, hfst::HfstStrings2FstTokenizer& tok,
{
std::string S = escape_special_characters(*s);
- StringPairVector spv
+ StringPairVector spv
= tok.tokenize_string_pair(S, false);
for (StringPairVector::const_iterator it = spv.begin();
@@ -1059,7 +1059,7 @@ line_to_lookup_path(char** s, hfst::HfstStrings2FstTokenizer& tok,
}
case APERTIUM_INPUT:
{
- char* real_s
+ char* real_s
= static_cast<char*>(calloc(sizeof(char),strlen(*s)+1));
*markup = static_cast<char*>(calloc(sizeof(char), strlen(*s)+1));
@@ -1158,7 +1158,7 @@ lookup_simple(const HfstOneLevelPath& s, HfstTransducer& t, bool* infinity)
/* Replace all strings \a str1 in \a symbol with \a str2. */
-static std::string replace_all(std::string symbol,
+static std::string replace_all(std::string symbol,
const std::string &str1,
const std::string &str2)
{
@@ -1168,20 +1168,20 @@ static std::string replace_all(std::string symbol,
symbol.erase(pos, str1.size()); // erase str1
symbol.insert(pos, str2); // insert str2 instead
pos = symbol.find // find next str1
- (str1, pos+str2.size());
+ (str1, pos+str2.size());
}
return symbol;
}
-static std::string get_print_format(const std::string &s)
+static std::string get_print_format(const std::string &s)
{
if (is_epsilon(s))
return std::string(strdup(epsilon_format));
- if (quote_special)
+ if (quote_special)
{
- return
+ return
replace_all
( replace_all
( replace_all
@@ -1194,7 +1194,7 @@ static std::string get_print_format(const std::string &s)
}
static void print_lookup_string(const StringVector &s) {
- for (StringVector::const_iterator it = s.begin();
+ for (StringVector::const_iterator it = s.begin();
it != s.end(); it++) {
fprintf(stderr, "%s", get_print_format(*it).c_str());
}
@@ -1218,7 +1218,7 @@ bool is_possible_to_get_result(const HfstOneLevelPath & s,
// which transducer in the cascade we are handling
static unsigned int transducer_number=0;
-void lookup_fd_and_print(HfstBasicTransducer &t, HfstOneLevelPaths& results,
+void lookup_fd_and_print(HfstBasicTransducer &t, HfstOneLevelPaths& results,
const HfstOneLevelPath& s, ssize_t limit = -1)
{
(void)limit; // FIX ???
@@ -1227,10 +1227,10 @@ void lookup_fd_and_print(HfstBasicTransducer &t, HfstOneLevelPaths& results,
HfstTwoLevelPaths results_spv;
StringPairVector path_spv;
- if (is_possible_to_get_result(s, cascade_symbols_seen[transducer_number],
+ if (is_possible_to_get_result(s, cascade_symbols_seen[transducer_number],
cascade_unknown_or_identity_seen[transducer_number]))
{
- t.lookup_fd(s.second, results_spv, &infinite_cutoff);
+ t.lookup(s.second, results_spv, &infinite_cutoff);
}
if (print_pairs) {
@@ -1249,7 +1249,7 @@ void lookup_fd_and_print(HfstBasicTransducer &t, HfstOneLevelPaths& results,
if (it == results_spv.begin())
lowest_weight = it->first;
if (beam < 0 || it->first <= (lowest_weight + beam))
- {
+ {
/* print the lookup string */
print_lookup_string(s.second);
fprintf(outfile, "\t");
@@ -1262,8 +1262,8 @@ void lookup_fd_and_print(HfstBasicTransducer &t, HfstOneLevelPaths& results,
fprintf(outfile, " ");
}
first_pair=false;
- fprintf(outfile, "%s:%s",
- get_print_format(IT->first).c_str(),
+ fprintf(outfile, "%s:%s",
+ get_print_format(IT->first).c_str(),
get_print_format(IT->second).c_str());
}
/* and the weight of that path. */
@@ -1319,7 +1319,7 @@ lookup_simple(const HfstOneLevelPath& s, HfstBasicTransducer& t, bool* infinity)
HfstOneLevelPaths* results = new HfstOneLevelPaths;
bool possible = is_possible_to_get_result
- (s, cascade_symbols_seen[transducer_number],
+ (s, cascade_symbols_seen[transducer_number],
cascade_unknown_or_identity_seen[transducer_number]);
if (possible && t.is_lookup_infinitely_ambiguous(s))
@@ -1474,7 +1474,7 @@ print_lookups(const HfstOneLevelPaths& kvs,
HfstOneLevelPaths*
-perform_lookups(HfstOneLevelPath& origin, std::vector<HfstTransducer>& cascade,
+perform_lookups(HfstOneLevelPath& origin, std::vector<HfstTransducer>& cascade,
bool unknown, bool* infinite)
{
HfstOneLevelPaths* kvs;
@@ -1497,7 +1497,7 @@ perform_lookups(HfstOneLevelPath& origin, std::vector<HfstTransducer>& cascade,
}
HfstOneLevelPaths*
-perform_lookups(HfstOneLevelPath& origin, std::vector<HfstBasicTransducer>& cascade,
+perform_lookups(HfstOneLevelPath& origin, std::vector<HfstBasicTransducer>& cascade,
bool unknown, bool* infinite)
{
HfstOneLevelPaths* kvs;
@@ -1537,7 +1537,7 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
hfst::ImplementationType type = trans.get_type();
std::set<std::string> symbols_seen;
- if (type != HFST_OL_TYPE && type != HFST_OLW_TYPE)
+ if (type != HFST_OL_TYPE && type != HFST_OLW_TYPE)
{
only_optimized_lookup = false;
}
@@ -1562,12 +1562,12 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
}
if (transducer_n==1)
{
- verbose_printf("Reading %s...\n", inputname);
+ verbose_printf("Reading %s...\n", inputname);
}
else
{
verbose_printf("Reading %s..." SIZE_T_SPECIFIER "\n", inputname,
- transducer_n);
+ transducer_n);
}
if (! invert)
@@ -1583,7 +1583,7 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
}
// add multicharacter symbols to mc_symbols
- if (type == hfst::SFST_TYPE ||
+ if (type == hfst::SFST_TYPE ||
type == hfst::TROPICAL_OPENFST_TYPE ||
type == hfst::LOG_OPENFST_TYPE ||
type == hfst::FOMA_TYPE)
@@ -1592,7 +1592,7 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
for (HfstBasicTransducer::const_iterator it = basic.begin();
it != basic.end(); it++)
{
- for (HfstBasicTransducer::HfstTransitions::const_iterator
+ for (hfst::implementations::HfstBasicTransitions::const_iterator
tr_it = it->begin(); tr_it != it->end(); tr_it++)
{
std::string mcs = tr_it->get_input_symbol();
@@ -1620,8 +1620,8 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
inputstream.close();
- if (print_pairs &&
- (inputstream.get_type() == HFST_OL_TYPE ||
+ if (print_pairs &&
+ (inputstream.get_type() == HFST_OL_TYPE ||
inputstream.get_type() == HFST_OLW_TYPE) ) {
error(EXIT_FAILURE, 0, "pair printing not supported on "
"optimized lookup transducers");
@@ -1633,14 +1633,14 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
char* line = 0;
size_t llen = 0;
- hfst::HfstStrings2FstTokenizer input_tokenizer(mc_symbols,
+ hfst::HfstStrings2FstTokenizer input_tokenizer(mc_symbols,
std::string(epsilon_format));
if (!only_optimized_lookup)
{
char* format_string = hfst_strformat(cascade[0].get_type());
if (!silent) {
- warning(0, 0,
+ warning(0, 0,
"It is not possible to perform fast lookups with %s "
"format automata.\n"
"Using HFST basic transducer format "
@@ -1739,7 +1739,7 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
unknown, &infinite);
}
- if (! print_pairs) {
+ if (! print_pairs) {
// printing was already done in function lookup_fd
print_lookups(*kvs, *kv, markup, unknown, infinite, outstream);
fflush(outstream);
@@ -1749,7 +1749,7 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
print_prompt();
} // while lines in input
- if (show_progress_bar)
+ if (show_progress_bar)
{
fprintf(stderr, "%ld/%ld... Done\n", filepos, filesize);
}
@@ -1757,7 +1757,7 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
if (print_statistics)
{
fprintf(outstream, "Strings\tFound\tMissing\tResults\n"
- "%lu\t%lu\t%lu\t%lu\n",
+ "%lu\t%lu\t%lu\t%lu\n",
inputs, analysed, no_analyses, analyses);
fprintf(outstream, "Coverage\tAmbiguity\n"
"%f\t%f\n",
@@ -1799,7 +1799,7 @@ int main( int argc, char **argv ) {
{
fclose(inputfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
verbose_printf("Output formats:\n"
" regular:`%s'`%s...'`%s',\n"
@@ -1814,12 +1814,12 @@ int main( int argc, char **argv ) {
epsilon_format, space_format, show_flags);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
- try
+ try
{
instream = (inputfile != stdin) ?
new HfstInputStream(inputfilename) :
new HfstInputStream();
- }
+ }
catch(const HfstException e)
{
error(EXIT_FAILURE, 0, "%s is not a valid transducer file",
diff --git a/tools/src/hfst-format.cc b/tools/src/hfst-format.cc
index 57495dd..b744e62 100644
--- a/tools/src/hfst-format.cc
+++ b/tools/src/hfst-format.cc
@@ -51,7 +51,7 @@ static char * format_to_test = NULL;
void
print_usage()
{
- // c.f.
+ // c.f.
// http://www.gnu.org/prep/standards/standards.html#g_t_002d_002dhelp
fprintf(message_out, "Usage: %s [OPTIONS...] [INFILE]\n"
"determine HFST transducer format\n"
@@ -60,12 +60,12 @@ print_usage()
print_common_program_options(message_out);
print_common_unary_program_options(message_out);
fprintf
- (message_out,
+ (message_out,
"Tool-specific options:\n"
" -l, --list-formats List available transducer formats\n"
" and print them to standard output\n");
fprintf
- (message_out,
+ (message_out,
" -t, --test-format FMT Whether the format FMT is available,\n"
" exits with 0 if it is, else with 1\n");
fprintf(message_out, "\n");
@@ -94,7 +94,7 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "1:2:lt:",
long_options, &option_index);
if (-1 == c)
@@ -184,7 +184,7 @@ parse_options(int argc, char** argv)
if (HfstTransducer::is_implementation_type_available
(hfst::HFST_OL_TYPE))
fprintf(stdout, " Optimized lookup (weighted) "
- "optimized-lookup-unweighted, olu\n");
+ "optimized-lookup-unweighted, olu\n");
if (HfstTransducer::is_implementation_type_available
(hfst::HFST_OLW_TYPE))
@@ -210,7 +210,7 @@ parse_options(int argc, char** argv)
else if ((argc - optind) == 1)
{
inputfilename = argv[optind];
- }
+ }
}
hfst::HfstInputStream is(inputfilename);
return is.get_type();
diff --git a/tools/src/hfst-fst2fst.cc b/tools/src/hfst-fst2fst.cc
index 7e3b0ed..c7fae4a 100644
--- a/tools/src/hfst-fst2fst.cc
+++ b/tools/src/hfst-fst2fst.cc
@@ -55,6 +55,16 @@ ImplementationType output_type = hfst::UNSPECIFIED_TYPE;
bool hfst_format = true;
std::string options = "";
+void set_output_type(ImplementationType type)
+{
+ if (output_type != hfst::UNSPECIFIED_TYPE)
+ {
+ error(EXIT_FAILURE, 0,
+ "Output type defined several times.");
+ }
+ output_type = type;
+}
+
void
print_usage()
{
@@ -78,7 +88,7 @@ print_usage()
" -Q --quick When converting to optimized-lookup, don't try hard to compress\n");
fprintf(message_out, "\n");
print_common_unary_program_parameter_instructions(message_out);
- fprintf(message_out,
+ fprintf(message_out,
"FMT must be name of a format usable by libhfst, i.e. one of the following:\n"
"{ foma, openfst-tropical, openfst-log, sfst, xfsm\n"
" optimized-lookup-weighted, optimized-lookup-unweighted }.\n"
@@ -100,7 +110,7 @@ parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
- // add tool-specific options here
+ // add tool-specific options here
{"use-backend-format", no_argument, 0, 'b'},
{"format", required_argument, 0, 'f'},
{"sfst", no_argument, 0, 'S'},
@@ -114,8 +124,8 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "SFtlOwQf:bx",
long_options, &option_index);
if (-1 == c)
@@ -129,32 +139,42 @@ parse_options(int argc, char** argv)
#include "inc/getopt-cases-common.h"
#include "inc/getopt-cases-unary.h"
// add tool-specific cases here
- case 'f':
- output_type = hfst_parse_format_name(optarg);
+ case 'f':
+ set_output_type(hfst_parse_format_name(optarg));
+#ifndef HAVE_XFSM
+ if (output_type == hfst::XFSM_TYPE)
+ error(EXIT_FAILURE, 0,
+ "xfsm back-end is not available");
+#endif
break;
case 'b':
hfst_format=false;
break;
case 'S':
- output_type = hfst::SFST_TYPE;
+ set_output_type(hfst::SFST_TYPE);
break;
case 'F':
- output_type = hfst::FOMA_TYPE;
+ set_output_type(hfst::FOMA_TYPE);
break;
case 'x':
- output_type = hfst::XFSM_TYPE;
+#ifdef HAVE_XFSM
+ set_output_type(hfst::XFSM_TYPE);
+#else
+ error(EXIT_FAILURE, 0,
+ "xfsm back-end is not available");
+#endif
break;
case 't':
- output_type = hfst::TROPICAL_OPENFST_TYPE;
+ set_output_type(hfst::TROPICAL_OPENFST_TYPE);
break;
case 'l':
- output_type = hfst::LOG_OPENFST_TYPE;
+ set_output_type(hfst::LOG_OPENFST_TYPE);
break;
case 'O':
- output_type = hfst::HFST_OL_TYPE;
+ set_output_type(hfst::HFST_OL_TYPE);
break;
case 'w':
- output_type = hfst::HFST_OLW_TYPE;
+ set_output_type(hfst::HFST_OLW_TYPE);
break;
case 'Q':
options = "quick";
@@ -165,7 +185,7 @@ parse_options(int argc, char** argv)
if (output_type == hfst::UNSPECIFIED_TYPE)
{
- error(EXIT_FAILURE, 0,
+ error(EXIT_FAILURE, 0,
"You must specify an output type "
"(one of -S, -F, -t, -x, -l, -O, or -w)");
}
@@ -196,7 +216,7 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
char* inputname = hfst_get_name(orig, inputfilename);
if (transducer_n == 1)
{
- verbose_printf("Converting %s...\n", inputname);
+ verbose_printf("Converting %s...\n", inputname);
}
else
{
@@ -238,10 +258,10 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
char* format_description = hfst_strformat(output_type);
- if (hfst_format && (output_type != hfst::XFSM_TYPE))
+ if (hfst_format && (output_type != hfst::XFSM_TYPE))
{
verbose_printf("Writing %s format transducers with HFST3 headers\n",
format_description);
@@ -267,13 +287,19 @@ int main( int argc, char **argv ) {
try {
instream = (inputfile != stdin) ?
new HfstInputStream(inputfilename) : new HfstInputStream();
- }
+ }
catch(const FileIsInGZFormatException e)
{
error(EXIT_FAILURE, 0, "%s seems to be a gzipped native foma file, you must first unzip it",
inputfilename);
return EXIT_FAILURE;
}
+ catch(const ImplementationTypeNotAvailableException e)
+ {
+ error(EXIT_FAILURE, 0, "%s is in %s format which is not available",
+ inputfilename, hfst::implementation_type_to_format(e.get_type()));
+ return EXIT_FAILURE;
+ }
catch(const HfstException e) {
error(EXIT_FAILURE, 0, "%s is not a valid transducer file",
inputfilename);
diff --git a/tools/src/hfst-fst2strings.cc b/tools/src/hfst-fst2strings.cc
index 8995270..5fac0a5 100644
--- a/tools/src/hfst-fst2strings.cc
+++ b/tools/src/hfst-fst2strings.cc
@@ -127,7 +127,7 @@ print_usage()
" %s lexical.hfst generates all forms of lexical.hfst\n"
"\n", program_name);
- fprintf(message_out,
+ fprintf(message_out,
"Known bugs:\n"
" Does not work correctly for hfst optimized lookup format.\n"
"\n"
@@ -167,7 +167,7 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT
"Swb:c:e:u:p:l:L:n:r:N:U:P:X:",
long_options, &option_index);
@@ -264,7 +264,7 @@ parse_options(int argc, char** argv)
}
/* Replace all strings \a str1 in \a symbol with \a str2. */
-static std::string replace_all(std::string symbol,
+static std::string replace_all(std::string symbol,
const std::string &str1,
const std::string &str2)
{
@@ -274,13 +274,13 @@ static std::string replace_all(std::string symbol,
symbol.erase(pos, str1.size()); // erase str1
symbol.insert(pos, str2); // insert str2 instead
pos = symbol.find // find next str1
- (str1, pos+str2.size());
+ (str1, pos+str2.size());
}
return symbol;
}
-static std::string get_print_format(const std::string &s)
+static std::string get_print_format(const std::string &s)
{
// print epsilon as defined by the user or use the default
if (hfst::is_epsilon(s))
@@ -290,8 +290,8 @@ static std::string get_print_format(const std::string &s)
return std::string(s);
// escape spaces and colons as they have a special meaning
- return
- replace_all
+ return
+ replace_all
(replace_all
( replace_all
(std::string(s), " ", "@_SPACE_@"),
@@ -354,13 +354,13 @@ public:
// continue searching, break off this path
}
}
- if (input_exclude.length() > 0 &&
+ if (input_exclude.length() > 0 &&
(istring.find(input_exclude) != std::string::npos))
{
return RetVal(true, false);
// continue searching, break off this path
}
- if (output_exclude.length() > 0 &&
+ if (output_exclude.length() > 0 &&
(ostring.find(output_exclude) != std::string::npos))
{
return RetVal(true, false);
@@ -374,15 +374,15 @@ public:
// the path passed the checks. Print it if it is final
if (final)
{
- if (print_in_pairstring_format)
+ if (print_in_pairstring_format)
{
bool first_pair=true;
for (StringPairVector::const_iterator it = path.second.begin();
- it != path.second.end(); it++)
+ it != path.second.end(); it++)
{
- if ((not filter_fd) ||
+ if ((not filter_fd) ||
(not FdOperation::is_diacritic(it->first))) {
- if (print_spaces && not first_pair)
+ if (print_spaces && not first_pair)
{
*out_ << " ";
}
@@ -393,19 +393,19 @@ public:
if (it->first.compare(it->second) != 0)
{
- if ((not filter_fd) ||
+ if ((not filter_fd) ||
(not FdOperation::is_diacritic(it->second)))
*out_ << ":"
<< get_print_format(it->second);
}
}
- if (display_weights)
+ if (display_weights)
{
*out_ << "\t" << path.first;
}
*out_ << "\n";
}
- else
+ else
{
bool is_automaton=true;
@@ -413,7 +413,7 @@ public:
for (StringPairVector::const_iterator it = path.second.begin();
it != path.second.end(); ++it)
{
- if ((not filter_fd) ||
+ if ((not filter_fd) ||
(not FdOperation::is_diacritic(it->first)))
{
if (print_spaces && not first_symbol)
@@ -437,7 +437,7 @@ public:
for (StringPairVector::const_iterator it = path.second.begin();
it != path.second.end(); ++it)
{
- if ((not filter_fd) ||
+ if ((not filter_fd) ||
(not FdOperation::is_diacritic(it->second)))
{
if (print_spaces)
@@ -458,7 +458,7 @@ public:
count++;
}
// continue until we've printed max_num strings
- return RetVal((max_num < 1) || (count < max_num), true);
+ return RetVal((max_num < 1) || (count < max_num), true);
}
};
@@ -477,10 +477,10 @@ process_stream(HfstInputStream& instream, std::ostream& outstream)
HfstTransducer t(instream);
/* Pairstring format is not supported on optimized lookup format. */
- if (print_in_pairstring_format &&
- (instream.get_type() == HFST_OL_TYPE ||
+ if (print_in_pairstring_format &&
+ (instream.get_type() == HFST_OL_TYPE ||
instream.get_type() == HFST_OLW_TYPE) ) {
- fprintf(stderr,
+ fprintf(stderr,
"Error: option --print-in-pairstring-format not supported on "
" optimized lookup transducers, exiting program\n" );
exit(1);
@@ -489,10 +489,10 @@ process_stream(HfstInputStream& instream, std::ostream& outstream)
if(input_prefix != "")
verbose_printf("input_prefix: '%s'\n", input_prefix.c_str());
- if(beam >= 0)
+ if(beam >= 0)
{
verbose_printf("Finding the weight of the best path...\n");
- try
+ try
{
HfstTransducer tc(t);
tc.n_best(1);
@@ -506,7 +506,7 @@ process_stream(HfstInputStream& instream, std::ostream& outstream)
}
catch (const FunctionNotImplementedException & e)
{
- if (instream.get_type() == hfst::HFST_OL_TYPE ||
+ if (instream.get_type() == hfst::HFST_OL_TYPE ||
instream.get_type() == hfst::HFST_OLW_TYPE)
{
error(EXIT_FAILURE, 0, "option --beam not implemented for optimized lookup format");
@@ -526,15 +526,15 @@ process_stream(HfstInputStream& instream, std::ostream& outstream)
if(nbest_strings > 0)
{
- verbose_printf("Pruning transducer to %i best path(s)...\n",
+ verbose_printf("Pruning transducer to %i best path(s)...\n",
nbest_strings);
- try
+ try
{
t.n_best(nbest_strings);
}
catch (const FunctionNotImplementedException & e)
{
- if (instream.get_type() == hfst::HFST_OL_TYPE ||
+ if (instream.get_type() == hfst::HFST_OL_TYPE ||
instream.get_type() == hfst::HFST_OLW_TYPE)
{
error(EXIT_FAILURE, 0, "option --nbest not implemented for optimized lookup format");
@@ -553,7 +553,7 @@ process_stream(HfstInputStream& instream, std::ostream& outstream)
}
else
{
- if(max_random_strings <= 0 && max_strings <= 0 && max_input_length <= 0
+ if(max_random_strings <= 0 && max_strings <= 0 && max_input_length <= 0
&& max_output_length <= 0 &&
cycles < 0 && t.is_cyclic())
{
@@ -567,7 +567,7 @@ process_stream(HfstInputStream& instream, std::ostream& outstream)
if(max_strings > 0)
verbose_printf("Finding at most %i path(s)...\n", max_strings);
else if(max_random_strings > 0)
- verbose_printf("Finding at most %i random path(s)...\n",
+ verbose_printf("Finding at most %i random path(s)...\n",
max_random_strings);
else
verbose_printf("Finding strings...\n");
@@ -579,7 +579,7 @@ process_stream(HfstInputStream& instream, std::ostream& outstream)
if(eval_fd)
t.extract_paths_fd(cb, cycles, filter_fd);
else
- t.extract_paths(cb, cycles);
+ t.extract_paths(cb, cycles);
verbose_printf("Printed %i string(s)\n", cb.count);
}
/* random strings */
@@ -595,7 +595,7 @@ process_stream(HfstInputStream& instream, std::ostream& outstream)
}
}
catch (const FunctionNotImplementedException & e) {
- if (instream.get_type() == hfst::HFST_OL_TYPE ||
+ if (instream.get_type() == hfst::HFST_OL_TYPE ||
instream.get_type() == hfst::HFST_OLW_TYPE)
{
error(EXIT_FAILURE, 0, "option --random not implemented for optimized lookup format");
@@ -637,7 +637,7 @@ int main( int argc, char **argv ) {
if (max_strings > 0 && max_random_strings > 0 && !silent)
{
- fprintf(stderr,
+ fprintf(stderr,
"warning: option --max_strings ignored, --random used\n");
max_strings = -1;
}
@@ -655,7 +655,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-fst2txt.cc b/tools/src/hfst-fst2txt.cc
index c8f3105..072d728 100644
--- a/tools/src/hfst-fst2txt.cc
+++ b/tools/src/hfst-fst2txt.cc
@@ -91,7 +91,7 @@ print_usage()
"If OUTFILE or INFILE is missing or -, "
"standard streams will be used.\n"
"Unless explicitly requested with option -w or -D, "
- "weights are printed\n"
+ "weights are printed\n"
"if and only if the transducer is in weighted format.\n"
"TFMT is one of {att, dot, prolog, pckimmo}.\n"
);
@@ -120,8 +120,8 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "wDnf:",
long_options, &option_index);
if (-1 == c)
@@ -145,13 +145,13 @@ parse_options(int argc, char** argv)
break;
case 'f':
if ((strcmp(optarg, "att") == 0) || (strcmp(optarg, "AT&T") == 0) ||
- (strcmp(optarg, "openfst") == 0) ||
+ (strcmp(optarg, "openfst") == 0) ||
(strcmp(optarg, "OpenFst") == 0))
{
format = ATT_TEXT;
}
- else if ((strcmp(optarg, "dot") == 0) ||
- (strcmp(optarg, "graphviz") == 0) ||
+ else if ((strcmp(optarg, "dot") == 0) ||
+ (strcmp(optarg, "graphviz") == 0) ||
(strcmp(optarg, "GraphViz") == 0))
{
format = DOT_TEXT;
@@ -190,7 +190,7 @@ process_stream(HfstInputStream& instream, FILE* outf)
{
transducer_n++;
HfstTransducer* t=NULL;
- try
+ try
{
t = new HfstTransducer(instream);
}
@@ -205,17 +205,17 @@ process_stream(HfstInputStream& instream, FILE* outf)
}
if (transducer_n == 1)
{
- verbose_printf("Converting %s...\n", inputname);
+ verbose_printf("Converting %s...\n", inputname);
}
else
- {
+ {
if (instream.get_type() == hfst::XFSM_TYPE) {
error(EXIT_FAILURE, 0, "Writing more than one transducer in text format to file not supported for xfsm transducers,\n"
"use [hfst-head|hfst-tail|hfst-split] to extract individual transducers from input");
- return EXIT_FAILURE;
+ return EXIT_FAILURE;
}
verbose_printf("Converting %s..." SIZE_T_SPECIFIER "\n", inputname,
- transducer_n);
+ transducer_n);
}
@@ -255,7 +255,7 @@ process_stream(HfstInputStream& instream, FILE* outf)
break;
case PROLOG_TEXT:
{
- try
+ try
{
if (type == hfst::XFSM_TYPE) {
t->write_xfsm_transducer_in_prolog_format(outfilename); // no name or weights printed
@@ -300,7 +300,7 @@ process_stream(HfstInputStream& instream, FILE* outf)
}
-int main( int argc, char **argv )
+int main( int argc, char **argv )
{
#ifdef WINDOWS
_setmode(0, _O_BINARY);
@@ -318,7 +318,7 @@ int main( int argc, char **argv )
fclose(inputfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
@@ -335,29 +335,29 @@ int main( int argc, char **argv )
{
if (format == DOT_TEXT) {
error(EXIT_FAILURE, 0, "Output format 'dot' not supported for xfsm transducers, use 'prolog'");
- return EXIT_FAILURE;
+ return EXIT_FAILURE;
}
if (format == PCKIMMO_TEXT) {
error(EXIT_FAILURE, 0, "Output format 'pckimmo' not supported for xfsm transducers, use 'prolog'");
- return EXIT_FAILURE;
+ return EXIT_FAILURE;
}
if (format == ATT_TEXT) {
error(EXIT_FAILURE, 0, "Output format 'att' not supported for xfsm transducers, use 'prolog'");
- return EXIT_FAILURE;
+ return EXIT_FAILURE;
}
if (use_numbers) {
error(EXIT_FAILURE, 0, "Option '--use-numbers' not supported for xfsm transducers");
- return EXIT_FAILURE;
+ return EXIT_FAILURE;
}
if (strcmp(inputfilename, "<stdin>") == 0) {
error(EXIT_FAILURE, 0, "Reading from standard input not supported for xfsm transducers,\n"
"use 'hfst-fst2txt [--input|-i] INFILE' instead");
- return EXIT_FAILURE;
+ return EXIT_FAILURE;
}
if (strcmp(outfilename, "<stdout>") == 0) {
error(EXIT_FAILURE, 0, "Writing to standard output not supported for xfsm transducers,\n"
"use 'hfst-fst2txt [--output|-o] OUTFILE' instead");
- return EXIT_FAILURE;
+ return EXIT_FAILURE;
}
}
diff --git a/tools/src/hfst-getopt.cc b/tools/src/hfst-getopt.cc
index ce1e487..cf8f1c4 100644
--- a/tools/src/hfst-getopt.cc
+++ b/tools/src/hfst-getopt.cc
@@ -21,7 +21,7 @@ int optind = 1;
std::vector<char *> free_arguments;
std::vector<char *> other_arguments;
-char getopt_long(int argc, char * /*const*/ argv [], const char * optstring,
+int getopt_long(int argc, char * /*const*/ argv [], const char * optstring,
const struct option * longopts, int * longindex)
{
// check that there are more args
diff --git a/tools/src/hfst-getopt.h b/tools/src/hfst-getopt.h
index eceb9fd..27ec38f 100644
--- a/tools/src/hfst-getopt.h
+++ b/tools/src/hfst-getopt.h
@@ -14,5 +14,5 @@ extern char * optarg;
extern int optopt;
extern int optind;
-char getopt_long(int argc, char * /*const*/ argv [], const char * optstring,
+int getopt_long(int argc, char * /*const*/ argv [], const char * optstring,
const struct option * longopts, int * longindex);
diff --git a/tools/src/hfst-grep.cc b/tools/src/hfst-grep.cc
index bb8142e..47714f7 100644
--- a/tools/src/hfst-grep.cc
+++ b/tools/src/hfst-grep.cc
@@ -105,12 +105,12 @@ print_usage()
fprintf(message_out, "Usage: %s [OPTIONS...] PATTERN [FILE...]\n"
"Search for PATTERN in each FILE or standard input.\n"
"Pattern is, by default, a Xerox regular expression (XRE).\n"
- "Example: hfst-grep 'h e l l o %% w o r l d' menu.h menu.c\n"
+ "Example: hfst-grep 'h e l l o %% w o r l d' menu.h menu.c\n"
"\n", program_name);
// options, grouped
print_common_program_options(message_out);
- fprintf(message_out,
+ fprintf(message_out,
" -9, --format=TYPE compile expressions to TYPE "
"automata\n");
fprintf(message_out, "\n");
@@ -221,7 +221,7 @@ parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
- // add tool-specific options here
+ // add tool-specific options here
{"format", required_argument, 0, '9'},
{"extended-regexp", no_argument, 0, 'E'},
{"fixed-strings", no_argument, 0, 'F'},
@@ -266,8 +266,8 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT
"EFGPXe:f:IwxzqmbnOad:D:rLlcZA:B:C:uU9:",
long_options, &option_index);
@@ -412,7 +412,7 @@ parse_options(int argc, char** argv)
print_short_help();
return EXIT_FAILURE;
}
- else
+ else
{
regexp = strdup(argv[optind]);
optind++;
@@ -497,12 +497,12 @@ read_matcher(HfstInputStream& instream)
}
if (transducer_n==1)
{
- verbose_printf("Reading matcher %s...\n", inputname);
+ verbose_printf("Reading matcher %s...\n", inputname);
}
else
{
verbose_printf("Reading matcher %s..." SIZE_T_SPECIFIER "\n", inputname,
- transducer_n);
+ transducer_n);
}
if (transducer_n > 1)
{
@@ -557,7 +557,7 @@ extend_matcher_with_options()
verbose_printf("Adding color codes to match boundaries...\n");
HfstTransducer colorStart("@_EPSILON_SYMBOL_@", "[31m", format);
HfstTransducer colorEnd("@_EPSILON_SYMBOL_@", "[00m", format);
- HfstTransducer coloured =
+ HfstTransducer coloured =
colorStart.concatenate(*matcher).concatenate(colorEnd);
matcher = new HfstTransducer(coloured, format);
}
@@ -567,7 +567,7 @@ extend_matcher_with_options()
verbose_printf("Adding brackets to match boundaries...\n");
HfstTransducer colorStart("@_EPSILON_SYMBOL_@", "{{{", format);
HfstTransducer colorEnd("@_EPSILON_SYMBOL_@", "}}}", format);
- HfstTransducer coloured =
+ HfstTransducer coloured =
colorStart.concatenate(*matcher).concatenate(colorEnd);
matcher = new HfstTransducer(coloured, format);
}
@@ -577,7 +577,7 @@ extend_matcher_with_options()
"(currently space)...\n");
HfstTransducer nonWordCharLeft(" ", format);
HfstTransducer nonWordCharRight(" ", format);
- HfstTransducer wordBounded =
+ HfstTransducer wordBounded =
nonWordCharLeft.concatenate(*matcher).concatenate(nonWordCharRight);
matcher = new HfstTransducer(wordBounded, format);
}
@@ -588,7 +588,7 @@ extend_matcher_with_options()
HfstTransducer rightAny("@_IDENTITY_SYMBOL_@", format);
leftAny.repeat_star();
rightAny.repeat_star();
- HfstTransducer oneMatch =
+ HfstTransducer oneMatch =
leftAny.concatenate(*matcher).concatenate(rightAny);
matcher = new HfstTransducer(oneMatch, format);
matcher->repeat_plus();
diff --git a/tools/src/hfst-guess.cc b/tools/src/hfst-guess.cc
index c3fb85c..590b63d 100644
--- a/tools/src/hfst-guess.cc
+++ b/tools/src/hfst-guess.cc
@@ -1,4 +1,4 @@
-//! @file hfst-guess.cc
+//! @file hfst-guess.cc
//!
//! @brief Tool for compiling a guesser (and
//! generator) for guessing analyses/paradigms of unknown words.
@@ -71,7 +71,7 @@ size_t get_size_t(const std::string &str)
in >> i;
if (in.fail())
- { return -1; }
+ { throw "fail"; }
return i;
}
@@ -155,8 +155,8 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "f:m:n:g:",
long_options, &option_index);
if (-1 == c)
@@ -185,23 +185,29 @@ parse_options(int argc, char** argv)
break;
case 'n':
- max_number_of_guesses = get_size_t(optarg);
-
- if (max_number_of_guesses < 0)
+ try
+ {
+ max_number_of_guesses = get_size_t(optarg);
+ }
+ catch (const char * msg)
{
+ (void)msg;
error(EXIT_FAILURE, 0, "Invalid maximal number of guesses %s. "
- "Give a positive int.", optarg);
+ "Give a positive int.", optarg);
}
break;
case 'm':
- max_number_of_forms = get_size_t(optarg);
-
- if (max_number_of_forms < 0)
+ try
+ {
+ max_number_of_forms = get_size_t(optarg);
+ }
+ catch (const char * msg)
{
+ (void)msg;
error(EXIT_FAILURE, 0, "Invalid maximal number of generated "
- "forms %s. Give a positive int.", optarg);
+ "forms %s. Give a positive int.", optarg);
}
break;
@@ -215,7 +221,7 @@ parse_options(int argc, char** argv)
return EXIT_CONTINUE;
}
-int main( int argc, char **argv )
+int main( int argc, char **argv )
{
#ifdef WINDOWS
_setmode(0, _O_BINARY);
@@ -234,19 +240,19 @@ int main( int argc, char **argv )
fclose(inputfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream * instream = NULL;
- try
+ try
{
instream = (inputfile != stdin ?
- new HfstInputStream(inputfilename) :
+ new HfstInputStream(inputfilename) :
new HfstInputStream());
- }
- catch(const HfstException e)
+ }
+ catch(const HfstException e)
{
error(EXIT_FAILURE, 0, "%s is not a valid transducer file",
inputfilename);
@@ -255,13 +261,13 @@ int main( int argc, char **argv )
std::ostream * out;
- try
+ try
{
out = (outfile != stdout ?
new std::ofstream(outfilename) :
&std::cout);
- }
- catch(...)
+ }
+ catch(...)
{
error(EXIT_FAILURE, 0, "%s cannot be opened for writing.",
outfilename);
@@ -366,7 +372,7 @@ int main( int argc, char **argv )
(*out) << line << "\t" << *it << std::endl;
}
- }
+ }
(*out) << std::endl;
}
diff --git a/tools/src/hfst-guessify.cc b/tools/src/hfst-guessify.cc
index 86501fe..910285d 100644
--- a/tools/src/hfst-guessify.cc
+++ b/tools/src/hfst-guessify.cc
@@ -1,6 +1,6 @@
//! @file hfst-guessify.cc
//!
-//! @brief Tool for compiling a guesser and model form generator from a
+//! @brief Tool for compiling a guesser and model form generator from a
//! morphological analyzer.
//!
//! @author Miikka Silfverberg (HFST Team)
@@ -130,8 +130,8 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "p:G",
long_options, &option_index);
if (-1 == c)
@@ -183,13 +183,13 @@ process_stream(HfstInputStream& instream, HfstOutputStream &out)
if (compile_generator)
- {
+ {
verbose_printf("Compiling generator and storing guesser and "
- "generator.\n");
+ "generator.\n");
}
else
- {
- verbose_printf("Storing guesser.\n");
+ {
+ verbose_printf("Storing guesser.\n");
}
store_guesser(guesser,out,compile_generator);
@@ -201,7 +201,7 @@ process_stream(HfstInputStream& instream, HfstOutputStream &out)
}
-int main( int argc, char **argv )
+int main( int argc, char **argv )
{
#ifdef WINDOWS
_setmode(0, _O_BINARY);
@@ -221,19 +221,19 @@ int main( int argc, char **argv )
fclose(inputfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream * instream = NULL;
- try
+ try
{
instream = (inputfile != stdin ?
- new HfstInputStream(inputfilename) :
+ new HfstInputStream(inputfilename) :
new HfstInputStream());
- }
- catch(const HfstException e)
+ }
+ catch(const HfstException e)
{
error(EXIT_FAILURE, 0, "%s is not a valid transducer file",
inputfilename);
@@ -242,13 +242,13 @@ int main( int argc, char **argv )
HfstOutputStream * outstream = NULL;
- try
+ try
{
outstream = (outfile != stdout ?
- new HfstOutputStream(outfilename,HFST_OLW_TYPE) :
+ new HfstOutputStream(outfilename,HFST_OLW_TYPE) :
new HfstOutputStream(HFST_OLW_TYPE));
- }
- catch(const HfstException e)
+ }
+ catch(const HfstException e)
{
error(EXIT_FAILURE, 0, "%s cannot be opened for writing.",
outfilename);
diff --git a/tools/src/hfst-head.cc b/tools/src/hfst-head.cc
index 1ba2ada..5dfc167 100644
--- a/tools/src/hfst-head.cc
+++ b/tools/src/hfst-head.cc
@@ -93,12 +93,12 @@ parse_options(int argc, char** argv)
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
{"n-first", required_argument, 0, 'n'},
- // add tool-specific options here
+ // add tool-specific options here
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "n:",
long_options, &option_index);
if (-1 == c)
@@ -141,7 +141,7 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
{
inputname = strdup(inputfilename);
}
- verbose_printf("Forwarding %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
+ verbose_printf("Forwarding %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
outstream << trans;
}
}
@@ -176,7 +176,7 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
{
inputname = strdup(inputfilename);
}
- verbose_printf("Forwarding %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
+ verbose_printf("Forwarding %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
outstream << trans;
first_but_n.pop_front();
}
@@ -209,7 +209,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-info.cc b/tools/src/hfst-info.cc
index 4ff37e1..bf1b088 100644
--- a/tools/src/hfst-info.cc
+++ b/tools/src/hfst-info.cc
@@ -134,7 +134,7 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
"a:e:f:m:",
long_options, &option_index);
if (-1 == c)
@@ -255,15 +255,15 @@ int main (int argc, char * argv[])
}
}
verbose_printf("HFST info version: %s\n"
- "HFST packaging: %s\n"
- "HFST version: %s\n"
- "HFST long version: %ld\n"
- "HFST configuration revision: %s\n",
- hfst_tool_version,
- PACKAGE_STRING,
- PACKAGE_VERSION,
- HFST_LONGVERSION,
- HFST_REVISION);
+ "HFST packaging: %s\n"
+ "HFST version: %s\n"
+ "HFST long version: %ld\n",
+ // "HFST configuration revision: %s\n",
+ hfst_tool_version,
+ PACKAGE_STRING,
+ PACKAGE_VERSION,
+ HFST_LONGVERSION);
+ // HFST_REVISION);
#if HAVE_OPENFST
verbose_printf("OpenFst supported\n");
#endif
diff --git a/tools/src/hfst-invert.cc b/tools/src/hfst-invert.cc
index 9127a21..49b4f9e 100644
--- a/tools/src/hfst-invert.cc
+++ b/tools/src/hfst-invert.cc
@@ -81,12 +81,12 @@ parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
- // add tool-specific options here
+ // add tool-specific options here
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT,
long_options, &option_index);
if (-1 == c)
@@ -121,11 +121,11 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
char* inputname = hfst_get_name(trans, inputfilename);
if (transducer_n==1)
{
- verbose_printf("Inverting %s...\n", inputname);
+ verbose_printf("Inverting %s...\n", inputname);
}
else
{
- verbose_printf("Inverting %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
+ verbose_printf("Inverting %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
}
free(inputname);
trans.invert();
@@ -160,7 +160,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-lexc-compiler.cc b/tools/src/hfst-lexc-compiler.cc
index f5c9869..136fb88 100644
--- a/tools/src/hfst-lexc-compiler.cc
+++ b/tools/src/hfst-lexc-compiler.cc
@@ -114,7 +114,7 @@ print_usage()
" <[dog::1]+> # ; Use weights in regular expressions\n"
"\n"
"Using weights has an effect only if FORMAT is weighted, i.e.\n"
- "{ openfst-tropical, openfst-log, optimized-lookup-weighted }.\n"
+ "{ openfst-tropical, openfst-log, optimized-lookup-weighted }.\n"
"\n",
program_name, program_name );
print_report_bugs();
@@ -145,7 +145,7 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
"Ef:o:AFMRx:X:W",
long_options, &option_index);
if (-1 == c)
diff --git a/tools/src/hfst-lexc-wrapper.cc b/tools/src/hfst-lexc-wrapper.cc
index 0c61902..e8bf144 100644
--- a/tools/src/hfst-lexc-wrapper.cc
+++ b/tools/src/hfst-lexc-wrapper.cc
@@ -115,7 +115,7 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
"f:o:l::u::X",
long_options, &option_index);
if (-1 == c)
@@ -149,7 +149,7 @@ parse_options(int argc, char** argv)
#if HAVE_FOMA
if (format == hfst::UNSPECIFIED_TYPE)
{
- if (!silent)
+ if (!silent)
{
warning(0, 0, "Defaulting to foma type "
"(since it has native lexc support)\n");
@@ -251,7 +251,7 @@ parse_options(int argc, char** argv)
hfst_fseek(lexcfiles[i], -1, SEEK_END);
long file_length = hfst_ftell(lexcfiles[i]);
rewind(lexcfiles[i]);
- char* fdata = static_cast<char*>(malloc(sizeof(char) *
+ char* fdata = static_cast<char*>(malloc(sizeof(char) *
(file_length +1)));
hfst_fread(fdata, sizeof(char), file_length,
lexcfiles[i]);
@@ -327,7 +327,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- if (start_readline)
+ if (start_readline)
{
lexc_readline_loop(format);
return EXIT_FAILURE;
@@ -349,7 +349,7 @@ int main( int argc, char **argv ) {
}
catch (ImplementationTypeNotAvailableException itnae)
{
- verbose_printf("Caught exception like this:\n%s\n",
+ verbose_printf("Caught exception like this:\n%s\n",
itnae().c_str());
error(EXIT_FAILURE, 0, "You were trying to compile with format "
"that has been turned off from libhfst\n"
diff --git a/tools/src/hfst-lookup.cc b/tools/src/hfst-lookup.cc
index 130bf83..7e45c7a 100644
--- a/tools/src/hfst-lookup.cc
+++ b/tools/src/hfst-lookup.cc
@@ -54,7 +54,7 @@ using hfst::hfst_fprintf_console;
#include "HfstTransducer.h"
#include "HfstInputStream.h"
#include "HfstOutputStream.h"
-#include "implementations/HfstTransitionGraph.h"
+#include "implementations/HfstBasicTransducer.h"
#include "inc/globals-common.h"
#include "inc/globals-unary.h"
@@ -98,9 +98,16 @@ static bool pipe_input = false;
static bool pipe_output = false;
static size_t linen = 0;
static bool lookup_given = false;
-static size_t infinite_cutoff = 5;
+static size_t infinite_cutoff=5;
+static size_t max_number=-1;
+static size_t MAX_NUMBER=5;
static float beam=-1;
+#define CASCADE_UNION 1
+#define CASCADE_PRIORITY_UNION 2
+#define CASCADE_COMPOSITION 3
+static int cascade_ = CASCADE_UNION;
+
// symbols actually seen in (non-ol) transducers
static std::vector<std::set<std::string> > cascade_symbols_seen;
static std::vector<bool> cascade_unknown_or_identity_seen;
@@ -168,7 +175,7 @@ static const char* XEROX_END_SETF = "%n";
static const char* XEROX_EMPTY_BEGIN_SETF = "";
static const char* XEROX_EMPTY_LOOKUPF = "%i\t%i+?\t%w%n";
static const char* XEROX_EMPTY_END_SETF = "%n";
-// ¶ ¶+?
+// ¶ ¶+?
static const char* XEROX_UNKNOWN_BEGIN_SETF = "";
static const char* XEROX_UNKNOWN_LOOKUPF = "%i\t%i+?\t%w%n";
static const char* XEROX_UNKNOWN_END_SETF = "%n";
@@ -236,7 +243,7 @@ print_usage()
"\n", program_name);
print_common_program_options(message_out);
- fprintf(message_out,
+ fprintf(message_out,
"Input/Output options:\n"
" -i, --input=INFILE Read input transducer from INFILE\n"
" -o, --output=OUTFILE Write output to OUTFILE\n"
@@ -250,14 +257,18 @@ print_usage()
" -x, --statistics Print statistics\n"
" -X, --xfst=VARIABLE Toggle xfst VARIABLE\n"
" -c, --cycles=INT How many times to follow input epsilon cycles\n"
+ " (only for non-lookup-optimized transducers)\n"
+ " -n, --max-number=INT Maximum number of results printed for each input\n"
+ " (only for lookup-optimized transducers)\n"
" -b, --beam=B Output only analyses whose weight is within B from\n"
" the best analysis\n"
" -t, --time-cutoff=S Limit search after having used S seconds per input\n"
- " (currently only works in optimized-lookup mode\n"
+ " (only for lookup-optimized transducers)\n"
+ " -C, --cascade=CASCADE How multiple transducers in input are handled\n"
" -P, --progress Show neat progress bar if possible\n");
fprintf(message_out, "\n");
print_common_unary_program_parameter_instructions(message_out);
- fprintf(message_out,
+ fprintf(message_out,
"OFORMAT is one of {xerox,cg,apertium}, xerox being default\n"
"IFORMAT is one of {text,spaced,apertium}, default being text,\n"
"unless OFORMAT is apertium\n"
@@ -271,6 +282,10 @@ print_usage()
"results from all transducers is printed for each input string.\n");
fprintf(message_out, "\n");
+ fprintf(message_out, "CASCADE must be one of { union, priority-union, composition }.\n"
+ "If not specified, defaults to {union}.\n");
+ fprintf(message_out, "\n");
+
fprintf(message_out, "STREAM can be { input, output, both }. If not given, defaults to {both}.\n"
#ifdef _MSC_VER
"If input file is not specified with -I, input is read interactively via the\n"
@@ -284,12 +299,11 @@ print_usage()
#endif
fprintf(message_out, "\n");
- fprintf(message_out,
+ fprintf(message_out,
"Todo:\n"
- " For optimized lookup format, only strings that pass "
- "flag diacritic checks\n"
- " are printed and flag diacritic symbols are not printed.\n"
- " Support VARIABLE 'print-space' for optimized lookup format\n");
+ " Support --xfst=obey-flags for optimized lookup format.\n"
+ " Support --cycles for optimized lookup format.\n");
+
fprintf(message_out,
"\n"
"Known bugs:\n"
@@ -318,6 +332,7 @@ parse_options(int argc, char** argv)
{"input-format", required_argument, 0, 'F'},
{"statistics", no_argument, 0, 'x'},
{"cycles", required_argument, 0, 'c'},
+ {"max-number", required_argument, 0, 'n'},
{"xfst", required_argument, 0, 'X'},
{"epsilon-format", required_argument, 0, 'e'},
{"epsilon-format2", required_argument, 0, 'E'},
@@ -325,12 +340,13 @@ parse_options(int argc, char** argv)
{"time-cutoff", required_argument, 0, 't'},
{"pipe-mode", optional_argument, 0, 'p'},
{"progress", no_argument, 0, 'P'},
+ {"cascade", required_argument, 0, 'C'},
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
- HFST_GETOPT_UNARY_SHORT "I:O:F:xc:X:e:E:b:t:p::P",
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ HFST_GETOPT_UNARY_SHORT "I:O:F:xc:n:X:e:E:b:t:p::PC:",
long_options, &option_index);
if (-1 == c)
{
@@ -432,19 +448,22 @@ parse_options(int argc, char** argv)
{
quote_special = true;
}
- else if (strcmp(optarg, "obey-flags") == 0)
- {
- obey_flags = false;
- }
- else
+ else if (strcmp(optarg, "obey-flags") == 0)
+ {
+ obey_flags = false;
+ }
+ else
{
error(EXIT_FAILURE, 0, "Xfst variable %s unrecognised",
optarg);
}
+ break;
case 'c':
infinite_cutoff = (size_t)atoi(hfst_strdup(optarg));
break;
-
+ case 'n':
+ max_number = (size_t)atoi(hfst_strdup(optarg));
+ break;
case 'p':
if (optarg == NULL)
{ pipe_input = true; pipe_output = true; }
@@ -463,6 +482,18 @@ parse_options(int argc, char** argv)
case 'P':
show_progress_bar = true;
break;
+
+ case 'C':
+ if (strcmp(optarg, "union") == 0)
+ { cascade_ = CASCADE_UNION; }
+ else if (strcmp(optarg, "priority-union") == 0)
+ { cascade_ = CASCADE_PRIORITY_UNION; }
+ else if (strcmp(optarg, "composition") == 0)
+ { cascade_ = CASCADE_COMPOSITION; }
+ else
+ { error(EXIT_FAILURE, 0, "--cascade argument %s unrecognised, possible values are\n"
+ "{ union, priority-union, composition }", optarg); }
+ break;
#include "inc/getopt-cases-error.h"
}
}
@@ -537,28 +568,6 @@ static void print_prompt()
static std::string get_print_format(const std::string &s) ;
-bool
-is_valid_flag_diacritic_path(StringVector arcs)
- {
- //if (not print_pairs)
- // fprintf(stderr, "Allowing all flag paths!\n");
- FlagDiacriticTable FdT;
- bool res = FdT.is_valid_string(arcs);
- if (!res)
- {
- verbose_printf("blocked by flags: ");
- for (StringVector::const_iterator s = arcs.begin();
- s != arcs.end();
- ++s)
- {
- verbose_printf("%s ", s->c_str());
- }
- }
- return res;
- }
-
-
-
int
lookup_printf(const char* format, const HfstOneLevelPath* input,
const HfstOneLevelPath* result, const char* markup,
@@ -791,7 +800,7 @@ lookup_printf(const char* format, const HfstOneLevelPath* input,
// give up trying
anal_start = lookupform;
}
- b = static_cast<char*>(calloc(sizeof(char),
+ b = static_cast<char*>(calloc(sizeof(char),
anal_start - lookupform + 1));
b = static_cast<char*>(memcpy(b, lookupform, anal_start - lookupform));
a = static_cast<char*>(calloc(sizeof(char),
@@ -860,7 +869,7 @@ lookup_printf(const char* format, const HfstOneLevelPath* input,
src++;
}
else if (*src == 'w')
- {
+ {
int skip = 0;
#ifdef _MSC_VER
if (w == std::numeric_limits<float>::infinity())
@@ -1012,7 +1021,7 @@ line_to_lookup_path(char** s, hfst::HfstStrings2FstTokenizer& tok,
std::string S = escape_special_characters(*s);
- StringPairVector spv
+ StringPairVector spv
= tok.tokenize_string_pair(S, true);
for (StringPairVector::const_iterator it = spv.begin();
@@ -1032,7 +1041,7 @@ line_to_lookup_path(char** s, hfst::HfstStrings2FstTokenizer& tok,
{
std::string S = escape_special_characters(*s);
- StringPairVector spv
+ StringPairVector spv
= tok.tokenize_string_pair(S, false);
for (StringPairVector::const_iterator it = spv.begin();
@@ -1046,7 +1055,7 @@ line_to_lookup_path(char** s, hfst::HfstStrings2FstTokenizer& tok,
}
case APERTIUM_INPUT:
{
- char* real_s
+ char* real_s
= static_cast<char*>(calloc(sizeof(char),strlen(*s)+1));
*markup = static_cast<char*>(calloc(sizeof(char), strlen(*s)+1));
@@ -1115,22 +1124,39 @@ line_to_lookup_path(char** s, hfst::HfstStrings2FstTokenizer& tok,
return rv;
}
+void lookup_fd_and_print(HfstBasicTransducer * tr, HfstTransducer * TR, HfstOneLevelPaths& results,
+ const HfstOneLevelPath& s, size_t * limit = NULL, bool print_pairs_at_this_point = false,
+ bool print_fail = false, const HfstOneLevelPath * input_to_print = NULL,
+ bool no_newline = false);
+
HfstOneLevelPaths*
-lookup_simple(const HfstOneLevelPath& s, HfstTransducer& t, bool* infinity)
+lookup_simple(const HfstOneLevelPath& s, HfstTransducer& t, bool* infinity, bool print_pairs_at_this_point=false, bool print_fail=false, const HfstOneLevelPath * input_to_print = NULL, bool no_newline=false)
{
- HfstOneLevelPaths* results = 0;
+ HfstOneLevelPaths* results = new HfstOneLevelPaths;
if (time_cutoff == 0.0 && t.is_lookup_infinitely_ambiguous(s.second))
{
- if (!silent && infinite_cutoff > 0) {
- warning(0, 0, "Got infinite results, number of cycles limited to " SIZE_T_SPECIFIER "",
- infinite_cutoff);
+ size_t maxnum = (max_number == -1)? MAX_NUMBER : max_number;
+ if (!silent) {
+ if (max_number == -1)
+ warning(0, 0, "Got infinite results, number of results limited to " SIZE_T_SPECIFIER "\n"
+ "(can be controlled with --max-number=N)",
+ maxnum);
+ else
+ warning(0, 0, "Got infinite results, number of results limited to " SIZE_T_SPECIFIER "",
+ maxnum);
}
- results = t.lookup_fd(s.second, infinite_cutoff, time_cutoff);
+ if (print_pairs)
+ lookup_fd_and_print(NULL, &t, *results, s, &maxnum, print_pairs_at_this_point, print_fail, input_to_print, no_newline);
+ else
+ results = t.lookup_fd(s.second, maxnum, time_cutoff);
*infinity = true;
}
else
{
- results = t.lookup_fd(s.second, -1, time_cutoff);
+ if (print_pairs)
+ lookup_fd_and_print(NULL, &t, *results, s, &max_number, print_pairs_at_this_point, print_fail, input_to_print, no_newline);
+ else
+ results = t.lookup_fd(s.second, max_number, time_cutoff);
}
if (results->size() == 0)
@@ -1144,7 +1170,7 @@ lookup_simple(const HfstOneLevelPath& s, HfstTransducer& t, bool* infinity)
/* Replace all strings \a str1 in \a symbol with \a str2. */
-static std::string replace_all(std::string symbol,
+static std::string replace_all(std::string symbol,
const std::string &str1,
const std::string &str2)
{
@@ -1154,20 +1180,20 @@ static std::string replace_all(std::string symbol,
symbol.erase(pos, str1.size()); // erase str1
symbol.insert(pos, str2); // insert str2 instead
pos = symbol.find // find next str1
- (str1, pos+str2.size());
+ (str1, pos+str2.size());
}
return symbol;
}
-static std::string get_print_format(const std::string &s)
+static std::string get_print_format(const std::string &s)
{
if (is_epsilon(s))
return std::string(strdup(epsilon_format));
- if (quote_special)
+ if (quote_special)
{
- return
+ return
replace_all
( replace_all
( replace_all
@@ -1180,12 +1206,28 @@ static std::string get_print_format(const std::string &s)
}
static void print_lookup_string(const StringVector &s) {
- for (StringVector::const_iterator it = s.begin();
+ for (StringVector::const_iterator it = s.begin();
it != s.end(); it++) {
- fprintf(stderr, "%s", get_print_format(*it).c_str());
+#ifdef WINDOWS
+ if (!pipe_output)
+ hfst_fprintf_console(outfile, "%s", get_print_format(*it).c_str());
+ else
+#endif
+ fprintf(outfile, "%s", get_print_format(*it).c_str());
}
}
+static std::string get_lookup_string(const StringVector &s) {
+ std::string retval("");
+ for (StringVector::const_iterator it = s.begin();
+ it != s.end(); it++) {
+ retval += get_print_format(*it);
+ }
+ return retval;
+}
+
+
+
bool is_possible_to_get_result(const HfstOneLevelPath & s,
const StringSet & symbols_seen,
bool unknown_or_identity_seen)
@@ -1204,65 +1246,142 @@ bool is_possible_to_get_result(const HfstOneLevelPath & s,
// which transducer in the cascade we are handling
static unsigned int transducer_number=0;
-void lookup_fd_and_print(HfstBasicTransducer &t, HfstOneLevelPaths& results,
- const HfstOneLevelPath& s, ssize_t limit = -1)
-{
- (void)limit; // FIX ???
+void lookup_fd_and_print(HfstBasicTransducer * tr, HfstTransducer * TR, HfstOneLevelPaths& results,
+ const HfstOneLevelPath& s, size_t * limit /* = NULL*/, bool print_pairs_at_this_point/* = false*/,
+ bool print_fail /* = false*/, const HfstOneLevelPath * input_to_print /* = NULL*/,
+ bool no_newline /* = false*/)
+{
/* If we want a StringPairVector representation */
HfstTwoLevelPaths results_spv;
StringPairVector path_spv;
- if (is_possible_to_get_result(s, cascade_symbols_seen[transducer_number],
- cascade_unknown_or_identity_seen[transducer_number]))
+ if (tr != NULL)
+ {
+ if (is_possible_to_get_result(s, cascade_symbols_seen[transducer_number],
+ cascade_unknown_or_identity_seen[transducer_number]))
+ {
+ tr->lookup(s.second, results_spv, limit,
+ NULL /*no weight limit, variable 'beam' defines which paths are printed */,
+ obey_flags);
+ }
+ }
+ else
{
- t.lookup_fd(s.second, results_spv, &infinite_cutoff);
+ // TODO: is copying slow?
+ std::string lookup_str("");
+ for (std::vector<std::string>::const_iterator it = s.second.begin(); it != s.second.end(); it++)
+ {
+ lookup_str += *it;
+ }
+ HfstTwoLevelPaths * htlp = TR->lookup_pairs(lookup_str, *limit, time_cutoff);
+ results_spv = HfstTwoLevelPaths(*htlp);
+ delete htlp;
}
- if (print_pairs) {
+ if (print_pairs_at_this_point && print_pairs) {
/* No results, print just the lookup string. */
if (results_spv.size() == 0) {
- print_lookup_string(s.second);
- fprintf(outfile, "\n");
+ if (print_fail)
+ {
+ std::string input = get_lookup_string(s.second);
+#ifdef WINDOWS
+ if (!pipe_output)
+ hfst_fprintf_console(outfile, "%s\t%s+?\tinf\n\n", input.c_str(), input.c_str());
+ else
+#endif
+ fprintf(outfile, "%s\t%s+?\tinf\n\n", input.c_str(), input.c_str());
+ fflush(outfile);
+ }
}
- else { // HERE!!!
+ else {
float lowest_weight = -1;
/* For all result strings, */
for (HfstTwoLevelPaths::const_iterator it
= results_spv.begin(); it != results_spv.end(); it++) {
- if (it == results_spv.begin())
- lowest_weight = it->first;
- if (beam < 0 || it->first <= (lowest_weight + beam))
- {
- /* print the lookup string */
- print_lookup_string(s.second);
- fprintf(outfile, "\t");
-
- /* and the path that yielded the result string */
- bool first_pair=true;
- for (StringPairVector::const_iterator IT = it->second.begin();
- IT != it->second.end(); IT++) {
- if (print_space && ! first_pair) {
- fprintf(outfile, " ");
+ // Extract output side for testing if it is a valid flag diacritic path
+ StringVector sv;
+ for (StringPairVector::const_iterator spv_it = it->second.begin();
+ spv_it != it->second.end(); spv_it++)
+ {
+ sv.push_back(spv_it->second);
+ }
+
+ if (it == results_spv.begin())
+ lowest_weight = it->first;
+ if (beam < 0 || it->first <= (lowest_weight + beam))
+ {
+ /* print the lookup string */
+ if (input_to_print != NULL)
+ print_lookup_string(input_to_print->second);
+ else
+ print_lookup_string(s.second);
+
+#ifdef WINDOWS
+ if (!pipe_output)
+ hfst_fprintf_console(outfile, "\t");
+ else
+#endif
+ fprintf(outfile, "\t");
+
+ /* and the path that yielded the result string */
+ bool first_pair=true;
+ for (StringPairVector::const_iterator IT = it->second.begin();
+ IT != it->second.end(); IT++) {
+ if (show_flags || ! FdOperation::is_diacritic(IT->second))
+ {
+#ifdef WINDOWS
+ if (!pipe_output)
+ {
+ if (print_space && ! first_pair) {
+ hfst_fprintf_console(outfile, " ");
+ }
+ hfst_fprintf_console(outfile, "%s:%s",
+ get_print_format(IT->first).c_str(),
+ get_print_format(IT->second).c_str());
+ first_pair=false;
+ }
+ else
+#endif
+ {
+ if (print_space && ! first_pair) {
+ fprintf(outfile, " ");
+ }
+ fprintf(outfile, "%s:%s",
+ get_print_format(IT->first).c_str(),
+ get_print_format(IT->second).c_str());
+ first_pair=false;
+ }
+ }
}
- first_pair=false;
- fprintf(outfile, "%s:%s",
- get_print_format(IT->first).c_str(),
- get_print_format(IT->second).c_str());
+ /* and the weight of that path (add the weight of input). */
+#ifdef WINDOWS
+ if (!pipe_output)
+ hfst_fprintf_console(outfile, "\t%f\n", it->first + s.first);
+ else
+#endif
+ fprintf(outfile, "\t%f\n", it->first + s.first);
}
- /* and the weight of that path. */
- fprintf(outfile, "\t%f\n", it->first);
- }
+
}
- fprintf(outfile, "\n");
+ if (!no_newline)
+ {
+#ifdef WINDOWS
+ if (!pipe_output)
+ hfst_fprintf_console(outfile, "\n");
+ else
+#endif
+ fprintf(outfile, "\n");
+ }
+
}
fflush(outfile);
}
// Convert HfstTwoLevelPaths into HfstOneLevelPaths
- for (HfstTwoLevelPaths::const_iterator it = results_spv.begin();
+ for (HfstTwoLevelPaths::const_iterator it = results_spv.begin();
it != results_spv.end(); it++)
{
StringVector sv;
@@ -1274,52 +1393,31 @@ void lookup_fd_and_print(HfstBasicTransducer &t, HfstOneLevelPaths& results,
HfstOneLevelPath path(it->first, sv);
results.insert(path);
}
-
- HfstOneLevelPaths filtered;
- for (HfstOneLevelPaths::iterator res = results.begin();
- res != results.end();
- ++res)
- {
- if (is_valid_flag_diacritic_path(res->second) || !obey_flags)
- {
- StringVector unflagged;
- for (StringVector::const_iterator arc = res->second.begin();
- arc != res->second.end();
- arc++)
- {
- if (show_flags || ! FdOperation::is_diacritic(*arc))
- {
- unflagged.push_back(*arc);
- }
- }
- filtered.insert(HfstOneLevelPath(res->first, unflagged));
- }
- }
- results = filtered;
}
+
HfstOneLevelPaths*
-lookup_simple(const HfstOneLevelPath& s, HfstBasicTransducer& t, bool* infinity)
+lookup_simple(const HfstOneLevelPath& s, HfstBasicTransducer& t, bool* infinity, bool print_pairs_at_this_point=false, bool print_fail=false, const HfstOneLevelPath * input_to_print = NULL, bool no_newline=false)
{
HfstOneLevelPaths* results = new HfstOneLevelPaths;
bool possible = is_possible_to_get_result
- (s, cascade_symbols_seen[transducer_number],
+ (s, cascade_symbols_seen[transducer_number],
cascade_unknown_or_identity_seen[transducer_number]);
- if (possible && t.is_lookup_infinitely_ambiguous(s))
+ if (possible && time_cutoff == 0.0 && t.is_lookup_infinitely_ambiguous(s, obey_flags))
{
if (!silent && infinite_cutoff > 0) {
warning(0, 0, "Got infinite results, number of cycles limited to " SIZE_T_SPECIFIER "",
infinite_cutoff);
}
- lookup_fd_and_print(t, *results, s, infinite_cutoff);
+ lookup_fd_and_print(&t, NULL, *results, s, &infinite_cutoff, print_pairs_at_this_point, print_fail, input_to_print, no_newline);
*infinity = true;
}
else
{
- lookup_fd_and_print(t, *results, s);
+ lookup_fd_and_print(&t, NULL, *results, s, NULL, print_pairs_at_this_point, print_fail, input_to_print, no_newline);
}
if (results->size() == 0)
@@ -1334,7 +1432,7 @@ lookup_simple(const HfstOneLevelPath& s, HfstBasicTransducer& t, bool* infinity)
HfstOneLevelPaths*
-lookup_cascading(const HfstOneLevelPath& s, vector<HfstTransducer> cascade,
+lookup_cascading(const HfstOneLevelPath& s, vector<HfstTransducer> & cascade,
bool* infinity)
{
HfstOneLevelPaths* results = new HfstOneLevelPaths;
@@ -1342,7 +1440,65 @@ lookup_cascading(const HfstOneLevelPath& s, vector<HfstTransducer> cascade,
// go through all transducers in the cascade
for (unsigned int i = 0; i < cascade.size(); i++)
{
- HfstOneLevelPaths* result = lookup_simple(s, cascade[i], infinity);
+ HfstOneLevelPaths* result = NULL;
+
+ if ((cascade_ == CASCADE_COMPOSITION) && (i != 0))
+ {
+ result = new HfstOneLevelPaths;
+ // use previous value of 'results' as input to composition
+ for (HfstOneLevelPaths::const_iterator it = results->begin();
+ it != results->end(); it++)
+ {
+ HfstOneLevelPaths * one_result = lookup_simple(*it, cascade[i], infinity, ((i+1) == cascade.size()), false, &s, true);
+ //HfstOneLevelPaths * one_result = lookup_simple(*it, cascade[i], infinity);
+ for (HfstOneLevelPaths::const_iterator IT = one_result->begin();
+ IT != one_result->end(); IT++)
+ {
+ // add the weights
+ result->insert(HfstOneLevelPath(IT->first + it->first, IT->second));
+ }
+ delete one_result;
+ }
+ // zero 'results'
+ delete results;
+ results = new HfstOneLevelPaths();
+
+ // cascading composition done
+ if ( ((i+1) == cascade.size()) && print_pairs )
+ {
+ // no results from cascading composition
+ if (result->size() == 0)
+ {
+ std::string input;
+ for (StringVector::const_iterator it = s.second.begin(); it != s.second.end(); it++)
+ {
+ input += *it;
+ }
+#ifdef WINDOWS
+ if (!pipe_output)
+ hfst_fprintf_console(outfile, "%s\t%s+?\tinf\n\n", input.c_str(), input.c_str());
+ else
+#endif
+ fprintf(outfile, "%s\t%s+?\tinf\n\n", input.c_str(), input.c_str());
+ }
+ // results printed
+ else
+ {
+#ifdef WINDOWS
+ if (!pipe_output)
+ hfst_fprintf_console(outfile, "\n");
+ else
+#endif
+ fprintf(outfile, "\n");
+ }
+ fflush(outfile);
+ }
+ }
+ else
+ {
+ result = lookup_simple(s, cascade[i], infinity);
+ }
+
if (infinity)
{
verbose_printf("Inf results @ level %u\n", i);
@@ -1356,6 +1512,13 @@ lookup_cascading(const HfstOneLevelPath& s, vector<HfstTransducer> cascade,
{
results->insert(*it);
}
+ delete result;
+
+ if ( (cascade_ == CASCADE_PRIORITY_UNION) && (results->size() != 0) )
+ {
+ verbose_printf("results found @ level %u, skipping rest of transducers (--cascade=priority-union)\n", i);
+ break;
+ }
}
// all transducers gone through
@@ -1364,7 +1527,7 @@ lookup_cascading(const HfstOneLevelPath& s, vector<HfstTransducer> cascade,
HfstOneLevelPaths*
-lookup_cascading(const HfstOneLevelPath& s, vector<HfstBasicTransducer> cascade,
+lookup_cascading(const HfstOneLevelPath& s, vector<HfstBasicTransducer> & cascade,
bool* infinity)
{
HfstOneLevelPaths* results = new HfstOneLevelPaths;
@@ -1373,7 +1536,66 @@ lookup_cascading(const HfstOneLevelPath& s, vector<HfstBasicTransducer> cascade,
for (unsigned int i = 0; i < cascade.size(); i++)
{
transducer_number=i; // needed for lookup_simple
- HfstOneLevelPaths* result = lookup_simple(s, cascade[i], infinity);
+
+ HfstOneLevelPaths* result = NULL;
+ if ((cascade_ == CASCADE_COMPOSITION) && (i != 0))
+ {
+ result = new HfstOneLevelPaths;
+ // use previous value of 'results' as input to composition
+ for (HfstOneLevelPaths::const_iterator it = results->begin();
+ it != results->end(); it++)
+ {
+ //std::cerr << "using previous value of results as input to composition..." << std::endl;
+ // if last transducer in cascade, print results if --print-pairs is requested
+ HfstOneLevelPaths * one_result = lookup_simple(*it, cascade[i], infinity, ((i+1) == cascade.size()), false, &s, true);
+ for (HfstOneLevelPaths::const_iterator IT = one_result->begin();
+ IT != one_result->end(); IT++)
+ {
+ // add the weights
+ result->insert(HfstOneLevelPath(IT->first + it->first, IT->second));
+ }
+ delete one_result;
+ }
+ // zero 'results'
+ delete results;
+ results = new HfstOneLevelPaths();
+
+ // cascading composition done
+ if ( ((i+1) == cascade.size()) && print_pairs)
+ {
+ // no results from cascading composition
+ if (result->size() == 0)
+ {
+ std::string input;
+ for (StringVector::const_iterator it = s.second.begin(); it != s.second.end(); it++)
+ {
+ input += *it;
+ }
+#ifdef WINDOWS
+ if (!pipe_output)
+ hfst_fprintf_console(outfile, "%s\t%s+?\tinf\n\n", input.c_str(), input.c_str());
+ else
+#endif
+ fprintf(outfile, "%s\t%s+?\tinf\n\n", input.c_str(), input.c_str());
+ }
+ // results printed
+ else
+ {
+#ifdef WINDOWS
+ if (!pipe_output)
+ hfst_fprintf_console(outfile, "\n");
+ else
+#endif
+ fprintf(outfile, "\n");
+ }
+ fflush(outfile);
+ }
+ }
+ else
+ {
+ result = lookup_simple(s, cascade[i], infinity, (cascade_ != CASCADE_COMPOSITION), false);
+ }
+
if (infinity)
{
verbose_printf("Inf results @ level %u\n", i);
@@ -1382,20 +1604,26 @@ lookup_cascading(const HfstOneLevelPath& s, vector<HfstBasicTransducer> cascade,
{
verbose_printf("" SIZE_T_SPECIFIER " results @ level %u\n", result->size(), i);
}
+
for (HfstOneLevelPaths::const_iterator it = result->begin();
it != result->end(); it++)
{
results->insert(*it);
}
delete result;
+
+ if ( (cascade_ == CASCADE_PRIORITY_UNION) && (results->size() != 0) )
+ {
+ verbose_printf("results found @ level %u, skipping rest of transducers (--cascade=priority-union)\n", i);
+ break;
+ }
}
// all transducers gone through
-
return results;
}
-// HERE!!! limits kvs with beam
+// limits kvs with beam
void
print_lookups(const HfstOneLevelPaths& kvs,
const HfstOneLevelPath& kv, char* markup,
@@ -1460,7 +1688,7 @@ print_lookups(const HfstOneLevelPaths& kvs,
HfstOneLevelPaths*
-perform_lookups(HfstOneLevelPath& origin, std::vector<HfstTransducer>& cascade,
+perform_lookups(HfstOneLevelPath& origin, std::vector<HfstTransducer>& cascade,
bool unknown, bool* infinite)
{
HfstOneLevelPaths* kvs;
@@ -1468,7 +1696,7 @@ perform_lookups(HfstOneLevelPath& origin, std::vector<HfstTransducer>& cascade,
{
if (cascade.size() == 1)
{
- kvs = lookup_simple(origin, cascade[0], infinite);
+ kvs = lookup_simple(origin, cascade[0], infinite, true, true);
}
else
{
@@ -1482,8 +1710,9 @@ perform_lookups(HfstOneLevelPath& origin, std::vector<HfstTransducer>& cascade,
return kvs;
}
+
HfstOneLevelPaths*
-perform_lookups(HfstOneLevelPath& origin, std::vector<HfstBasicTransducer>& cascade,
+perform_lookups(HfstOneLevelPath& origin, std::vector<HfstBasicTransducer>& cascade,
bool unknown, bool* infinite)
{
HfstOneLevelPaths* kvs;
@@ -1491,7 +1720,7 @@ perform_lookups(HfstOneLevelPath& origin, std::vector<HfstBasicTransducer>& casc
{
if (cascade.size() == 1)
{
- kvs = lookup_simple(origin, cascade[0], infinite);
+ kvs = lookup_simple(origin, cascade[0], infinite, true, true);
}
else
{
@@ -1523,7 +1752,7 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
hfst::ImplementationType type = trans.get_type();
std::set<std::string> symbols_seen;
- if (type != HFST_OL_TYPE && type != HFST_OLW_TYPE)
+ if (type != HFST_OL_TYPE && type != HFST_OLW_TYPE)
{
only_optimized_lookup = false;
}
@@ -1534,16 +1763,16 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
}
if (transducer_n==1)
{
- verbose_printf("Reading %s...\n", inputname);
+ verbose_printf("Reading %s...\n", inputname);
}
else
{
verbose_printf("Reading %s..." SIZE_T_SPECIFIER "\n", inputname,
- transducer_n);
+ transducer_n);
}
// add multicharacter symbols to mc_symbols
- if (type == hfst::SFST_TYPE ||
+ if (type == hfst::SFST_TYPE ||
type == hfst::TROPICAL_OPENFST_TYPE ||
type == hfst::LOG_OPENFST_TYPE ||
type == hfst::FOMA_TYPE)
@@ -1552,7 +1781,7 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
for (HfstBasicTransducer::const_iterator it = basic.begin();
it != basic.end(); it++)
{
- for (HfstBasicTransducer::HfstTransitions::const_iterator
+ for (hfst::implementations::HfstBasicTransitions::const_iterator
tr_it = it->begin(); tr_it != it->end(); tr_it++)
{
std::string mcs = tr_it->get_input_symbol();
@@ -1580,12 +1809,29 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
inputstream.close();
- if (print_pairs &&
- (inputstream.get_type() == HFST_OL_TYPE ||
+ /*
+ if ((cascade_ == CASCADE_COMPOSITION || cascade_ == CASCADE_PRIORITY_UNION) &&
+ (inputstream.get_type() == HFST_OL_TYPE ||
+ inputstream.get_type() == HFST_OLW_TYPE) ) {
+ error(EXIT_FAILURE, 0, "option --cascade not supported on "
+ "optimized lookup transducers");
+ }*/
+
+ /*
+ if (print_pairs &&
+ (inputstream.get_type() == HFST_OL_TYPE ||
inputstream.get_type() == HFST_OLW_TYPE) ) {
error(EXIT_FAILURE, 0, "pair printing not supported on "
"optimized lookup transducers");
}
+ */
+
+ if ((!obey_flags) &&
+ (inputstream.get_type() == HFST_OL_TYPE ||
+ inputstream.get_type() == HFST_OLW_TYPE) ) {
+ error(EXIT_FAILURE, 0, "not obeying flags not supported on "
+ "optimized lookup transducers");
+ }
// if transducer type is other than optimized_lookup,
// convert to HfstBasicTransducer
@@ -1593,14 +1839,14 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
char* line = 0;
size_t llen = 0;
- hfst::HfstStrings2FstTokenizer input_tokenizer(mc_symbols,
+ hfst::HfstStrings2FstTokenizer input_tokenizer(mc_symbols,
std::string(epsilon_format));
if (!only_optimized_lookup)
{
char* format_string = hfst_strformat(cascade[0].get_type());
if (!silent) {
- warning(0, 0,
+ warning(0, 0,
"It is not possible to perform fast lookups with %s "
"format automata.\n"
"Using HFST basic transducer format "
@@ -1699,7 +1945,7 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
unknown, &infinite);
}
- if (! print_pairs) {
+ if (! print_pairs) {
// printing was already done in function lookup_fd
print_lookups(*kvs, *kv, markup, unknown, infinite, outstream);
fflush(outstream);
@@ -1709,7 +1955,7 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
print_prompt();
} // while lines in input
- if (show_progress_bar)
+ if (show_progress_bar)
{
fprintf(stderr, "%ld/%ld... Done\n", filepos, filesize);
}
@@ -1717,7 +1963,7 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
if (print_statistics)
{
fprintf(outstream, "Strings\tFound\tMissing\tResults\n"
- "%lu\t%lu\t%lu\t%lu\n",
+ "%lu\t%lu\t%lu\t%lu\n",
inputs, analysed, no_analyses, analyses);
fprintf(outstream, "Coverage\tAmbiguity\n"
"%f\t%f\n",
@@ -1759,7 +2005,7 @@ int main( int argc, char **argv ) {
{
fclose(inputfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
verbose_printf("Output formats:\n"
" regular:`%s'`%s...'`%s',\n"
@@ -1774,12 +2020,12 @@ int main( int argc, char **argv ) {
epsilon_format, space_format, show_flags);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
- try
+ try
{
instream = (inputfile != stdin) ?
new HfstInputStream(inputfilename) :
new HfstInputStream();
- }
+ }
catch(const HfstException e)
{
error(EXIT_FAILURE, 0, "%s is not a valid transducer file",
diff --git a/tools/src/hfst-minimize.cc b/tools/src/hfst-minimize.cc
index 1b21c77..04108bf 100644
--- a/tools/src/hfst-minimize.cc
+++ b/tools/src/hfst-minimize.cc
@@ -88,13 +88,13 @@ parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
- // add tool-specific options here
+ // add tool-specific options here
{"encode-weights", no_argument, 0, 'E'},
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "E",
long_options, &option_index);
if (-1 == c)
@@ -133,11 +133,11 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
char* inputname = hfst_get_name(trans, inputfilename);
if (transducer_n==1)
{
- verbose_printf("Minimizing %s...\n", inputname);
+ verbose_printf("Minimizing %s...\n", inputname);
}
else
{
- verbose_printf("Minimizing %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
+ verbose_printf("Minimizing %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
}
#ifdef PROFILE
@@ -194,7 +194,7 @@ int main( int argc, char **argv ) {
hfst::set_encode_weights(true);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-multiply.cc b/tools/src/hfst-multiply.cc
index b6a23ad..da261e7 100644
--- a/tools/src/hfst-multiply.cc
+++ b/tools/src/hfst-multiply.cc
@@ -88,12 +88,12 @@ parse_options(int argc, char** argv)
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
{"n-times", required_argument, 0, 'n'},
- // add tool-specific options here
+ // add tool-specific options here
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "n:",
long_options, &option_index);
if (-1 == c)
@@ -136,8 +136,8 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
}
- verbose_printf("Duplicate %s times " SIZE_T_SPECIFIER "..." SIZE_T_SPECIFIER "\n", inputname,
- dupe_count, transducer_n);
+ verbose_printf("Duplicate %s times " SIZE_T_SPECIFIER "..." SIZE_T_SPECIFIER "\n", inputname,
+ dupe_count, transducer_n);
for (unsigned int i = 0; i < dupe_count; i++)
{
outstream << trans;
@@ -170,7 +170,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-name.cc b/tools/src/hfst-name.cc
index 296df56..5b88712 100644
--- a/tools/src/hfst-name.cc
+++ b/tools/src/hfst-name.cc
@@ -95,8 +95,8 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "n:pt:",
long_options, &option_index);
if (-1 == c)
@@ -144,12 +144,12 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
if (transducer_n==1)
{
- verbose_printf("Naming %s...\n", inputfilename);
+ verbose_printf("Naming %s...\n", inputfilename);
}
else
{
- verbose_printf("Naming %s..." SIZE_T_SPECIFIER "\n",
- inputfilename, transducer_n);
+ verbose_printf("Naming %s..." SIZE_T_SPECIFIER "\n",
+ inputfilename, transducer_n);
}
HfstTransducer trans(instream);
@@ -210,7 +210,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-optimized-lookup.cc b/tools/src/hfst-optimized-lookup.cc
index a31e8a4..3b1758f 100644
--- a/tools/src/hfst-optimized-lookup.cc
+++ b/tools/src/hfst-optimized-lookup.cc
@@ -28,7 +28,7 @@
//#ifdef _MSC_VER
//# include <windows.h>
-//#endif
+//#endif
#ifdef _MSC_VER
# include "hfst-string-conversions.h"
@@ -75,14 +75,14 @@ bool print_usage(void)
"N must be a positive integer. B must be a non-negative float.\n" <<
"S must be a non-negative float. The default, 0.0, indicates no cutoff.\n"
"Options -n and -b are combined with AND, i.e. they both restrict the output.\n" <<
- "\n" <<
+ "\n" <<
"STREAM can be { input, output, both }. If not given, defaults to {both}.\n" <<
#ifdef _MSC_VER
"Input is read interactively via the console, i.e. line by line from the user.\n" <<
- "If you redirect input from a file, use --pipe-mode=input. Output is by default\n" <<
+ "If you redirect input from a file, use --pipe-mode=input. Output is by default\n" <<
"printed to the console. If you redirect output to a file, use --pipe-mode=output.\n" <<
#else
- "Input is read interactively line by line from the user. If you redirect input\n" <<
+ "Input is read interactively line by line from the user. If you redirect input\n" <<
"from a file, use --pipe-mode=input. --pipe-mode=output is ignored on non-windows\n" <<
"platforms.\n" <<
#endif
@@ -236,7 +236,7 @@ int main(int argc, char **argv)
strcmp(optarg, "out") == 0 || strcmp(optarg, "OUT") == 0)
{ pipe_output = true; }
else
- { std::cerr << "--pipe-mode argument " << std::string(optarg) << " unrecognised\n\n";
+ { std::cerr << "--pipe-mode argument " << std::string(optarg) << " unrecognised\n\n";
return EXIT_FAILURE; }
break;
@@ -437,8 +437,8 @@ void Encoder::read_input_symbols(KeyTable * kt)
ascii_symbols[(unsigned char)(*p)] = k;
}
// If there's an ascii tokenized symbol shadowing this, remove it
- if (strlen(p) > 1 &&
- (unsigned char)(*p) <= 127 &&
+ if (strlen(p) > 1 &&
+ (unsigned char)(*p) <= 127 &&
ascii_symbols[(unsigned char)(*p)] != NO_SYMBOL_NUMBER) {
ascii_symbols[(unsigned char)(*p)] = NO_SYMBOL_NUMBER;
}
@@ -466,7 +466,7 @@ void runTransducer (genericTransducer T)
input_string[i] = NO_SYMBOL_NUMBER;
}
- char * str = (char*)(malloc(MAX_IO_STRING*sizeof(char)));
+ char * str = (char*)(malloc(MAX_IO_STRING*sizeof(char)));
*str = 0;
char * old_str = str;
@@ -480,7 +480,7 @@ void runTransducer (genericTransducer T)
if (! hfst::get_line_from_console(linestr, MAX_IO_STRING*sizeof(char)))
break;
str = strdup(linestr.c_str());
- old_str = str;
+ old_str = str;
}
else
#endif
@@ -742,7 +742,7 @@ void IndexTableReader::get_index_vector(void)
{
size_t j = i * TransitionIndex::SIZE;
SymbolNumber * input = (SymbolNumber*)(TableIndices + j);
- TransitionTableIndex * index =
+ TransitionTableIndex * index =
(TransitionTableIndex*)(TableIndices + j + sizeof(SymbolNumber));
indices.push_back(new TransitionIndex(*input,*index));
}
@@ -766,9 +766,9 @@ void TransitionTableReader::get_transition_vector(void)
{
size_t j = i * Transition::SIZE;
SymbolNumber * input = (SymbolNumber*)(TableTransitions + j);
- SymbolNumber * output =
+ SymbolNumber * output =
(SymbolNumber*)(TableTransitions + j + sizeof(SymbolNumber));
- TransitionTableIndex * target =
+ TransitionTableIndex * target =
(TransitionTableIndex*)(TableTransitions + j + 2 * sizeof(SymbolNumber));
transitions.push_back(new Transition(*input,
*output,
@@ -785,7 +785,7 @@ bool TransitionTableReader::Matches(SymbolNumber s)
bool TransitionTableReader::get_finality(TransitionTableIndex i)
{
- if (i >= TRANSITION_TARGET_TABLE_START)
+ if (i >= TRANSITION_TARGET_TABLE_START)
{
return transitions[i - TRANSITION_TARGET_TABLE_START]->final();
}
@@ -891,7 +891,7 @@ void Transducer::try_epsilon_indices(SymbolNumber * input_symbol,
try_epsilon_transitions(input_symbol,
output_symbol,
original_output_string,
- indices[i]->target() -
+ indices[i]->target() -
TRANSITION_TARGET_TABLE_START);
}
}
@@ -940,7 +940,7 @@ void Transducer::find_index(SymbolNumber input,
input_symbol,
output_symbol,
original_output_string,
- indices[i+input]->target() -
+ indices[i+input]->target() -
TRANSITION_TARGET_TABLE_START);
}
}
@@ -1115,7 +1115,7 @@ void Transducer::printAnalyses(std::string prepend)
else
#endif
std::cout << prepend << "\t";
- }
+ }
#ifdef WINDOWS
if (!pipe_output)
@@ -1352,7 +1352,7 @@ void IndexTableReaderW::get_index_vector(void)
{
size_t j = i * TransitionWIndex::SIZE;
SymbolNumber * input = (SymbolNumber*)(TableIndices + j);
- TransitionTableIndex * index =
+ TransitionTableIndex * index =
(TransitionTableIndex*)(TableIndices + j + sizeof(SymbolNumber));
indices.push_back(new TransitionWIndex(*input,*index));
}
@@ -1376,9 +1376,9 @@ void TransitionTableReaderW::get_transition_vector(void)
{
size_t j = i * TransitionW::SIZE;
SymbolNumber * input = (SymbolNumber*)(TableTransitions + j);
- SymbolNumber * output =
+ SymbolNumber * output =
(SymbolNumber*)(TableTransitions + j + sizeof(SymbolNumber));
- TransitionTableIndex * target =
+ TransitionTableIndex * target =
(TransitionTableIndex*)(TableTransitions + j + 2 * sizeof(SymbolNumber));
Weight * weight =
(Weight*)(TableTransitions + j + 2 * sizeof(SymbolNumber) + sizeof(TransitionTableIndex));
@@ -1400,7 +1400,7 @@ bool TransitionTableReaderW::Matches(SymbolNumber s)
bool TransitionTableReaderW::get_finality(TransitionTableIndex i)
{
- if (i >= TRANSITION_TARGET_TABLE_START)
+ if (i >= TRANSITION_TARGET_TABLE_START)
{
return transitions[i - TRANSITION_TARGET_TABLE_START]->final();
}
@@ -1426,7 +1426,7 @@ void TransducerW::set_symbol_table(void)
void TransducerW::try_epsilon_transitions(SymbolNumber * input_symbol,
SymbolNumber * output_symbol,
- SymbolNumber *
+ SymbolNumber *
original_output_string,
TransitionTableIndex i)
{
@@ -1434,7 +1434,7 @@ void TransducerW::try_epsilon_transitions(SymbolNumber * input_symbol,
std::cerr << "try epsilon transitions " << i << " " << current_weight << std::endl;
#endif
- if (transitions.size() <= i)
+ if (transitions.size() <= i)
{
return;
}
@@ -1455,7 +1455,7 @@ void TransducerW::try_epsilon_transitions(SymbolNumber * input_symbol,
void TransducerWFd::try_epsilon_transitions(SymbolNumber * input_symbol,
SymbolNumber * output_symbol,
- SymbolNumber *
+ SymbolNumber *
original_output_string,
TransitionTableIndex i)
{
@@ -1521,7 +1521,7 @@ void TransducerW::try_epsilon_indices(SymbolNumber * input_symbol,
try_epsilon_transitions(input_symbol,
output_symbol,
original_output_string,
- indices[i]->target() -
+ indices[i]->target() -
TRANSITION_TARGET_TABLE_START);
}
}
@@ -1536,7 +1536,7 @@ void TransducerW::find_transitions(SymbolNumber input,
std::cerr << "find transitions " << i << " " << current_weight << std::endl;
#endif
- if (transitions.size() <= i)
+ if (transitions.size() <= i)
{
return;
}
@@ -1571,7 +1571,7 @@ void TransducerW::find_index(SymbolNumber input,
#if OL_FULL_DEBUG
std::cerr << "find index " << i << " " << current_weight << std::endl;
#endif
- if (indices.size() <= i)
+ if (indices.size() <= i)
{
return;
}
@@ -1583,7 +1583,7 @@ void TransducerW::find_index(SymbolNumber input,
input_symbol,
output_symbol,
original_output_string,
- indices[i+input]->target() -
+ indices[i+input]->target() -
TRANSITION_TARGET_TABLE_START);
}
}
@@ -1665,7 +1665,7 @@ void TransducerW::printAnalyses(std::string prepend)
{
#ifdef WINDOWS
if (!pipe_output)
- hfst_fprintf_console(stdout, "%s\t", prepend.c_str());
+ hfst_fprintf_console(stdout, "%s\t", prepend.c_str());
else
#endif
std::cout << prepend << "\t";
@@ -1824,7 +1824,7 @@ void TransducerWFdUniq::printAnalyses(std::string prepend)
}
std::multimap<Weight, std::string>::iterator display_it;
for (display_it = weight_sorted_map.begin();
- display_it != weight_sorted_map.end(), i < maxAnalyses;
+ display_it != weight_sorted_map.end() && i < maxAnalyses;
display_it++, i++)
{
if (outputType == xerox)
@@ -1903,7 +1903,7 @@ void TransducerW::get_analyses(SymbolNumber * input_symbol,
if (*input_symbol == NO_SYMBOL_NUMBER)
{
*output_symbol = NO_SYMBOL_NUMBER;
- if (transitions.size() <= i)
+ if (transitions.size() <= i)
{
return;
}
diff --git a/tools/src/hfst-optimized-lookup.h b/tools/src/hfst-optimized-lookup.h
index 7e19b4c..05ffa98 100644
--- a/tools/src/hfst-optimized-lookup.h
+++ b/tools/src/hfst-optimized-lookup.h
@@ -353,7 +353,7 @@ protected:
public:
// Each TransitionIndex has an input symbol and a target index.
- static const size_t SIZE =
+ static const size_t SIZE =
sizeof(SymbolNumber) + sizeof(TransitionTableIndex);
TransitionIndex(SymbolNumber input,
@@ -390,9 +390,9 @@ protected:
public:
- // Each transition has an input symbol an output symbol and
+ // Each transition has an input symbol an output symbol and
// a target index.
- static const size_t SIZE =
+ static const size_t SIZE =
2 * sizeof(SymbolNumber) + sizeof(TransitionTableIndex);
Transition(SymbolNumber input,
@@ -437,7 +437,7 @@ private:
void get_index_vector(void);
public:
IndexTableReader(FILE * f,
- TransitionTableIndex index_count):
+ TransitionTableIndex index_count):
number_of_table_entries(index_count)
{
table_size = number_of_table_entries*TransitionIndex::SIZE;
@@ -525,8 +525,8 @@ public:
bool get_finality(TransitionTableIndex i);
TransitionVector &operator() (void)
- {
- return transitions;
+ {
+ return transitions;
}
};
@@ -706,7 +706,7 @@ private:
public:
// Each TransitionIndex has an input symbol and a target index.
- static const size_t SIZE =
+ static const size_t SIZE =
sizeof(SymbolNumber) + sizeof(TransitionTableIndex);
TransitionWIndex(SymbolNumber input,
@@ -749,15 +749,15 @@ class TransitionW
{
private:
SymbolNumber input_symbol;
- SymbolNumber output_symbol;
+ SymbolNumber output_symbol;
TransitionTableIndex target_index;
Weight transition_weight;
public:
- // Each transition has an input symbol an output symbol and
+ // Each transition has an input symbol an output symbol and
// a target index, as well as a weight.
- static const size_t SIZE =
+ static const size_t SIZE =
2 * sizeof(SymbolNumber) + sizeof(TransitionTableIndex) + sizeof(Weight);
TransitionW(SymbolNumber input,
@@ -818,7 +818,7 @@ private:
void get_index_vector(void);
public:
IndexTableReaderW(FILE * f,
- TransitionTableIndex index_count):
+ TransitionTableIndex index_count):
number_of_table_entries(index_count)
{
table_size = number_of_table_entries*TransitionWIndex::SIZE;
@@ -906,8 +906,8 @@ public:
bool get_finality(TransitionTableIndex i);
TransitionWVector &operator() (void)
- {
- return transitions;
+ {
+ return transitions;
}
};
diff --git a/tools/src/hfst-pair-test-commandline.h b/tools/src/hfst-pair-test-commandline.h
index 23e0ad7..4ce22fe 100644
--- a/tools/src/hfst-pair-test-commandline.h
+++ b/tools/src/hfst-pair-test-commandline.h
@@ -106,8 +106,8 @@ void read_rule_names(size_t number_of_rules)
ifstream rule_name_in(rule_names_file_name,std::ios::in);
size_t counter = 0;
char rule_name[1000];
- while (rule_name_in.getline(rule_name,1000) and (counter < number_of_rules))
- {
+ while (rule_name_in.getline(rule_name,1000) and (counter < number_of_rules))
+ {
++counter;
rule_transducer_names->push_back(strdup(rule_name));
}
@@ -139,7 +139,7 @@ char * get_skip(void)
}
/* Display program name and possible parameters with their argument types. */
-void usage(void)
+void usage(void)
{
fprintf(message_out,"USAGE: %s [ OPTIONS ] [ FSTFILE ]\n",
program_name);
@@ -196,7 +196,7 @@ void help(void)
"<https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstPairTest>\n\n"
"Report bugs to <hfst-bugs at helsinki.fi>\n"
);
-}
+}
/* Parse the command line argument list. */
void parse_options(int argc, char * argv[])
{
@@ -228,12 +228,12 @@ void parse_options(int argc, char * argv[])
switch (c)
{
- case 'd': /* debug */
+ case 'd': /* debug */
debug = true;
fprintf(stderr,
"Debug mode.\n");
break;
- case 'v': /* debug */
+ case 'v': /* debug */
verbose = true;
fprintf(stderr,
"Verbose mode.\n");
@@ -273,16 +273,16 @@ void parse_options(int argc, char * argv[])
"Reading symbol-pair strings from the file %s.\n",
fst_file_name);
break;
- case 'S': /* spaces */
+ case 'S': /* spaces */
spaces = true;
if (verbose or debug)
fprintf(stderr,
"Input string pairs separated by spaces.\n");
break;
- case 's': /* silent */
+ case 's': /* silent */
silent = true;
break;
- case 'q': /* quiet */
+ case 'q': /* quiet */
silent = true;
break;
case 'R': /* read-symbols FILE */
@@ -311,7 +311,7 @@ void parse_options(int argc, char * argv[])
"missing a file name\n",argv[optind-1]);
usage();
exit(1);
- }
+ }
}
if (output_file_name == NULL)
@@ -324,4 +324,4 @@ void parse_options(int argc, char * argv[])
fst_file_name = strdup(argv[optind]);
}
-}
+}
diff --git a/tools/src/hfst-pair-test-transducers.h b/tools/src/hfst-pair-test-transducers.h
index aeee6d4..aa088bb 100644
--- a/tools/src/hfst-pair-test-transducers.h
+++ b/tools/src/hfst-pair-test-transducers.h
@@ -32,7 +32,7 @@ void display_pair_vector(KeyPairVector * v) {
++it )
{
KeyPair * kp = *it;
- Symbol
+ Symbol
input(get_key_symbol(kp->lower_char(),rule_keys)),
output(get_key_symbol(kp->upper_char(),rule_keys));
const char *
@@ -53,15 +53,15 @@ void display_pair_vector(KeyPairVector * v) {
}
/* Create the tokenizer, which will be used to convert input strings
- into KeyVectors.
+ into KeyVectors.
The tokenizer transducer can't handle epsilon, so it has to be replaced by
- an internal epsilon not likely to be encountered (PAIR::TEST::EPSILON). */
-void make_tokenizer(void)
+ an internal epsilon not likely to be encountered (PAIR::TEST::EPSILON). */
+void make_tokenizer(void)
{
/* epsilon_representation = rule_keys->get_unused_key();
- non_epsilon_rule_keys =
+ non_epsilon_rule_keys =
recode_key_table(rule_keys,internal_epsilon);
*/
tokenizer = longest_match_tokenizer2(rule_keys);
@@ -74,7 +74,7 @@ void make_tokenizer(void)
/* Return the symbol number corresponding to the symbol name symbol_name.
If symbol_name has no symbol number, return -1. */
int get_symbol_number(char * symbol_name) {
- if (not is_symbol(symbol_name))
+ if (not is_symbol(symbol_name))
{
if (debug)
{
@@ -86,10 +86,10 @@ int get_symbol_number(char * symbol_name) {
return get_symbol(symbol_name);
}
-/* Return the symbol number key corresponding to the symbol number
+/* Return the symbol number key corresponding to the symbol number
symbol_number. If symbol_number has no key, return -1. */
int get_symbol_number_key(Symbol symbol_number, char * symbol_name) {
- if (not is_symbol(symbol_number,rule_keys))
+ if (not is_symbol(symbol_number,rule_keys))
{
if (debug)
{
@@ -101,8 +101,8 @@ int get_symbol_number_key(Symbol symbol_number, char * symbol_name) {
}
/* Split symbol_pair_string at colon-character and attempt to code the pieces
- into a KeyPair accoring to the coding given by KeyTable rule_keys. If
- rule_keys, doesn't know one of the symbols, return NULL.
+ into a KeyPair accoring to the coding given by KeyTable rule_keys. If
+ rule_keys, doesn't know one of the symbols, return NULL.
symbol_pair_string should be set to a non-empty string. */
KeyPair * split_symbol_pair(char * symbol_pair_string) {
@@ -176,7 +176,7 @@ bool test_correspondence(KeyPairVector * input_vector,
}
bool passed_test;
// Rather complicatd way to transform a KeyPairVector into a transducer...
- TransducerHandle input_fst =
+ TransducerHandle input_fst =
disjunct_as_trie(HFST::create_empty_transducer(),
input_vector);
@@ -236,7 +236,7 @@ void delete_key_pair_vector(KeyPairVector * v)
{
for(KeyPairVector::iterator it = v->begin();
it != v->end();
- ++it )
+ ++it )
{
delete *it;
}
@@ -245,25 +245,25 @@ void delete_key_pair_vector(KeyPairVector * v)
/* Divide the string input_string at spaces. Skip multiple spaces. The segments
which are formed are symbol-pairs. These are divided at colon-characters.
- The string of symbol-pairs, which is formed, is coded into a KeyPairVector
+ The string of symbol-pairs, which is formed, is coded into a KeyPairVector
according to the coding given by KeyTable * rule_keys. */
KeyPairVector * split_at_spaces(char * input_string)
{
// Result vector contans the input symbol-pair string tokenized.
KeyPairVector * tokenized_input = new KeyPairVector;
- // If something goes wrong, the original string may be printed
+ // If something goes wrong, the original string may be printed
// as part of a debug-message.
char * original_input_string = strdup(input_string);
// A container for the string representations of the symbol-pairs, which
- // are obtained by splittng input_string at spaces. This is needed, since
+ // are obtained by splittng input_string at spaces. This is needed, since
// strtok can't be used in two tokenization tasks intertwined. Hence we need
- // to first split input_string into pairs and then split the pairs into
+ // to first split input_string into pairs and then split the pairs into
// symbols.
- SymbolNamePairVector symbol_name_pairs;
+ SymbolNamePairVector symbol_name_pairs;
- char * symbol_pair_string = strtok(input_string," ");
+ char * symbol_pair_string = strtok(input_string," ");
do {
// Divide the symbol-pair string into symbol-pairs at spaces.
// and store the pairs in symbol_name_pairs.
@@ -278,7 +278,7 @@ KeyPairVector * split_at_spaces(char * input_string)
++it)
{
symbol_pair_string = *it;
- KeyPair * symbol_pair =
+ KeyPair * symbol_pair =
split_symbol_pair(symbol_pair_string);
if (symbol_pair == NULL)
{
@@ -301,7 +301,7 @@ KeyPairVector * duplicate_keys(KeyVector * v) {
KeyPairVector * pv = new KeyPairVector;
for(KeyVector::iterator it = v->begin();
it != v->end();
- ++it )
+ ++it )
{
pv->push_back(new KeyPair(*it));
}
@@ -309,7 +309,7 @@ KeyPairVector * duplicate_keys(KeyVector * v) {
return pv;
}
-KeyPairVector * construct_pair_vector(vector<KeyVector*> &input_fragments)
+KeyPairVector * construct_pair_vector(vector<KeyVector*> &input_fragments)
{
if (input_fragments.size() == 0)
{ return new KeyPairVector; }
@@ -339,7 +339,7 @@ KeyPairVector * construct_pair_vector(vector<KeyVector*> &input_fragments)
{
for(KeyVector::iterator jt = (*it)->begin();
jt != (*it)->end() - 1;
- ++jt)
+ ++jt)
{
pairs->push_back(new KeyPair(*jt));
}
@@ -364,7 +364,7 @@ KeyPairVector * construct_pair_vector(vector<KeyVector*> &input_fragments)
pairs = NULL;
continue;
}
- else
+ else
{
Key first_key = *((*it)->begin());
pairs->push_back(new KeyPair(previous_key,first_key));
@@ -381,14 +381,14 @@ KeyPairVector * construct_pair_vector(vector<KeyVector*> &input_fragments)
return pairs;
}
-/* Split input_string into fragments at colon-characters. Do a longest prefix
- tokenization of the fragments obtained according to the coding given by
- KeyTable * rule_keys. If one of the fragments can't be tokenized return
- NULL. Build a KeyPairVector of the fragments by pairing up the last token
- and the first token of two adjacent fragments. The first token of the first
+/* Split input_string into fragments at colon-characters. Do a longest prefix
+ tokenization of the fragments obtained according to the coding given by
+ KeyTable * rule_keys. If one of the fragments can't be tokenized return
+ NULL. Build a KeyPairVector of the fragments by pairing up the last token
+ and the first token of two adjacent fragments. The first token of the first
fragment, the last token of the last fragment and all tokens n the middle of
a fragment represent identity-pairs. */
-KeyPairVector * tokenize(char * input_string)
+KeyPairVector * tokenize(char * input_string)
{
vector<KeyVector*> input_fragments;
@@ -463,7 +463,7 @@ KeyPairVector * tokenize(char * input_string)
void display_rule_keys(void)
{
- if ( rule_keys == NULL )
+ if ( rule_keys == NULL )
{
fprintf(stderr,"No key table stored with the rules.\n");
}
@@ -475,7 +475,7 @@ void display_rule_keys(void)
/* Unless the key table rule_keys has been set, attempt to read a key table
from the file named char * symbol_file_name. Do nothing, if rule_keys is
already set. exit, if rule_keys is not set and alphabet cannot be read. */
-void get_alphabet(char * symbol_file_name)
+void get_alphabet(char * symbol_file_name)
{
if ( rule_keys != NULL )
{
@@ -508,7 +508,7 @@ FstVector * read_rule_transducers(void)
// in order to give diagnostics.
int format = UNWEIGHTED;
- if ( fst_file_name == NULL )
+ if ( fst_file_name == NULL )
{
// Count the rules, in order to display information.
size_t counter = 0;
@@ -519,7 +519,7 @@ FstVector * read_rule_transducers(void)
"that you're actually giving something as input)\n\n");
while ((format = read_format()) == UNWEIGHTED)
{
- if ( rule_keys == NULL and has_symbol_table(cin))
+ if ( rule_keys == NULL and has_symbol_table(cin))
{
rule_keys = create_key_table();
rules->push_back(read_transducer(cin,rule_keys));
@@ -538,7 +538,7 @@ FstVector * read_rule_transducers(void)
ifstream fst_in(fst_file_name);
while ((format = read_format(fst_in)) == UNWEIGHTED)
{
- if ( rule_keys == NULL and has_symbol_table(fst_in))
+ if ( rule_keys == NULL and has_symbol_table(fst_in))
{
rule_keys = create_key_table();
rules->push_back(read_transducer(fst_in,rule_keys));
@@ -565,7 +565,7 @@ FstVector * read_rule_transducers(void)
fprintf(stderr,
"Error: You gave weighted input transducers.\n");
}
- else
+ else
{
fprintf(stderr,
"Error: Unknown input transducer format.\n");
diff --git a/tools/src/hfst-pair-test.cc b/tools/src/hfst-pair-test.cc
index 89fdd71..fd236ce 100644
--- a/tools/src/hfst-pair-test.cc
+++ b/tools/src/hfst-pair-test.cc
@@ -77,7 +77,7 @@ print_usage()
"\n", program_name);
print_common_program_options(message_out);
- fprintf(message_out,
+ fprintf(message_out,
"Input/Output options:\n"
" -i, --input=INFILE Read input rule file from INFILE\n"
" -o, --output=OUTFILE Write test output to OUTFILE\n"
@@ -111,11 +111,11 @@ print_usage()
"positive mode, all of the pair strings should be allowed and in\n"
"negative mode they should be disallowed. In Xerox mode the cases\n"
"are read from a twolc source file and both positive and negative\n"
- "cases can occur.\n"
+ "cases can occur.\n"
);
fprintf(message_out, "\n");
fprintf(message_out,
- "Ordinarily, positive test mode is in use. Option -N switches to\n"
+ "Ordinarily, positive test mode is in use. Option -N switches to\n"
"negative test mode. The exit code for a successful test is 0. \n"
"The exit code is 1 otherwise. A successful test will print\n"
"\"Test passed\". A failing test prints \"Test failed\" and\n"
@@ -170,8 +170,8 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "I:NX",
long_options, &option_index);
if (-1 == c)
@@ -208,15 +208,15 @@ parse_options(int argc, char** argv)
#include "inc/check-params-unary.h"
if (inputfilename == std::string("<stdin>"))
- {
- error(EXIT_FAILURE, 0,
+ {
+ error(EXIT_FAILURE, 0,
"The rule transducer file needs to be given using option -i."//,
/*inputfilename*/);
}
return EXIT_CONTINUE;
}
-std::string &replace_all_substr(const std::string &substr,
+std::string &replace_all_substr(const std::string &substr,
const std::string &repl,
std::string &str)
{
@@ -231,7 +231,7 @@ std::string &replace_all_substr(const std::string &substr,
#define PTPC "PAIR_TEST_PERC_COL"
// perc_escaped is a string where special symols are escaped using
// %. Transform it into a string where specail symbols are escaped
-// using \.
+// using \.
std::string backslash_escape(std::string perc_escaped)
{
replace_all_substr("%%", PTPP, perc_escaped);
@@ -248,11 +248,11 @@ HfstState get_target(const std::string &isymbol,
{
HfstState identity_target = -1;
- for (HfstBasicTransducer::HfstTransitions::const_iterator it = t[s].begin();
+ for (hfst::implementations::HfstBasicTransitions::const_iterator it = t[s].begin();
it != t[s].end();
++it)
{
- if (it->get_input_symbol() == isymbol and
+ if (it->get_input_symbol() == isymbol and
it->get_output_symbol() == osymbol)
{ return it->get_target_state(); }
if (it->get_input_symbol() == "@_IDENTITY_SYMBOL_@" and
@@ -260,7 +260,7 @@ HfstState get_target(const std::string &isymbol,
{ identity_target = it->get_target_state(); }
}
- if
+ if
(isymbol == osymbol and known_symbols.find(isymbol) == known_symbols.end())
{ return identity_target; }
else
@@ -283,7 +283,7 @@ int test(const StringPairVector &tokenized_pair_string,
{
s = get_target(it->first,it->second,s,t,known_symbols);
if (s == (unsigned int)-1)
- {
+ {
if (positive)
{ return 1; }
else
@@ -297,7 +297,7 @@ int test(const StringPairVector &tokenized_pair_string,
{ return 1; }
else if (not is_final_state(s,t))
{ return 0; }
- else
+ else
{ return 1; }
}
@@ -417,22 +417,22 @@ int test(const StringPairVector &tokenized_pair_string,
if (positive)
{
if (positive_exit_code == 1 and not silent)
- {
- fprintf(outfile,"FAIL: %s REJECTED\n\n",pair_string.c_str());
+ {
+ fprintf(outfile,"FAIL: %s REJECTED\n\n",pair_string.c_str());
}
if (positive_exit_code == 0 and verbose)
- {
+ {
fprintf(outfile,"%s PASSED\n\n",pair_string.c_str()); }
return positive_exit_code;
}
else
{
if (negative_exit_code == 1 and not silent)
- {
- fprintf(outfile,"FAIL: %s PASSED\n\n",pair_string.c_str());
+ {
+ fprintf(outfile,"FAIL: %s PASSED\n\n",pair_string.c_str());
}
if (negative_exit_code == 0 and verbose)
- {
+ {
fprintf(outfile,"%s REJECTED\n\n",pair_string.c_str()); }
return negative_exit_code;
}
@@ -467,11 +467,11 @@ void get_symbols(HfstBasicTransducer &t,SymbolSet &known_symbols)
{
for (HfstBasicTransducer::const_iterator it = t.begin(); it != t.end(); ++it)
{
- for (HfstBasicTransducer::HfstTransitions::const_iterator jt =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator jt =
it->begin();
jt != it->end();
++jt)
- {
+ {
known_symbols.insert(jt->get_input_symbol());
known_symbols.insert(jt->get_output_symbol());
}
@@ -510,12 +510,12 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
transducer_n++;
if (transducer_n==1)
{
- verbose_printf("Reading %s...\n", inputfilename);
+ verbose_printf("Reading %s...\n", inputfilename);
}
else
{
verbose_printf("Reading %s..." SIZE_T_SPECIFIER "\n", inputfilename,
- transducer_n);
+ transducer_n);
}
HfstTransducer trans(inputstream);
rule_transducer_type = trans.get_type();
@@ -581,14 +581,14 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
(tokenized_pair_string.end(),
StringPair("@#@",hfst::internal_epsilon));
- new_exit_code =
+ new_exit_code =
test(tokenized_pair_string,line,grammar,rule_names,
positive_test,outfile,known_symbols);
}
catch (const hfst::UnescapedColsFound &e)
{
- error(EXIT_FAILURE, 0,
+ error(EXIT_FAILURE, 0,
"The correspondence %s contains unquoted colon-symbols. If "
"you want to input pairs where either symbol is epsilon, "
"use 0 e.g. \"m a s s 0:e s\".\n",
@@ -601,7 +601,7 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
{ exit_code = new_exit_code; }
} // while lines in input
- free(line);
+ free(line);
}
else
{
@@ -643,8 +643,8 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
test_case = strip_space(test_case);
positive_test_cases.push_back(test_case);
- verbose_printf("Positive test case: %s...\n",
- test_case.c_str());
+ verbose_printf("Positive test case: %s...\n",
+ test_case.c_str());
}
else if (is_negative_test_line(line))
{
@@ -653,26 +653,26 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
test_case = strip_space(test_case);
negative_test_cases.push_back(test_case);
- verbose_printf("Negative test case: %s %s...\n",
- line, test_case.c_str());
+ verbose_printf("Negative test case: %s %s...\n",
+ line, test_case.c_str());
}
else
{ continue; }
} // while lines in input
- free(line);
+ free(line);
if (positive_test_cases.size() % 2 != 0)
{
- error(EXIT_FAILURE, 0,
+ error(EXIT_FAILURE, 0,
"Got an odd number of positive test cases. Every input string\n"
"has to have an output string.\n");
}
if (negative_test_cases.size() % 2 != 0)
{
- error(EXIT_FAILURE, 0,
+ error(EXIT_FAILURE, 0,
"Got an odd number of negative test cases. Every input string\n"
"has to have an output string.\n");
}
@@ -683,12 +683,12 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
const std::string &output_case = positive_test_cases[i + 1];
StringPairVector test_case;
- try
+ try
{
// We need to convert the %-escaped input and output
// string to \-escpaed strings for input_toknizer.
test_case = input_tokenizer.tokenize_string_pair
- (backslash_escape(input_case) + ":" +
+ (backslash_escape(input_case) + ":" +
backslash_escape(output_case), false);
test_case.insert
(test_case.begin(),
@@ -700,10 +700,10 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
}
catch (const hfst::UnescapedColsFound &e)
{
- error(EXIT_FAILURE, 0,
+ error(EXIT_FAILURE, 0,
"The correspondence %s %s contains unescaped "
"colon-symbols. Escape them using %%.",
- input_case.c_str(), output_case.c_str(), line);
+ input_case.c_str(), output_case.c_str()/*, line*/);
}
int new_exit_code = test(test_case,
@@ -729,7 +729,7 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
// We need to convert the %-escaped input and output
// string to \-escpaed strings for input_toknizer.
test_case = input_tokenizer.tokenize_string_pair
- (backslash_escape(input_case) + ":" +
+ (backslash_escape(input_case) + ":" +
backslash_escape(output_case), false);
test_case.insert
(test_case.begin(),
@@ -740,10 +740,10 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
}
catch (const hfst::UnescapedColsFound &e)
{
- error(EXIT_FAILURE, 0,
+ error(EXIT_FAILURE, 0,
"The correspondence %s %s contains unquoted "
"colon-symbols. Quote them using %%.",
- input_case.c_str(), output_case.c_str(), line);
+ input_case.c_str(), output_case.c_str()/*, line*/);
}
int new_exit_code = test(test_case,
@@ -780,17 +780,17 @@ int main( int argc, char **argv ) {
{
fclose(inputfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
- try
+ try
{
instream = (inputfile != stdin) ?
new HfstInputStream(inputfilename) :
new HfstInputStream();
- }
+ }
catch(const HfstException e)
{
error(EXIT_FAILURE, 0, "%s is not a valid transducer file",
diff --git a/tools/src/hfst-pmatch.cc b/tools/src/hfst-pmatch.cc
index c8f821a..aa5d9cf 100644
--- a/tools/src/hfst-pmatch.cc
+++ b/tools/src/hfst-pmatch.cc
@@ -57,11 +57,18 @@ using std::pair;
#include "inc/globals-unary.h"
static bool blankline_separated = true;
-static bool extract_tags = false;
-static bool locate_mode = false;
+
+enum var_val{on, off, not_defined};
+static var_val count_patterns = not_defined;
+static var_val delete_patterns = not_defined;
+static var_val extract_patterns = not_defined;
+static var_val locate_mode = not_defined;
+static var_val mark_patterns = not_defined;
+static int max_recursion = -1;
+static int max_context = -1;
+
static double time_cutoff = 0.0;
static bool profile = false;
-std::string pmatch_filename;
void
print_usage()
@@ -71,13 +78,19 @@ print_usage()
"perform matching/lookup on text streams\n"
"\n", program_name);
print_common_program_options(message_out);
+ print_common_unary_program_options(message_out);
+ fprintf(message_out,
+ " -n --newline Newline as input separator (default is blank line)\n"
+ " -x --extract-patterns Only print tagged parts in output\n"
+ " -l --locate Only print locations of matches\n"
+ " -c --count-patterns Print the total number of matches when done\n"
+ " --delete-patterns Replace matches with opening tags\n"
+ " --no-mark-patterns Don't tag matched patterns\n"
+ " --max-context Upper limit to context length allowed\n"
+ " --max-recursion Upper limit for recursion\n"
+ " -t, --time-cutoff=S Limit search after having used S seconds per input\n"
+ " -p --profile Produce profiling data\n");
fprintf(message_out,
- " -n --newline Newline as input separator (default is blank line)\n"
- " -x --extract-tags Only print tagged parts in output\n"
- " -l --locate Only print locations of matches\n"
- " -t, --time-cutoff=S Limit search after having used S seconds per input\n"
- " -p --profile Produce profiling data\n");
- fprintf(message_out,
"Use standard streams for input and output.\n"
"\n"
);
@@ -96,7 +109,7 @@ void match_and_print(hfst_ol::PmatchContainer & container,
// Remove final newline
input_text.erase(input_text.size() -1, 1);
}
- if (!locate_mode) {
+ if (!container.is_in_locate_mode()) {
#ifndef _MSC_VER
outstream << container.match(input_text, time_cutoff);
#else
@@ -129,7 +142,7 @@ int process_input(hfst_ol::PmatchContainer & container,
while (true) {
#ifndef _MSC_VER
- if (!(hfst_getline(&line, &len, inputfile) > 0))
+ if (!(hfst_getline(&line, &len, stdin) > 0))
break;
#else
std::string linestr("");
@@ -138,7 +151,7 @@ int process_input(hfst_ol::PmatchContainer & container,
break;
line = strdup(linestr.c_str());
#endif
-
+
if (!blankline_separated) {
// newline separated
input_text = line;
@@ -156,6 +169,9 @@ int process_input(hfst_ol::PmatchContainer & container,
if (blankline_separated && !input_text.empty()) {
match_and_print(container, outstream, input_text);
}
+ if (count_patterns == on) {
+ outstream << "\n" << container.get_pattern_count_info() << "\n";
+ }
if (profile) {
outstream << "\n" << container.get_profiling_info() << "\n";
}
@@ -172,16 +188,22 @@ int parse_options(int argc, char** argv)
static const struct option long_options[] =
{
HFST_GETOPT_COMMON_LONG,
+ HFST_GETOPT_UNARY_LONG,
{"newline", no_argument, 0, 'n'},
- {"extract-tags", no_argument, 0, 'x'},
+ {"extract-patterns", no_argument, 0, 'x'},
{"locate", no_argument, 0, 'l'},
+ {"count-patterns", no_argument, 0, 'c'},
+ {"delete-patterns", no_argument, 0, 'z'},
+ {"no-mark-patterns", no_argument, 0, 'm'},
+ {"max-context", required_argument, 0, 'b'},
+ {"max-recursion", required_argument, 0, 'r'},
{"time-cutoff", required_argument, 0, 't'},
{"profile", no_argument, 0, 'p'},
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT "nxlt:p",
- long_options, &option_index);
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT HFST_GETOPT_UNARY_SHORT "nxlcdmq:r:t:p",
+ long_options, &option_index);
if (-1 == c)
{
break;
@@ -191,14 +213,40 @@ int parse_options(int argc, char** argv)
switch (c)
{
#include "inc/getopt-cases-common.h"
+#include "inc/getopt-cases-unary.h"
case 'n':
blankline_separated = false;
break;
case 'x':
- extract_tags = true;
+ extract_patterns = on;
break;
case 'l':
- locate_mode = true;
+ locate_mode = on;
+ break;
+ case 'c':
+ count_patterns = on;
+ break;
+ case 'z':
+ delete_patterns = on;
+ break;
+ case 'm':
+ mark_patterns = off;
+ break;
+ case 'b':
+ max_context = atoi(optarg);
+ if (max_context < 0)
+ {
+ std::cerr << "Invalid argument for --max-context\n";
+ return EXIT_FAILURE;
+ }
+ break;
+ case 'r':
+ max_recursion = atoi(optarg);
+ if (max_recursion < 0)
+ {
+ std::cerr << "Invalid argument for --max-recursion\n";
+ return EXIT_FAILURE;
+ }
break;
case 't':
time_cutoff = atof(optarg);
@@ -213,17 +261,10 @@ int parse_options(int argc, char** argv)
break;
#include "inc/getopt-cases-error.h"
}
-
- }
-
-// if (!inputNamed)
-// {
-// inputfile = stdin;
-// inputfilename = hfst_strdup("<stdin>");
-// }
+ }
// no more options, we should now be at the input filename
if ( (optind + 1) < argc)
{
@@ -232,13 +273,27 @@ int parse_options(int argc, char** argv)
}
else if ( (optind + 1) == argc)
{
- pmatch_filename = argv[(optind)];
+ if (inputfilename != NULL) {
+ std::cerr << "More than one input file given\n";
+ return EXIT_FAILURE;
+ } else {
+ inputfilename = hfst_strdup(argv[optind]);
+ inputfile = hfst_fopen(inputfilename, "r");
+ if (inputfile == stdin)
+ {
+ free(inputfilename);
+ inputfilename = hfst_strdup("<stdin>");
+ }
+ }
return EXIT_CONTINUE;
- }
- else
+ } else
{
- std::cerr << "No input file given\n";
- return EXIT_FAILURE;
+ if (inputfilename == NULL) {
+ std::cerr << "No input file given\n";
+ return EXIT_FAILURE;
+ } else {
+ return EXIT_CONTINUE;
+ }
}
@@ -257,23 +312,36 @@ int main(int argc, char ** argv)
if (retval != EXIT_CONTINUE) {
return retval;
}
- std::ifstream instream(pmatch_filename.c_str(),
+ std::ifstream instream(inputfilename,
std::ifstream::binary);
if (!instream.good()) {
- std::cerr << "Could not open file " << pmatch_filename << std::endl;
+ std::cerr << "Could not open file " << inputfilename << std::endl;
return EXIT_FAILURE;
}
try {
hfst_ol::PmatchContainer container(instream);
container.set_verbose(verbose);
- container.set_extract_tags_mode(extract_tags);
+ if (extract_patterns != not_defined)
+ container.set_extract_patterns(extract_patterns == on);
+ if (locate_mode != not_defined)
+ container.set_locate_mode(locate_mode == on);
+ if (count_patterns != not_defined)
+ container.set_count_patterns(count_patterns == on);
+ if (delete_patterns != not_defined)
+ container.set_delete_patterns(delete_patterns == on);
+ if (mark_patterns != not_defined)
+ container.set_mark_patterns(mark_patterns == on);
+ if (max_context >= 0)
+ container.set_max_context(max_context);
+ if (max_recursion >= 0)
+ container.set_max_recursion(max_recursion);
container.set_profile(profile);
#ifdef _MSC_VER
//hfst::print_output_to_console(true);
#endif
return process_input(container, std::cout);
} catch(HfstException & e) {
- std::cerr << "The archive in " << pmatch_filename << " doesn't look right."
+ std::cerr << "The archive in " << inputfilename << " doesn't look right."
"\nDid you make it with hfst-pmatch2fst or make sure it's in weighted optimized-lookup format?\n";
return 1;
}
diff --git a/tools/src/hfst-pmatch2fst.cc b/tools/src/hfst-pmatch2fst.cc
index 07c8ac6..089039a 100644
--- a/tools/src/hfst-pmatch2fst.cc
+++ b/tools/src/hfst-pmatch2fst.cc
@@ -25,6 +25,9 @@
#include <io.h>
#endif
+#ifndef WINDOWS
+#include <unistd.h>
+#endif
#include <iostream>
#include <fstream>
@@ -55,6 +58,7 @@ using std::pair;
#include "HfstTransducer.h"
#include "HfstInputStream.h"
#include "HfstOutputStream.h"
+#include "HfstExceptionDefs.h"
#include "implementations/ConvertTransducerFormat.h"
#include "parsers/PmatchCompiler.h"
#include "hfst-commandline.h"
@@ -91,15 +95,15 @@ print_usage()
// c.f. http://www.gnu.org/prep/standards/standards.html#g_t_002d_002dhelp
fprintf(message_out, "Usage: %s [OPTIONS...] [INFILE]\n"
"Compile regular expressions into transducer(s)\n (Experimental version)"
- "\n", program_name);
+ "\n", program_name);
print_common_program_options(message_out);
- print_common_unary_program_options(message_out);
+ print_common_unary_program_options(message_out);
fprintf(message_out, "String and format options:\n"
" -e, --epsilon=EPS Map EPS as zero\n"
" --flatten Compile in all RTNs\n");
fprintf(message_out, "\n");
- fprintf(message_out,
+ fprintf(message_out,
"If OUTFILE or INFILE is missing or -, standard streams will be used.\n"
"If EPS is not defined, the default representation of 0 is used\n"
"Weights are currently not implemented.\n"
@@ -109,7 +113,7 @@ print_usage()
fprintf(message_out, "Examples:\n"
" echo \"Define TOP UppercaseAlpha Alpha* LC({professor}) EndTag(ProfName);\" | %s \n"
" create matcher that tags \"professor Chomsky\" as \"professor <ProfName>Chomsky</ProfName>\"\n"
- "\n", program_name, program_name);
+ "\n", program_name);
print_report_bugs();
fprintf(message_out, "\n");
print_more_info();
@@ -132,7 +136,7 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "e:1:",
long_options, &option_index);
if (-1 == c)
@@ -159,6 +163,31 @@ parse_options(int argc, char** argv)
return EXIT_CONTINUE;
}
+#ifndef _GNU_SOURCE
+char * get_current_dir_name()
+{
+ size_t PATH_BUFSIZE = 1024;
+ size_t PATH_BUFSIZE_MAX = PATH_BUFSIZE * 8; // sanity
+ char * retval = (char *) NULL;
+ while(retval == NULL) {
+ retval = (char *) realloc(retval, PATH_BUFSIZE);
+ if(getcwd(retval, PATH_BUFSIZE) != 0) {
+ return retval;
+ }
+ if(errno == ERANGE && PATH_BUFSIZE < PATH_BUFSIZE_MAX) {
+ PATH_BUFSIZE *= 2;
+ continue;
+ }
+ if (errno == EACCES) {
+ throw std::runtime_error("Unable to access working directory");
+ }
+ // something else went wrong, just forget about it and try to go on
+ retval[0] = '\0';
+ return retval;
+ }
+}
+#endif
+
int
process_stream(HfstOutputStream& outstream)
{
@@ -177,11 +206,43 @@ process_stream(HfstOutputStream& outstream)
}
#endif
+ std::string includedir = "";
+#ifndef _MSC_VER
+ std::string inputfilename_str(inputfilename);
+ if (inputfile != stdin && inputfilename_str.size() > 0) {
+ if (inputfilename_str[0] == '/') {
+ // absolute path
+ includedir = inputfilename_str;
+ } else {
+ char * pwd = get_current_dir_name();
+ std::string tmp(pwd);
+ includedir = tmp + "/" + inputfilename_str;
+ free(pwd);
+ }
+ size_t slashpos = includedir.rfind('/');
+ if (slashpos == std::string::npos) {
+ // mysterious, we'll just use the working dir
+ includedir = "";
+ } else {
+ includedir = includedir.substr(0, slashpos + 1);
+ }
+ }
+#endif
+ comp.set_include_path(includedir);
+
while ((c = fgetc(inputfile)) != EOF) {
file_contents.push_back(c);
}
if (file_contents.size() > 1) {
- definitions = comp.compile(file_contents);
+ try
+ {
+ definitions = comp.compile(file_contents);
+ }
+ catch(HfstException & e)
+ {
+ std::cerr << e.name << std::endl;
+ return EXIT_FAILURE;
+ }
}
if (verbose) {
@@ -230,6 +291,7 @@ process_stream(HfstOutputStream& outstream)
// When done compiling everything, look for TOP and output it first.
if (definitions.count("TOP") == 1) {
+ std::map<std::string, std::string> properties = definitions["TOP"]->get_properties();
intermediate_tmp = hfst::implementations::ConversionFunctions::
hfst_transducer_to_hfst_basic_transducer(*definitions["TOP"]);
harmonized_tmp = hfst::implementations::ConversionFunctions::
@@ -240,6 +302,10 @@ process_stream(HfstOutputStream& outstream)
output_tmp = hfst::implementations::ConversionFunctions::
hfst_ol_to_hfst_transducer(harmonized_tmp);
output_tmp->set_name("TOP");
+ for(std::map<std::string, std::string>::iterator it = properties.begin();
+ it != properties.end(); ++it) {
+ output_tmp->set_property(it->first, it->second);
+ }
outstream << *output_tmp;
delete definitions["TOP"];
definitions.erase("TOP");
@@ -283,12 +349,13 @@ process_stream(HfstOutputStream& outstream)
std::cerr << program_name << ": Empty ruleset, nothing to write\n";
return EXIT_FAILURE;
}
+ outstream.close();
return EXIT_SUCCESS;
}
extern int pmatchdebug;
-int main( int argc, char **argv )
+int main( int argc, char **argv )
{
#ifdef WINDOWS
_setmode(1, _O_BINARY);
@@ -307,7 +374,7 @@ int main( int argc, char **argv )
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstOutputStream* outstream = (outfile != stdout) ?
diff --git a/tools/src/hfst-preprocess-for-optimized-lookup-format.cc b/tools/src/hfst-preprocess-for-optimized-lookup-format.cc
index b277a0e..5212c77 100644
--- a/tools/src/hfst-preprocess-for-optimized-lookup-format.cc
+++ b/tools/src/hfst-preprocess-for-optimized-lookup-format.cc
@@ -82,12 +82,12 @@ parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
- // add tool-specific options here
+ // add tool-specific options here
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT,
long_options, &option_index);
if (-1 == c)
@@ -122,20 +122,20 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
char* inputname = hfst_get_name(trans, inputfilename);
if (transducer_n==1)
{
- verbose_printf("Removing epsilons %s...\n", inputname);
+ verbose_printf("Removing epsilons %s...\n", inputname);
}
else
{
- verbose_printf("Removing epsilons %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
+ verbose_printf("Removing epsilons %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
}
trans.remove_epsilons();
if (transducer_n==1)
{
- verbose_printf("Rebuilding and fixing %s...\n", inputname);
+ verbose_printf("Rebuilding and fixing %s...\n", inputname);
}
else
{
- verbose_printf("Rebuilding and fisting %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
+ verbose_printf("Rebuilding and fisting %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
}
HfstBasicTransducer original(trans);
HfstBasicTransducer replication;
@@ -195,7 +195,7 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
int main( int argc, char **argv ) {
- hfst_set_program_name(argv[0], "0.1",
+ hfst_set_program_name(argv[0], "0.1",
"HfstPreprocessForOptimizedLookupFormat");
int retval = parse_options(argc, argv);
if (retval != EXIT_CONTINUE)
@@ -211,7 +211,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-proc/Makefile.am b/tools/src/hfst-proc/Makefile.am
index e57e32b..804f4ad 100644
--- a/tools/src/hfst-proc/Makefile.am
+++ b/tools/src/hfst-proc/Makefile.am
@@ -17,15 +17,6 @@
AM_CPPFLAGS = -I${top_srcdir}/libhfst/src -I${top_srcdir}/lib -I${top_builddir}/lib $(GLIB_CPPFLAGS)
-AM_CPPFLAGS+= -I${top_srcdir}/back-ends/foma \
- -I${top_srcdir}/back-ends
-
-if WANT_MINGW
- AM_CPPFLAGS+= -I${top_srcdir}/back-ends/openfstwin/src/include -DWINDOWS
-else
- AM_CPPFLAGS+= -I${top_srcdir}/back-ends/openfst/src/include
-endif
-
if WANT_PROC
MAYBE_PROC=hfst-apertium-proc
endif
diff --git a/tools/src/hfst-proc/alphabet.cc b/tools/src/hfst-proc/alphabet.cc
index 14dca41..fc726cc 100644
--- a/tools/src/hfst-proc/alphabet.cc
+++ b/tools/src/hfst-proc/alphabet.cc
@@ -152,7 +152,7 @@ Symbolizer::extract_symbol(std::istream& is) const
//////////Function definitions for ProcTransducerAlphabet
-ProcTransducerAlphabet::ProcTransducerAlphabet(std::istream& is,
+ProcTransducerAlphabet::ProcTransducerAlphabet(std::istream& is,
SymbolNumber symbol_count):
TransducerAlphabet(is, symbol_count), symbol_properties_table(), symbolizer(), blank_symbol(NO_SYMBOL_NUMBER)
{
@@ -312,7 +312,7 @@ ProcTransducerAlphabet::calculate_caps()
else
case_res = 0;
- if(case_res < 0)
+ if(case_res < 0)
{
symbol_properties_table[i].lower = i;
symbol_properties_table[i].upper = (switched=="")?NO_SYMBOL_NUMBER:symbolizer.find_symbol(switched.c_str());
@@ -345,7 +345,7 @@ ProcTransducerAlphabet::calculate_caps()
}
add_symbol(switched, prop);
if(printDebuggingInformationFlag)
- std::cout << "Added new symbol '" << switched << "' (" << symbol_table.size()-1 << ") as alternate case for '"
+ std::cout << "Added new symbol '" << switched << "' (" << symbol_table.size()-1 << ") as alternate case for '"
<< symbol_table[i] << "' (" << i << ")" << std::endl;
}
else
@@ -357,7 +357,7 @@ ProcTransducerAlphabet::calculate_caps()
if(printDebuggingInformationFlag &&
- symbol_properties_table[i].lower != NO_SYMBOL_NUMBER && symbol_properties_table[i].upper != NO_SYMBOL_NUMBER &&
+ symbol_properties_table[i].lower != NO_SYMBOL_NUMBER && symbol_properties_table[i].upper != NO_SYMBOL_NUMBER &&
symbol_to_string(symbol_properties_table[i].lower).length() != symbol_to_string(symbol_properties_table[i].upper).length())
{
std::cout << "Symbol " << i << "'s alternate case has a different string length" << std::endl;
@@ -457,18 +457,18 @@ ProcTransducerAlphabet::caps_helper_single(const char* c, int& case_res)
{"Ԁ","ԥ"}, // Cyrillic Supplement
{"Ḁ","ỿ"}};//Latin Extended Additional
- for(int i = 0; i < 21; i++)
+ for(int i = 0; i < 21; i++)
{
- if(strcmp(c,override_upper[i][0]) == 0)
+ if(strcmp(c,override_upper[i][0]) == 0)
{
case_res = 1;
return override_upper[i][1];
}
}
- for(int i = 0; i < 21; i++)
+ for(int i = 0; i < 21; i++)
{
- if(strcmp(c,override_lower[i][0]) == 0)
+ if(strcmp(c,override_lower[i][0]) == 0)
{
case_res = -1;
return override_lower[i][1];
@@ -605,7 +605,7 @@ ProcTransducerAlphabet::is_punctuation(const char* c) const
const char* individual_chars = "×÷";
for(int i=0;i<8;i++)
{
- if(strcmp(c,punct_ranges[i][0]) >= 0 &&
+ if(strcmp(c,punct_ranges[i][0]) >= 0 &&
strcmp(c,punct_ranges[i][1]) <= 0)
{
// a hack to filter out symbols (e.g. tags) that may start with punctuation
@@ -650,7 +650,7 @@ ProcTransducerAlphabet::is_compound_boundary(SymbolNumber symbol) const
{
extern bool processCompounds ;
std::string s = symbol_to_string(symbol);
- if(!processCompounds)
+ if(!processCompounds)
return false;
if(s == "+" || s[s.length()-1] == '+' ||
diff --git a/tools/src/hfst-proc/alphabet.h b/tools/src/hfst-proc/alphabet.h
index 75228d0..6527e48 100644
--- a/tools/src/hfst-proc/alphabet.h
+++ b/tools/src/hfst-proc/alphabet.h
@@ -22,7 +22,7 @@ class LetterTrie;
typedef std::vector<LetterTrie*> LetterTrieVector;
class Symbolizer;
-extern bool processCompounds ;
+extern bool processCompounds ;
class LetterTrie
{
@@ -31,7 +31,7 @@ class LetterTrie
SymbolNumberVector symbols;
/**
- * Whether our symbols vector or that of any of our children contains
+ * Whether our symbols vector or that of any of our children contains
* symbol number 0
*/
bool has_symbol_0() const;
@@ -77,7 +77,7 @@ class Symbolizer
SymbolNumber symbol_count;
public:
- Symbolizer(): letters(),
+ Symbolizer(): letters(),
ascii_symbols(std::numeric_limits<unsigned char>::max(),NO_SYMBOL_NUMBER),
symbol_count(0) {}
Symbolizer(const SymbolTable& st):
@@ -138,7 +138,7 @@ class ProcTransducerAlphabet : public TransducerAlphabet
/**
* The symbol number for a "blank" which is here considered to be a space.
- * This symbol requires special handling because it doubles as the
+ * This symbol requires special handling because it doubles as the
* symbolic representation for a superblank block of text
*/
SymbolNumber blank_symbol;
@@ -176,9 +176,9 @@ class ProcTransducerAlphabet : public TransducerAlphabet
void print_table() const;
- void add_symbol(const std::string& str, const SymbolProperties& symbol);
+ void add_symbol(const std::string& str, const SymbolProperties& symbol);
public:
- ProcTransducerAlphabet(std::istream& is, SymbolNumber symbol_count);
+ ProcTransducerAlphabet(std::istream& is, SymbolNumber symbol_count);
const Symbolizer& get_symbolizer(void) const;
SymbolNumber get_blank_symbol() const {return blank_symbol;}
@@ -187,9 +187,9 @@ class ProcTransducerAlphabet : public TransducerAlphabet
bool is_space(const char* c) const;
bool is_alphabetic(const char* c) const
- { return (c[0]!='\0' && !is_space(c) &&
+ { return (c[0]!='\0' && !is_space(c) &&
!is_punctuation(c)); }
- bool is_alphabetic(SymbolNumber symbol) const
+ bool is_alphabetic(SymbolNumber symbol) const
{return symbol_properties_table[symbol].alphabetic;}
bool is_lower(SymbolNumber symbol) const
@@ -204,7 +204,7 @@ class ProcTransducerAlphabet : public TransducerAlphabet
* lowercase equivalent
*/
SymbolNumber to_lower(SymbolNumber symbol) const
- {return symbol_properties_table[symbol].lower==NO_SYMBOL_NUMBER ?
+ {return symbol_properties_table[symbol].lower==NO_SYMBOL_NUMBER ?
symbol : symbol_properties_table[symbol].lower;}
/**
@@ -212,12 +212,12 @@ class ProcTransducerAlphabet : public TransducerAlphabet
* uppercase equivalent
*/
SymbolNumber to_upper(SymbolNumber symbol) const
- {return symbol_properties_table[symbol].upper==NO_SYMBOL_NUMBER ?
+ {return symbol_properties_table[symbol].upper==NO_SYMBOL_NUMBER ?
symbol : symbol_properties_table[symbol].upper;}
/**
- * Whether the symbol is an apertium-style tag (i.e. symbols starting
- * with < and ending with > )
+ * Whether the symbol is an apertium-style tag (i.e. symbols starting
+ * with < and ending with > )
*/
bool is_tag(SymbolNumber symbol) const;
diff --git a/tools/src/hfst-proc/applicators.cc b/tools/src/hfst-proc/applicators.cc
index 48708f5..1bc194f 100644
--- a/tools/src/hfst-proc/applicators.cc
+++ b/tools/src/hfst-proc/applicators.cc
@@ -13,6 +13,7 @@
#include <sstream>
#include "applicators.h"
#include "lookup-state.h"
+#include "lookup-path.h"
#include "formatter.h"
//////////Function definitions for TokenizationApplicator
@@ -31,8 +32,8 @@ TokenizationApplicator::process_token(const Token& t) const
s << "Reserved char: '" << t.character << "'";
else if(t.type == None)
s << "None/EOF";
-
- s << " (to_symbol: " << token_stream.to_symbol(t) << ")"
+
+ s << " (to_symbol: " << token_stream.to_symbol(t) << ")"
<< " (alphabetic: " << token_stream.is_alphabetic(t) << ")";
if(t.type == Symbol)
{
@@ -84,7 +85,7 @@ AnalysisApplicator::apply()
size_t last_stream_location = 0;
TokenVector surface_form;
ProcResult analyzed_forms;
-
+
Token next_token;
while((next_token = token_stream.get_token()).type != None)
{
@@ -101,13 +102,13 @@ AnalysisApplicator::apply()
}
if(next_token.type == ReservedCharacter)
stream_error(std::string("Found unexpected character ")+next_token.character+" unescaped in stream");
-
+
if(surface_form.size() > 0 && state.is_final())
{
LookupPathSet finals = state.get_finals_set();
if (caps_mode == DictionaryCase || caps_mode == CaseSensitiveDictionaryCase)
{
- analyzed_forms = formatter.process_finals(finals,
+ analyzed_forms = formatter.process_finals(finals,
Unknown);
}
else
@@ -116,16 +117,16 @@ AnalysisApplicator::apply()
token_stream.get_capitalization_state(surface_form));
}
last_stream_location = token_stream.get_pos()-1;
-
+
if(printDebuggingInformationFlag)
std::cout << "Final paths (" << finals.size() << ") found and saved, stream location is " << last_stream_location << std::endl;
}
-
+
state.step(token_stream.to_symbol(next_token), caps_mode);
-
+
if(printDebuggingInformationFlag)
std::cout << "After stepping, there are " << state.num_active() << " active paths" << std::endl;
-
+
if(state.is_active())
{
surface_form.push_back(next_token);
@@ -137,7 +138,7 @@ AnalysisApplicator::apply()
if(formatter.preserve_nonalphabetic())
token_stream << next_token;
}
- else if(analyzed_forms.size() == 0 ||
+ else if(analyzed_forms.size() == 0 ||
(token_stream.is_alphabetic(token_stream.at(last_stream_location)) &&
token_stream.is_alphabetic(surface_form[surface_form.size()-1])))
{
@@ -155,7 +156,7 @@ AnalysisApplicator::apply()
// we overstepped the word by one token
token_stream.move_back(1);
}
-
+
if(!token_stream.is_alphabetic(surface_form[0]))
{
if(formatter.preserve_nonalphabetic())
@@ -167,14 +168,14 @@ AnalysisApplicator::apply()
//size_t word_length = token_stream.first_nonalphabetic(surface_form);
//if(word_length == string::npos)
size_t word_length = surface_form.size();
-
- int revert_count = surface_form.size()-word_length+
- next_token_is_part_of_word ? 0 : 1;
-
+
+ int revert_count = (surface_form.size()-word_length+
+ next_token_is_part_of_word) ? 0 : 1;
+
if(printDebuggingInformationFlag)
- std::cout << "word_length=" << word_length << ", surface_form.size()=" << surface_form.size()
+ std::cout << "word_length=" << word_length << ", surface_form.size()=" << surface_form.size()
<< ", moving back " << revert_count << " characters" << std::endl;
-
+
formatter.print_unknown_word(TokenVector(surface_form.begin(),
surface_form.begin()+word_length));
token_stream.move_back(revert_count);
@@ -185,21 +186,21 @@ AnalysisApplicator::apply()
// the number of symbols on the end of surface_form that aren't a part
// of the transduction(s) found
int revert_count = token_stream.diff_prev(last_stream_location+1);
- formatter.print_word(TokenVector(surface_form.begin(),
+ formatter.print_word(TokenVector(surface_form.begin(),
surface_form.end()-revert_count),
- analyzed_forms);
+ analyzed_forms);
token_stream.move_back(revert_count+1);
}
-
+
state.reset();
surface_form.clear();
analyzed_forms.clear();
}
}
-
+
if(verboseFlag)
std::cout << std::endl << "Got None/EOF symbol; done." << std::endl;
-
+
// print any valid transductions stored
if(analyzed_forms.size() != 0)// && token_stream.get_pos() == last_stream_location)
formatter.print_word(surface_form, analyzed_forms);
@@ -237,16 +238,16 @@ GenerationApplicator::apply()
else
form.push_back(next_token);
}
-
+
//Figure out how to output the word
-
+
if(prefix_char == '*' || prefix_char == '@') // the word is unanalyzed (*) or untranslated (@)
{
if(mode != gm_clean)
token_stream.write_escaped(std::string(1,prefix_char));
-
+
std::string word = token_stream.read_delimited('$');
-
+
if(prefix_char == '*' || (prefix_char == '@' && mode != gm_all))
{
size_t loc = word.find('<');
@@ -262,7 +263,7 @@ GenerationApplicator::apply()
else
{
// first try unsplit
-
+
if(!lookup(form, false))
{
std::vector<TokenVector> parts = split(form);
@@ -272,7 +273,7 @@ GenerationApplicator::apply()
lookup(*it, true);
}
}
-
+
if(prefix_char == '#') // if there is an invariant part remaining
{
std::string invariant = token_stream.read_delimited('$');
@@ -307,10 +308,58 @@ GenerationApplicator::split(const TokenVector& tokens) const
}
if(start != tokens.end())
res.push_back(TokenVector(start, tokens.end()));
-
+
return res;
}
+
+LookupPathSet
+GenerationApplicator::preprocess_finals(const LookupPathSet& finals) const
+{
+ LookupPathSet goodcmp_finals(LookupPathW::compare_weights);
+ // insertion sort :)
+ for (LookupPathSet::const_iterator it = finals.begin(); it != finals.end(); ++it)
+ {
+ goodcmp_finals.insert(*it);
+ }
+
+ // Keep only the N best weight classes
+ int classes_found = -1;
+ Weight last_weight_class = 0.0;
+ LookupPathSet goodweight_finals(LookupPathW::compare_weights);
+ for(LookupPathSet::const_iterator it = goodcmp_finals.begin(); it != goodcmp_finals.end(); it++)
+ {
+ LookupPathW* pw = dynamic_cast<LookupPathW*>(*it);
+ if(pw != NULL) {
+ Weight current_weight = pw->get_weight();
+ if (classes_found == -1) // we're just starting
+ {
+ classes_found = 1;
+ last_weight_class = current_weight;
+ }
+ else if (last_weight_class != current_weight) // we might want to ignore the rest due to weight classes
+ {
+ last_weight_class = current_weight;
+ ++classes_found;
+ }
+ if (classes_found > maxWeightClasses)
+ {
+ break;
+ }
+ }
+ goodweight_finals.insert(*it);
+ }
+ // Keep no more than maxAnalyses
+ LookupPathSet clipped_finals(LookupPathW::compare_weights);
+ LookupPathSet::const_iterator it = goodweight_finals.begin();
+ for(int i=0; i < maxAnalyses && it != goodweight_finals.end(); i++, it++)
+ {
+ clipped_finals.insert(*it);
+ }
+ return clipped_finals;
+}
+
+
bool
GenerationApplicator::lookup(const TokenVector& tokens, bool generate_on_fail)
{
@@ -318,24 +367,31 @@ GenerationApplicator::lookup(const TokenVector& tokens, bool generate_on_fail)
state.lookup(token_stream.to_symbols(tokens), caps_mode);
if(state.is_final()) // generation succeeded
{
- CapitalizationState capitalization_state =
- caps_mode==DictionaryCase ?
- Unknown :
- token_stream.get_capitalization_state(TokenVector(tokens.begin(),tokens.size()>1?tokens.begin()+2:tokens.end()));
LookupPathSet finals = state.get_finals_set();
-
+ CapitalizationState caps_change = Unknown;
+ if(caps_mode!=DictionaryCase) {
+ CapitalizationState lm_caps_state = token_stream.get_capitalization_state((*finals.begin())->get_output_symbols());
+ CapitalizationState sf_caps_state = token_stream.get_capitalization_state(TokenVector(tokens.begin(),tokens.size()>1?tokens.begin()+2:tokens.end()));
+ if(sf_caps_state != lm_caps_state) {
+ // We *only* change capitalization if lm and sf caps state differs
+ caps_change = sf_caps_state;
+ }
+ }
+
if(printDebuggingInformationFlag)
std::cout << "Generated " << finals.size() << " forms" << std::endl;
-
- token_stream.put_symbols((*finals.begin())->get_output_symbols(),capitalization_state);
- if(finals.size() > 1)
+
+ LookupPathSet new_finals = preprocess_finals(finals);
+
+ token_stream.put_symbols((*new_finals.begin())->get_output_symbols(),caps_change);
+ if(new_finals.size() > 1)
{
- LookupPathSet::const_iterator it=finals.begin();
+ LookupPathSet::const_iterator it=new_finals.begin();
it++; // start from begin+1
- for(;it!=finals.end();it++)
+ for(;it!=new_finals.end();it++)
{
token_stream.ostream() << '/';
- token_stream.put_symbols((*it)->get_output_symbols(),capitalization_state);
+ token_stream.put_symbols((*it)->get_output_symbols(),caps_change);
}
}
return true;
@@ -366,4 +422,3 @@ GenerationApplicator::lookup(const TokenVector& tokens, bool generate_on_fail)
}
return false;
}
-
diff --git a/tools/src/hfst-proc/applicators.h b/tools/src/hfst-proc/applicators.h
index 3e83f4a..9f97f7f 100644
--- a/tools/src/hfst-proc/applicators.h
+++ b/tools/src/hfst-proc/applicators.h
@@ -13,6 +13,7 @@
#ifndef _HFST_PROC_APPLICATORS_H_
#define _HFST_PROC_APPLICATORS_H_
+#include "lookup-path.h"
#include "tokenizer.h"
#include "transducer.h"
@@ -27,7 +28,7 @@ class Applicator
public:
Applicator(const ProcTransducer& t, TokenIOStream& ts): transducer(t), token_stream(ts) {}
virtual ~Applicator() {}
-
+
virtual void apply() = 0;
};
@@ -60,15 +61,18 @@ class GenerationApplicator: public Applicator
private:
GenerationMode mode;
CapitalizationMode caps_mode;
-
+
/**
* Split the given token string into a set of token strings to generate with,
* breaking on + after apertium-style tags.
*/
std::vector<TokenVector> split(const TokenVector& tokens) const;
-
+
bool lookup(const TokenVector& tokens, bool generate_on_fail);
+ LookupPathSet preprocess_finals(const LookupPathSet& finals) const ;
+
+
public:
GenerationApplicator(const ProcTransducer& t, TokenIOStream& ts,
GenerationMode m, CapitalizationMode c):
diff --git a/tools/src/hfst-proc/buffer.h b/tools/src/hfst-proc/buffer.h
index f65dcb4..928bc92 100644
--- a/tools/src/hfst-proc/buffer.h
+++ b/tools/src/hfst-proc/buffer.h
@@ -65,20 +65,20 @@ private:
delete[] buf;
}
-public:
+public:
/**
* Constructor
* @param buf_size buffer size
*/
- Buffer(size_t const buf_size=2048):
+ Buffer(size_t const buf_size=2048):
size(buf_size), buf(NULL), currentpos(0), lastpos(0)
{
if(buf_size == 0)
{
cerr << "Error: Cannot create empty buffer." << endl;
exit(EXIT_FAILURE);
- }
+ }
buf = new T[buf_size];
}
@@ -224,7 +224,7 @@ public:
{
return currentpos + size - prevpos;
}
- }
+ }
/**
* Return the range size between the buffer current position and a
@@ -250,7 +250,7 @@ public:
*/
bool isEmpty() const
{
- return currentpos == lastpos;
+ return currentpos == lastpos;
}
/**
diff --git a/tools/src/hfst-proc/formatter.cc b/tools/src/hfst-proc/formatter.cc
index 0c9e08f..55080fb 100644
--- a/tools/src/hfst-proc/formatter.cc
+++ b/tools/src/hfst-proc/formatter.cc
@@ -118,6 +118,76 @@ OutputFormatter::preprocess_finals(const LookupPathSet& finals) const
return clipped_finals;
}
+//////////Function definitions for TransliterateOutputFormatter
+
+ProcResult
+TransliterateOutputFormatter::process_finals(const LookupPathSet& finals, CapitalizationState caps) const
+{
+ ProcResult results;
+ LookupPathSet new_finals = preprocess_finals(finals);
+
+ for(LookupPathSet::const_iterator it=new_finals.begin(); it!=new_finals.end(); it++)
+ {
+ std::ostringstream res;
+ res << token_stream.get_alphabet().symbols_to_string((*it)->get_output_symbols(), caps);
+ if(dynamic_cast<const LookupPathW*>(*it) != NULL && displayWeightsFlag)
+ res << '~' << dynamic_cast<const LookupPathW*>(*it)->get_weight() << '~';
+
+ results.push_back(res.str());
+ }
+ return results;
+}
+
+void
+TransliterateOutputFormatter::print_word(const TokenVector& surface_form,
+ ProcResult const &analyzed_forms) const
+{
+ // any superblanks in the surface form should not be printed as part of the
+ // analysis output, but should be output directly afterwards
+ TokenVector output_surface_form;
+ std::vector<unsigned int> superblanks;
+ for(TokenVector::const_iterator it=surface_form.begin(); it!=surface_form.end(); it++)
+ {
+ if(it->type == Superblank)
+ {
+ output_surface_form.push_back(Token::as_symbol(token_stream.to_symbol(*it)));
+ superblanks.push_back(it->superblank_index);
+ }
+ else
+ {
+ output_surface_form.push_back(*it);
+ }
+ }
+
+ if(printDebuggingInformationFlag)
+ {
+ std::cout << "surface_form consists of " << output_surface_form.size() << " tokens" << std::endl;
+ }
+
+ bool first = true;
+ for(ProcResult::const_iterator it=analyzed_forms.begin(); it!=analyzed_forms.end(); it++)
+ {
+ if(!first)
+ {
+ token_stream.ostream() << "/" ;
+ }
+ token_stream.ostream() << *it;
+ first = false;
+ }
+
+ for(size_t i=0;i<superblanks.size();i++)
+ {
+ token_stream.ostream() << token_stream.get_superblank(superblanks[i]);
+ }
+}
+void
+TransliterateOutputFormatter::print_unknown_word(const TokenVector& surface_form) const
+{
+ token_stream.write_escaped(surface_form);
+}
+
+
+
//////////Function definitions for ApertiumOutputFormatter
ProcResult
@@ -138,6 +208,8 @@ ApertiumOutputFormatter::process_finals(const LookupPathSet& finals, Capitalizat
return results;
}
+////////
+
void
ApertiumOutputFormatter::print_word(const TokenVector& surface_form,
ProcResult const &analyzed_forms) const
diff --git a/tools/src/hfst-proc/formatter.h b/tools/src/hfst-proc/formatter.h
index d696867..c259f15 100644
--- a/tools/src/hfst-proc/formatter.h
+++ b/tools/src/hfst-proc/formatter.h
@@ -57,7 +57,7 @@ class OutputFormatter
*/
virtual ProcResult process_finals(const LookupPathSet& finals,
CapitalizationState state) const = 0;
- virtual void print_word(const TokenVector& surface_form,
+ virtual void print_word(const TokenVector& surface_form,
ProcResult const &analyzed_forms) const = 0;
virtual void print_unknown_word(const TokenVector& surface_form) const = 0;
@@ -68,6 +68,20 @@ class OutputFormatter
virtual bool preserve_nonalphabetic() const = 0;
};
+class TransliterateOutputFormatter: public OutputFormatter
+{
+ public:
+ TransliterateOutputFormatter(TokenIOStream& s, bool f): OutputFormatter(s,f) {}
+
+ ProcResult process_finals(const LookupPathSet& finals,
+ CapitalizationState state) const;
+ void print_word(const TokenVector& surface_form,
+ ProcResult const &analyzed_forms) const;
+ void print_unknown_word(const TokenVector& surface_form) const;
+
+ bool preserve_nonalphabetic() const {return true;}
+};
+
class ApertiumOutputFormatter: public OutputFormatter
{
public:
@@ -75,7 +89,7 @@ class ApertiumOutputFormatter: public OutputFormatter
ProcResult process_finals(const LookupPathSet& finals,
CapitalizationState state) const;
- void print_word(const TokenVector& surface_form,
+ void print_word(const TokenVector& surface_form,
ProcResult const &analyzed_forms) const;
void print_unknown_word(const TokenVector& surface_form) const;
@@ -90,7 +104,7 @@ class CGOutputFormatter: public OutputFormatter
ProcResult process_finals(const LookupPathSet& finals,
CapitalizationState caps) const;
- void print_word(const TokenVector& surface_form,
+ void print_word(const TokenVector& surface_form,
ProcResult const &analyzed_forms) const;
void print_unknown_word(const TokenVector& surface_form) const;
@@ -105,7 +119,7 @@ class XeroxOutputFormatter: public OutputFormatter
ProcResult process_finals(const LookupPathSet& finals,
CapitalizationState state) const;
- void print_word(const TokenVector& surface_form,
+ void print_word(const TokenVector& surface_form,
ProcResult const &analyzed_forms) const;
void print_unknown_word(const TokenVector& surface_form) const;
diff --git a/tools/src/hfst-proc/hfst-proc.cc b/tools/src/hfst-proc/hfst-proc.cc
index c0d293a..94ff304 100644
--- a/tools/src/hfst-proc/hfst-proc.cc
+++ b/tools/src/hfst-proc/hfst-proc.cc
@@ -60,7 +60,7 @@ static bool handle_hfst3_header(std::istream& is)
HFST_THROW(HfstException);
return false;
}
- char * headervalue = new char[remaining_header_len];
+ char * headervalue = (char*) malloc(remaining_header_len);
while(remaining_header_len > 0) {
is.getline(headervalue, remaining_header_len + 1, '\0');
remaining_header_len -= strlen(headervalue) + 1;
@@ -69,13 +69,13 @@ static bool handle_hfst3_header(std::istream& is)
remaining_header_len -= strlen(headervalue) + 1;
if (strcmp(headervalue, "HFST_OL") &&
strcmp(headervalue, "HFST_OLW")) {
- delete headervalue;
+ free((void*)headervalue);
HFST_THROW(TransducerHasWrongTypeException);
return false;
}
}
}
- delete headervalue;
+ free((void*)headervalue);
if (remaining_header_len == 0) {
return true;
} else {
@@ -149,7 +149,7 @@ bool print_version(void)
{
std::cout <<
"\n" <<
- "hfst-proc 0.0" <<
+ "hfst-proc 0.0" <<
#ifdef HAVE_CONFIG_H
" (" << PACKAGE_STRING << ")" <<
#endif
@@ -189,6 +189,7 @@ int main(int argc, char **argv)
{"non-marked-gen", no_argument, 0, 'n'},
{"debugged-gen", no_argument, 0, 'd'},
{"tokenize", no_argument, 0, 't'},
+ {"transliterate", no_argument, 0, 'j'},
{"apertium", no_argument, 0, 'p'},
{"xerox", no_argument, 0, 'x'},
{"cg", no_argument, 0, 'C'},
@@ -198,7 +199,7 @@ int main(int argc, char **argv)
{"show-raw-in-cg", no_argument, 0, 'r'},
{"analyses", required_argument, 0, 'N'},
// -l is probably too error prone to document
- {"weight-classes", required_argument, 0, 'l'},
+ {"weight-classes", required_argument, 0, 'l'},
{"case-sensitive", no_argument, 0, 'c'},
{"dictionary-case",no_argument, 0, 'w'},
{"null-flush", no_argument, 0, 'z'},
@@ -207,7 +208,7 @@ int main(int argc, char **argv)
};
int option_index = 0;
- int c = getopt_long(argc, argv, "hVvqsagndtpxCkeWrN:l:cwzX", long_options, &option_index);
+ int c = getopt_long(argc, argv, "hVvqjsagndtpxCkeWrN:l:cwzX", long_options, &option_index);
if (c == -1) // no more options to look at
break;
@@ -263,6 +264,7 @@ int main(int argc, char **argv)
case 'p':
case 'C':
case 'x':
+ case 'j':
if(output_type == 0)
output_type = c;
else
@@ -303,7 +305,7 @@ int main(int argc, char **argv)
}
break;
- case 'e':
+ case 'e':
processCompounds = true;
break;
case 'c':
@@ -420,7 +422,7 @@ int main(int argc, char **argv)
catch (TransducerHasWrongTypeException &ex)
{
std::cerr << "Transducer must be in HFST optimized lookup format." << std::endl;
- return EXIT_FAILURE;
+ return EXIT_FAILURE;
}
ProcTransducer t(in);
if(verboseFlag)
@@ -454,6 +456,9 @@ int main(int argc, char **argv)
case 'x':
output_formatter = (OutputFormatter*)new XeroxOutputFormatter(token_stream, filter_compound_analyses);
break;
+ case 'j':
+ output_formatter = (OutputFormatter*)new TransliterateOutputFormatter(token_stream, filter_compound_analyses);
+ break;
default:
output_formatter = (OutputFormatter*)new ApertiumOutputFormatter(token_stream, filter_compound_analyses);
}
diff --git a/tools/src/hfst-proc/lookup-path.h b/tools/src/hfst-proc/lookup-path.h
index 8cdba48..cb5e337 100644
--- a/tools/src/hfst-proc/lookup-path.h
+++ b/tools/src/hfst-proc/lookup-path.h
@@ -47,7 +47,7 @@ class LookupPath
SymbolNumberVector output_symbols;
public:
- LookupPath(const ProcTransducer& t, const TransitionTableIndex initial):
+ LookupPath(const ProcTransducer& t, const TransitionTableIndex initial):
transducer(t), index(initial), final(false), output_symbols() {}
LookupPath(const LookupPath& o):
@@ -134,7 +134,7 @@ class LookupPathW : public LookupPath
*/
Weight final_weight;
public:
- LookupPathW(const ProcTransducer& t, const TransitionTableIndex initial):
+ LookupPathW(const ProcTransducer& t, const TransitionTableIndex initial):
LookupPath(t, initial), weight(0.0f), final_weight(0.0f) {}
LookupPathW(const LookupPathW& o): LookupPath(o), weight(o.weight),
final_weight(o.final_weight) {}
diff --git a/tools/src/hfst-proc/lookup-state.cc b/tools/src/hfst-proc/lookup-state.cc
index c277c6c..b7b9d8f 100644
--- a/tools/src/hfst-proc/lookup-state.cc
+++ b/tools/src/hfst-proc/lookup-state.cc
@@ -70,7 +70,7 @@ LookupState::clear_paths()
bool
LookupState::is_final() const
{
- for(LookupPathVector::const_iterator i=paths.begin();
+ for(LookupPathVector::const_iterator i=paths.begin();
i!=paths.end(); ++i)
{
TransitionTableIndex index = (*i)->get_index();
@@ -267,8 +267,8 @@ LookupState::apply_input(const SymbolNumber input, const SymbolNumber altinput)
}
bool
-LookupState::try_index(LookupPathVector& new_paths,
- const LookupPath& path,
+LookupState::try_index(LookupPathVector& new_paths,
+ const LookupPath& path,
const SymbolNumber input) const
{
//??? is the +1 here correct?
@@ -288,7 +288,7 @@ LookupState::try_index(LookupPathVector& new_paths,
bool
LookupState::try_transitions(LookupPathVector& new_paths,
- const LookupPath& path,
+ const LookupPath& path,
const SymbolNumber input) const
{
bool found = false;
diff --git a/tools/src/hfst-proc/lookup-state.h b/tools/src/hfst-proc/lookup-state.h
index e2f21b3..c311bd9 100644
--- a/tools/src/hfst-proc/lookup-state.h
+++ b/tools/src/hfst-proc/lookup-state.h
@@ -102,7 +102,7 @@ class LookupState
* @param input the input symbol to look up in the transition index table
* @return whether the path has a continuation with the given input
*/
- bool try_index(LookupPathVector& new_paths,
+ bool try_index(LookupPathVector& new_paths,
const LookupPath& path, const SymbolNumber input) const;
/**
diff --git a/tools/src/hfst-proc/tokenizer.cc b/tools/src/hfst-proc/tokenizer.cc
index 6591d8e..4028901 100644
--- a/tools/src/hfst-proc/tokenizer.cc
+++ b/tools/src/hfst-proc/tokenizer.cc
@@ -18,7 +18,7 @@
std::set<char> TokenIOStream::escaped_chars;
-TokenIOStream::TokenIOStream(std::istream& i, std::ostream& o,
+TokenIOStream::TokenIOStream(std::istream& i, std::ostream& o,
const ProcTransducerAlphabet& a, bool flush,
bool raw):
is(i), os(o), alphabet(a), null_flush(flush), is_raw(raw),
@@ -49,20 +49,37 @@ TokenIOStream::initialize_escaped_chars()
void
TokenIOStream::do_null_flush()
{
- std::cout << '\0';
+ os << '\0';
os.flush();
if(os.bad())
std::cerr << "Could not flush file" << std::endl;
}
CapitalizationState
+TokenIOStream::get_capitalization_state(const SymbolNumberVector& symbols) const
+{
+ if(symbols.size() == 0)
+ return Unknown;
+
+ const SymbolNumber& first=symbols[0], last=symbols.size()>1?symbols[1]:symbols[0];
+
+ if(alphabet.is_lower(first) && alphabet.is_lower(last))
+ return LowerCase;
+ if(alphabet.is_upper(first) && alphabet.is_lower(last))
+ return FirstUpperCase;
+ if(alphabet.is_upper(first) && alphabet.is_upper(last))
+ return UpperCase;
+ return Unknown;
+}
+
+CapitalizationState
TokenIOStream::get_capitalization_state(const TokenVector& tokens) const
{
if(tokens.size() == 0)
return Unknown;
-
- const Token& first=tokens[0], last=tokens.size()>1?tokens[tokens.size()-1]:tokens[0];
-
+
+ const Token& first=tokens[0], last=tokens.size()>1?tokens[1]:tokens[0];
+
if(first.type != Symbol || last.type != Symbol)
return Unknown;
if(alphabet.is_lower(first.symbol) && alphabet.is_lower(last.symbol))
@@ -88,7 +105,7 @@ TokenIOStream::read_utf8_char(std::istream& is)
int c = is.peek();
if(is.eof())
return retval;
-
+
if (c <= 127)
u8len = 1;
else if ( (c & (128 + 64 + 32 + 16)) == (128 + 64 + 32 + 16) )
@@ -103,7 +120,7 @@ TokenIOStream::read_utf8_char(std::istream& is)
retval.resize(u8len+1);
is.get(&retval[0], u8len+1, '\0');
retval.resize(strlen(&retval[0]));
-
+
return retval;
}
@@ -129,7 +146,7 @@ TokenIOStream::is_alphabetic(const Token& t) const
SymbolNumber s = to_symbol(t);
if(s != 0 && s != NO_SYMBOL_NUMBER)
return alphabet.is_alphabetic(s);
-
+
switch(t.type)
{
case Character:
@@ -147,7 +164,7 @@ TokenIOStream::first_nonalphabetic(const TokenVector& s) const
if(!is_alphabetic(s[i]))
return i;
}
-
+
return string::npos;
}
@@ -155,10 +172,10 @@ int
TokenIOStream::read_escaped()
{
int c = is.get();
-
+
if(c == EOF || escaped_chars.find(c) == escaped_chars.end())
stream_error("Found non-reserved character after backslash");
-
+
return c;
}
@@ -167,13 +184,13 @@ TokenIOStream::read_delimited(const char delim)
{
std::string result;
int c = EOF;
-
+
while(is && c != delim)
{
c = is.get();
if(c == EOF)
break;
-
+
result += c;
if(c == '\\')
result += read_escaped();
@@ -292,7 +309,7 @@ TokenIOStream::get_token()
{
if(!token_buffer.isEmpty())
return token_buffer.next();
-
+
Token token = read_token();
if(token.type != None)
token_buffer.add(token);
@@ -346,4 +363,3 @@ TokenIOStream::tokens_to_string(const TokenVector& t, bool raw) const
res += token_to_string(*it,raw);
return res;
}
-
diff --git a/tools/src/hfst-proc/tokenizer.h b/tools/src/hfst-proc/tokenizer.h
index b8b5033..e88b1e3 100644
--- a/tools/src/hfst-proc/tokenizer.h
+++ b/tools/src/hfst-proc/tokenizer.h
@@ -40,9 +40,9 @@ struct Token
char character[5];
unsigned int superblank_index; // see TokenIOStream::superblank_bucket
};
-
+
Token(): type(None), symbol(0) {}
-
+
void set_none() {type=None;}
void set_symbol(SymbolNumber s) {type=Symbol; symbol=s;}
void set_character(const char* c)
@@ -50,13 +50,13 @@ struct Token
void set_character(char c) {type=Character; character[0]=c; character[1]='\0';}
void set_superblank(unsigned int i) {type=Superblank; superblank_index=i;}
void set_reservedcharacter(char c) {type=ReservedCharacter; character[0]=c; character[1]='\0';}
-
+
static Token as_symbol(SymbolNumber s) {Token t; t.set_symbol(s); return t;}
static Token as_character(const char* c) {Token t; t.set_character(c); return t;}
static Token as_character(char c) {Token t; t.set_character(c); return t;}
static Token as_superblank(unsigned int i) {Token t; t.set_superblank(i); return t;}
static Token as_reservedcharacter(char c) {Token t; t.set_reservedcharacter(c); return t;}
-
+
bool operator<(const Token& rhs) const
{
if(type != rhs.type)
@@ -92,49 +92,49 @@ class TokenIOStream
*/
static std::set<char> escaped_chars;
static void initialize_escaped_chars();
-
+
std::istream& is;
std::ostream& os;
const ProcTransducerAlphabet& alphabet;
bool null_flush;
bool is_raw;
-
+
const Symbolizer& symbolizer;
-
+
/**
* All superblanks found in the input stream all stored here, and are
* indexed by Token objects
*/
std::vector<std::string> superblank_bucket;
-
+
Buffer<Token> token_buffer;
-
+
void do_null_flush();
-
+
/**
* Reads a UTF-8 char (1-4 bytes) from the input stream, returning it as a
* character string
*/
std::string read_utf8_char();
-
+
/**
* Called after a backslash has been found in the stream to read an escaped
* character. Fails on stream error or if the next character isn't a proper
* escaped character
*/
int read_escaped();
-
+
/**
* Generate an escaped copy of the given string
*/
std::string escape(const std::string& str) const;
-
+
/**
* Make a token from the next character(s) in the stream by attempting to
* get a symbol, and reverting to reading a character if that fails
*/
Token make_token();
-
+
/**
* Read the next token in the stream, handling escaped characters
*/
@@ -142,21 +142,21 @@ class TokenIOStream
public:
TokenIOStream(std::istream& i, std::ostream& o, const ProcTransducerAlphabet& a,
bool flush, bool raw);
-
+
size_t get_pos() const {return token_buffer.getPos();}
size_t diff_prev(size_t pos) const {return token_buffer.diffPrevPos(pos);}
Token at(size_t pos) const {return token_buffer.get(pos);}
void move_back(size_t count) {token_buffer.back(count);}
-
+
const ProcTransducerAlphabet& get_alphabet() const {return alphabet;}
-
+
bool is_space(const Token& t) const;
bool is_alphabetic(const Token& t) const;
-
+
/**
* Get a symbol representation of the token. The conversion depends on the
* token type
- *
+ *
* None - NO_SYMBOL_NUMBER
* Symbol - the unmodified symbol
* Character - blank_symbol if is_space returns true for the character,
@@ -166,15 +166,20 @@ class TokenIOStream
*/
SymbolNumber to_symbol(const Token& t) const;
SymbolNumberVector to_symbols(const TokenVector& t) const;
-
+
+ /**
+ * Calculate the capitalization properties of the given analysis.
+ */
+ CapitalizationState get_capitalization_state(const SymbolNumberVector& symbols) const;
+
/**
* Calculate the capitalization properties of the given word, which should
* contain all symbols
*/
CapitalizationState get_capitalization_state(const TokenVector& tokens) const;
-
+
size_t first_nonalphabetic(const TokenVector& s) const;
-
+
/**
* Read into the the stream until the delimiting character is found. The
* delimiting character is read and included in the string. Charater escaping
@@ -183,45 +188,45 @@ class TokenIOStream
* the delimiting character
*/
std::string read_delimited(const char delim);
-
+
/**
* Read the next token from the input stream/buffer
*/
Token get_token();
-
+
/**
* Write a token to the output stream
*/
void put_token(const Token& t);
-
+
void put_tokens(const TokenVector& t);
void put_symbols(const SymbolNumberVector& s, CapitalizationState caps=Unknown);
-
+
/**
* Get the string representation of the given token
* @param raw if true, then don't do any character escaping
*/
std::string token_to_string(const Token& t, bool raw=false) const;
-
+
std::string tokens_to_string(const TokenVector& t, bool raw=false) const;
-
+
/**
* Read the next token from the input stream/buffer
*/
TokenIOStream& operator>>(Token& t) {t=get_token(); return *this;}
-
+
/**
* Write the string representation of a token to the output stream
*/
TokenIOStream& operator<<(const Token& t) {put_token(t); return *this;}
-
+
std::ostream& ostream() {return os;}
-
+
void write_escaped(const std::string str) {os << escape(str);}
void write_escaped(const TokenVector& t) {os << tokens_to_string(t);}
-
+
std::string get_superblank(size_t i) const {return superblank_bucket[i];}
-
+
/**
* Reads a UTF-8 char (1-4 bytes) an the input stream, returning it as a
* character string
diff --git a/tools/src/hfst-program-options.cc b/tools/src/hfst-program-options.cc
index ec0c537..646a979 100644
--- a/tools/src/hfst-program-options.cc
+++ b/tools/src/hfst-program-options.cc
@@ -19,8 +19,8 @@
void print_common_program_options(FILE *file) {
fprintf(file, "Common options:\n"
- " -h, --help Print help message\n"
- " -V, --version Print version info\n"
+ " -h, --help Print help message\n"
+ " -V, --version Print version info\n"
" -v, --verbose Print verbosely while processing\n"
" -q, --quiet Only print fatal erros and requested "
"output\n"
@@ -53,7 +53,7 @@ void print_common_unary_program_options(FILE *file) {
void print_common_unary_program_parameter_instructions(FILE *file) {
- fprintf(file,
+ fprintf(file,
"If OUTFILE or INFILE is missing or -, standard streams will be used.\n"
"Format of result depends on format of INFILE\n"
);
@@ -64,7 +64,7 @@ void print_common_unary_program_parameter_instructions(FILE *file) {
// compose
// concatenate
// conjunct
-// disjunct,
+// disjunct,
void print_common_binary_program_options(FILE *file) {
fprintf(file, "Input/Output options:\n"
@@ -76,7 +76,7 @@ void print_common_binary_program_options(FILE *file) {
void print_common_binary_program_parameter_instructions(FILE *file) {
- fprintf(file,
+ fprintf(file,
"If OUTFILE, or either INFILE1 or INFILE2 is missing or -,\n"
"standard streams will be used.\n"
"INFILE1, INFILE2, or both, must be specified.\n"
@@ -84,12 +84,12 @@ void print_common_binary_program_parameter_instructions(FILE *file) {
"both should have the same format.\n"
);
- fprintf(file,
+ fprintf(file,
"\n"
"The operation is applied pairwise for INFILE1 and INFILE2\n"
"that must have the same number of transducers.\n"
"If INFILE2 has only one transducer, the operation is applied for\n"
- "each transducer in INFILE1 keeping the second transducer constant.\n"
+ "each transducer in INFILE1 keeping the second transducer constant.\n"
);
}
diff --git a/tools/src/hfst-program-options.h b/tools/src/hfst-program-options.h
index 8dba329..7a7759e 100644
--- a/tools/src/hfst-program-options.h
+++ b/tools/src/hfst-program-options.h
@@ -59,7 +59,7 @@ void print_common_unary_string_program_options(FILE *file);
// " -W, --write-symbols-to=FILENAME Write symbol table to file FILENAME\n"
-// Two transducers to one transducer: compose, concatenate, conjunct, disjunct,
+// Two transducers to one transducer: compose, concatenate, conjunct, disjunct,
void print_common_binary_program_options(FILE *file);
void print_common_binary_program_parameter_instructions(FILE *file);
#define HFST_GETOPT_BINARY_SHORT "1:2:o:C"
diff --git a/tools/src/hfst-project.cc b/tools/src/hfst-project.cc
index 6b65ff9..e8882ca 100644
--- a/tools/src/hfst-project.cc
+++ b/tools/src/hfst-project.cc
@@ -86,13 +86,13 @@ parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
- // add tool-specific options here
+ // add tool-specific options here
{"project", required_argument, 0, 'p'},
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "p:",
long_options, &option_index);
if (-1 == c)
@@ -154,7 +154,7 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
{
if (project_input)
{
- verbose_printf("Projecting first %s...\n", inputname);
+ verbose_printf("Projecting first %s...\n", inputname);
}
else
{
@@ -217,7 +217,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-prune-alphabet.cc b/tools/src/hfst-prune-alphabet.cc
index 0703b53..39e791d 100644
--- a/tools/src/hfst-prune-alphabet.cc
+++ b/tools/src/hfst-prune-alphabet.cc
@@ -87,14 +87,14 @@ parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
- // add tool-specific options here
+ // add tool-specific options here
{"force", no_argument, 0, 'f'},
{"safe", no_argument, 0, 'S'},
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "fS",
long_options, &option_index);
if (-1 == c)
@@ -177,7 +177,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-push-weights.cc b/tools/src/hfst-push-weights.cc
index 346d48b..d9e5791 100644
--- a/tools/src/hfst-push-weights.cc
+++ b/tools/src/hfst-push-weights.cc
@@ -87,13 +87,13 @@ parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
- // add tool-specific options here
+ // add tool-specific options here
{"push", required_argument, 0, 'p'},
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "p:",
long_options, &option_index);
if (-1 == c)
@@ -151,7 +151,7 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
{
if (push_initial)
{
- verbose_printf("Pushing towards start %s...\n", inputname);
+ verbose_printf("Pushing towards start %s...\n", inputname);
}
else
{
@@ -214,7 +214,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-regexp2fst.cc b/tools/src/hfst-regexp2fst.cc
index 25ad3a5..6672fc3 100644
--- a/tools/src/hfst-regexp2fst.cc
+++ b/tools/src/hfst-regexp2fst.cc
@@ -76,6 +76,7 @@ static hfst::ImplementationType output_format = hfst::UNSPECIFIED_TYPE;
static bool harmonize=true;
static bool harmonize_flags=false;
+static bool minimize_result=true;
void
print_usage()
@@ -83,10 +84,10 @@ print_usage()
// c.f. http://www.gnu.org/prep/standards/standards.html#g_t_002d_002dhelp
fprintf(message_out, "Usage: %s [OPTIONS...] [INFILE]\n"
"Compile (weighted) regular expressions into transducer(s)"
- "\n", program_name);
+ "\n", program_name);
print_common_program_options(message_out);
- print_common_unary_program_options(message_out);
- fprintf(message_out,
+ print_common_unary_program_options(message_out);
+ fprintf(message_out,
"String and format options:\n"
" -f, --format=FMT Write result in FMT format\n"
" -j, --disjunct Disjunct all regexps instead of transforming\n"
@@ -101,20 +102,21 @@ print_usage()
" -x, --xerox-composition=VALUE Whether flag diacritics are treated as ordinary\n"
" symbols in composition (default is false).\n"
" -X, --xfst=VARIABLE Toggle xfst compatibility option VARIABLE.\n"
-"Harmonization:\n"
+"Harmonization and optimization options:\n"
" -H, --do-not-harmonize Do not expand '?' symbols.\n"
" -F, --harmonize-flags Harmonize flag diacritics.\n"
" -E, --encode-weights Encode weights when minimizing (default is false).\n"
+" -M, --do-not-minimize Determinize result instead of minimizing it.\n"
);
fprintf(message_out, "\n");
- fprintf(message_out,
+ fprintf(message_out,
"If OUTFILE or INFILE is missing or -, standard streams will be used.\n"
"FMT must be one of the following: "
"{foma, sfst, openfst-tropical, openfst-log}.\n"
"If EPS is not defined, the default representation of 0 is used\n"
- "VALUEs recognized are {true,ON,yes} and {false,OFF,no}.\n"
- "Xfst variables are {flag-is-epsilon (default OFF)}.\n"
+ "VALUEs recognized are {true,ON,yes} and {false,OFF,no}.\n"
+ "Xfst variables are {flag-is-epsilon (default OFF)}.\n"
"\n"
);
@@ -126,7 +128,7 @@ print_usage()
//" echo \" {cat}:{dog} ; 3 \" | %s legacy way of defining weights\n"
" echo \" cat ; dog ; \"3\" \" | %s -S create transducers\n"
" \"cat\" and \"dog\" and \"3\"\n"
- "\n", program_name, program_name, program_name, program_name, program_name);
+ "\n", program_name, program_name, program_name, program_name);
print_report_bugs();
fprintf(message_out, "\n");
print_more_info();
@@ -157,11 +159,12 @@ parse_options(int argc, char** argv)
{"encode-weights", no_argument, 0, 'E'},
{"xerox-composition", required_argument, 0, 'x'},
{"xfst", required_argument, 0, 'X'},
+ {"do-not-minimize", no_argument, 0, 'M'},
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
- HFST_GETOPT_UNARY_SHORT "je:lSf:HFEx:X:"/*"123"*/,
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ HFST_GETOPT_UNARY_SHORT "je:lSf:HFEx:X:M"/*"123"*/,
long_options, &option_index);
if (-1 == c)
{
@@ -205,6 +208,9 @@ parse_options(int argc, char** argv)
case 'E':
encode_weights=true;
break;
+ case 'M':
+ minimize_result=false;
+ break;
case 'x':
{
const char * argument = hfst_strdup(optarg);
@@ -268,9 +274,10 @@ process_stream(HfstOutputStream& outstream)
comp.set_error_stream(&std::cerr);
comp.set_harmonization(harmonize);
comp.set_flag_harmonization(harmonize_flags);
+ hfst::set_minimization(minimize_result);
HfstTransducer disjunction(output_format);
- char delim = (line_separated)? '\n' : ';';
+ char delim = (line_separated)? '\n' : ';';
char* first_line = 0;
if (!line_separated)
@@ -283,7 +290,16 @@ process_stream(HfstOutputStream& outstream)
{
transducer_n++;
verbose_printf("Compiling expression #%i\n", (int)transducer_n);
- compiled = comp.compile_first(filebuf_, chars_read);
+ try
+ {
+ compiled = comp.compile_first(filebuf_, chars_read);
+ }
+ catch (const HfstException & e)
+ {
+ error(EXIT_FAILURE, 0, "%s: XRE parsing failed "
+ "in expression #%u separated by semicolons:\n%s", inputfilename,
+ (unsigned int)transducer_n, e.what().c_str());
+ }
if (compiled == NULL)
{
if (comp.contained_only_comments())
@@ -291,14 +307,14 @@ process_stream(HfstOutputStream& outstream)
if (transducer_n == 1)
{
error(EXIT_FAILURE, 0, "%s: XRE parsing failed: expression #%u "
- "contains only whitespace or comments", inputfilename,
+ "contains only whitespace or comments", inputfilename,
(unsigned int)transducer_n);
}
break;
}
else
{
- error(EXIT_FAILURE, 0, "%s: XRE parsing failed"
+ error(EXIT_FAILURE, 0, "%s: XRE parsing failed "
"in expression #%u separated by semicolons", inputfilename,
(unsigned int)transducer_n);
}
@@ -358,7 +374,15 @@ process_stream(HfstOutputStream& outstream)
transducer_n++;
HfstTransducer* compiled;
verbose_printf("Compiling expression %u\n", line_count);
- compiled = comp.compile(exp);
+ try
+ {
+ compiled = comp.compile(exp);
+ }
+ catch (HfstException & e)
+ {
+ error_at_line(EXIT_FAILURE, 0, inputfilename, line_count,
+ "XRE parsing failed");
+ }
if (compiled == NULL)
{
if (!comp.contained_only_comments())
@@ -389,7 +413,7 @@ process_stream(HfstOutputStream& outstream)
{
if (delim == '\n')
{
- hfst_set_name(disjunction,
+ hfst_set_name(disjunction,
"?",
"xre");
}
@@ -406,7 +430,7 @@ process_stream(HfstOutputStream& outstream)
//extern int xredebug;
-int main( int argc, char **argv )
+int main( int argc, char **argv )
{
#ifdef WINDOWS
_setmode(1, _O_BINARY);
@@ -434,13 +458,14 @@ int main( int argc, char **argv )
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstOutputStream* outstream = (outfile != stdout) ?
new HfstOutputStream(outfilename, output_format) :
new HfstOutputStream(output_format);
process_stream(*outstream);
+ delete outstream;
if (encode_weights)
{
diff --git a/tools/src/hfst-remove-epsilons.cc b/tools/src/hfst-remove-epsilons.cc
index cd9cc1c..3164c34 100644
--- a/tools/src/hfst-remove-epsilons.cc
+++ b/tools/src/hfst-remove-epsilons.cc
@@ -81,12 +81,12 @@ parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
- // add tool-specific options here
+ // add tool-specific options here
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT,
long_options, &option_index);
if (-1 == c)
@@ -128,11 +128,11 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
}
if (transducer_n==1)
{
- verbose_printf("Removing epsilons %s...\n", inputname);
+ verbose_printf("Removing epsilons %s...\n", inputname);
}
else
{
- verbose_printf("Removing epsilons %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
+ verbose_printf("Removing epsilons %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
}
trans.remove_epsilons();
hfst_set_name(trans, trans, "remove-epsilons");
@@ -167,7 +167,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-repeat.cc b/tools/src/hfst-repeat.cc
index 8bc2069..432e902 100644
--- a/tools/src/hfst-repeat.cc
+++ b/tools/src/hfst-repeat.cc
@@ -70,7 +70,7 @@ print_usage()
" -t, --to=TNUM repeat at most TNUM times\n");
fprintf(message_out, "\n");
print_common_unary_program_parameter_instructions(message_out);
- fprintf(message_out,
+ fprintf(message_out,
"FNUM and TNUM must be positive integers or infinities "
"as parsed by strtod(3)\n"
"if FNUM is omitted it defaults to 0, if TNUM is omitted it defaults to Inf\n"
@@ -95,12 +95,12 @@ parse_options(int argc, char** argv)
HFST_GETOPT_UNARY_LONG,
{"from", required_argument, 0, 'f'},
{"to", required_argument, 0, 't'},
- // add tool-specific options here
+ // add tool-specific options here
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "f:t:",
long_options, &option_index);
if (-1 == c)
@@ -174,7 +174,7 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
{
if (!from_infinity && !to_infinity)
{
- verbose_printf("Repeating [%lu..%lu] %s... " SIZE_T_SPECIFIER "\n",
+ verbose_printf("Repeating [%lu..%lu] %s... " SIZE_T_SPECIFIER "\n",
at_least, at_most, inputname, transducer_n);
}
else if (from_infinity && to_infinity)
@@ -196,7 +196,7 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
if (!from_infinity && !to_infinity)
{
trans.repeat_n_to_k(at_least, at_most);
- char* composed_name = static_cast<char*>(malloc(sizeof(char) *
+ char* composed_name = static_cast<char*>(malloc(sizeof(char) *
(strlen("repeat-%lu-to-%lu"))
+ 1 + 32 + 32));
if (sprintf(composed_name, "repeat-%lu-to-%lu",
@@ -204,7 +204,7 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
{
hfst_set_name(trans, trans, composed_name);
}
- composed_name = static_cast<char*>(malloc(sizeof(char) *
+ composed_name = static_cast<char*>(malloc(sizeof(char) *
(strlen("_%lu^%lu"))
+ 1 + 32 + 32));
if (sprintf(composed_name, "_%lu^%lu",
@@ -223,16 +223,16 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
else if (!from_infinity && to_infinity)
{
trans.repeat_n_plus(at_least);
- char* composed_name = static_cast<char*>(malloc(sizeof(char) *
- (strlen("repeat-%lu-plus"))
+ char* composed_name = static_cast<char*>(malloc(sizeof(char) *
+ (strlen("repeat-%lu-plus"))
+ 1 + 32 + 32));
if (sprintf(composed_name, "repeat-%lu-plus",
at_least) > 0)
{
hfst_set_name(trans, trans, composed_name);
}
- composed_name = static_cast<char*>(malloc(sizeof(char) *
- (strlen("_%lu^∞"))
+ composed_name = static_cast<char*>(malloc(sizeof(char) *
+ (strlen("_%lu^∞"))
+ 1 + 32 + 32));
if (sprintf(composed_name, "_%lu^∞",
at_least) > 0)
@@ -275,7 +275,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
if (!from_infinity && !to_infinity)
{
diff --git a/tools/src/hfst-reverse.cc b/tools/src/hfst-reverse.cc
index 9269f8c..2691664 100644
--- a/tools/src/hfst-reverse.cc
+++ b/tools/src/hfst-reverse.cc
@@ -81,12 +81,12 @@ parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
- // add tool-specific options here
+ // add tool-specific options here
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT,
long_options, &option_index);
if (-1 == c)
@@ -121,11 +121,11 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
char* inputname = hfst_get_name(trans, inputfilename);
if (transducer_n==1)
{
- verbose_printf("Reversing %s...\n", inputname);
+ verbose_printf("Reversing %s...\n", inputname);
}
else
{
- verbose_printf("Reversing %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
+ verbose_printf("Reversing %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
}
trans.reverse();
@@ -161,7 +161,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-reweight.cc b/tools/src/hfst-reweight.cc
index c21a72c..1c31d29 100644
--- a/tools/src/hfst-reweight.cc
+++ b/tools/src/hfst-reweight.cc
@@ -42,7 +42,7 @@
#include "HfstTransducer.h"
#include "HfstInputStream.h"
#include "HfstOutputStream.h"
-#include "implementations/HfstTransitionGraph.h"
+#include "implementations/HfstBasicTransducer.h"
#include "inc/globals-common.h"
#include "inc/globals-unary.h"
@@ -143,7 +143,7 @@ parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
- // add tool-specific options here
+ // add tool-specific options here
{"addition", required_argument, 0, 'a'},
{"multiplier", required_argument, 0, 'b'},
{"function", required_argument, 0, 'F'},
@@ -158,8 +158,8 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "a:b:F:l:u:I:O:S:eT:A",
long_options, &option_index);
if (-1 == c)
@@ -320,7 +320,7 @@ reweight(float w, const char* i, const char* o)
{
return w;
}
- if ((symbol != 0) && ((strcmp(i, symbol) != 0) &&
+ if ((symbol != 0) && ((strcmp(i, symbol) != 0) &&
(strcmp(o, symbol) != 0) ) )
{
// symbol doesn't match, don't apply
@@ -372,7 +372,7 @@ do_reweight(HfstTransducer& trans)
rebuilt[source_state] = state_count;
state_count++;
}
- for (HfstBasicTransducer::HfstTransitions::const_iterator arc =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator arc =
state->begin();
arc != state->end();
++arc)
@@ -391,7 +391,7 @@ do_reweight(HfstTransducer& trans)
HfstBasicTransition nu(rebuilt[arc->get_target_state()],
arc->get_input_symbol(),
arc->get_output_symbol(),
- reweight(arc->get_weight(),
+ reweight(arc->get_weight(),
arc->get_input_symbol().c_str(),
arc->get_output_symbol().c_str()));
replication.add_transition(rebuilt[source_state], nu);
@@ -414,11 +414,11 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
char* inputname = hfst_get_name(trans, inputfilename);
if (transducer_n==1)
{
- verbose_printf("Reweighting %s...\n", inputname);
+ verbose_printf("Reweighting %s...\n", inputname);
}
else
{
- verbose_printf("Reweighting %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
+ verbose_printf("Reweighting %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
}
if (NULL == tsv_file)
{
@@ -500,7 +500,7 @@ int main( int argc, char **argv ) {
_setmode(1, _O_BINARY);
#endif
- hfst_set_program_name(argv[0], "0.1",
+ hfst_set_program_name(argv[0], "0.1",
"HfstReweight");
int retval = parse_options(argc, argv);
if (retval != EXIT_CONTINUE)
@@ -516,7 +516,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
verbose_printf("Modifying weights %f < w < %f as %f * %s(w) + %f\n",
lower_bound, upper_bound, multiplier, funcname, addition);
diff --git a/tools/src/hfst-scanner.ll b/tools/src/hfst-scanner.ll
deleted file mode 100644
index ae9e0b7..0000000
--- a/tools/src/hfst-scanner.ll
+++ /dev/null
@@ -1,194 +0,0 @@
-%option 8Bit batch yylineno nounput noyywrap
-/*header-file="hfst-scanner.h"*/
-
-/* the "incl" state is used to pick up the name of an include file */
-%x incl
-
-%{
-/*******************************************************************/
-/* */
-/* FILE scanner.ll */
-/* MODULE scanner */
-/* PROGRAM HFST */
-/* */
-/*******************************************************************/
-
-#include <string.h>
-
-#include "HfstCompiler.h"
-
-#ifdef YACC_USE_PARSER_H_EXTENSION
- #include "hfst-compiler.h"
-#else
- #include "hfst-compiler.hh"
-#endif
-
-#include "HfstBasic.h"
-#include "HfstUtf8.h"
-#include "HfstTransducer.h"
-
-using namespace hfst;
-using namespace basic;
-
-#define MAX_INCLUDE_DEPTH 10
-
-int Include_Stack_Ptr = 0;
-YY_BUFFER_STATE Include_Stack[MAX_INCLUDE_DEPTH];
-char *Name_Stack[MAX_INCLUDE_DEPTH];
-int Lineno_Stack[MAX_INCLUDE_DEPTH];
-
-bool Verbose=true;
-char *FileName=NULL;
-bool UTF8=true;
-
-static char *unquote(char *string, bool del_quote=true) {
- char *s=string, *result=string;
- if (del_quote)
- string++;
-
- while (*string) {
- if (*string == '\\')
- string++;
- *(s++) = *(string++);
- }
-
- if (del_quote)
- s--;
- *s = '\0';
-
- return fst_strdup(result);
-}
-
-static void print_lineno() {
- if (!Verbose)
- return;
- fputc('\r',stderr);
- for( int i=0; i<Include_Stack_Ptr; i++ )
- fputs(" ", stderr);
- fprintf(stderr,"%s: %d", FileName, yylineno);
-}
-
-extern void yyerror(char *text);
-
-%}
-
-CC [\x80-\xbf]
-C1 [A-Za-z0-9._/\-]
-C2 [A-Za-z0-9._/\-&()+,=?\^|~]
-C3 [A-Za-z0-9._/\-&()+,=?\^|~#<>]
-C4 [A-Za-z0-9._/\-&()+,=?\^|~$<>]
-C5 [\!-;\?-\[\]-\~=]
-FN [A-Za-z0-9._/\-*+]
-
-%%
-
-^[ \t]*\#use[ \t]*hopcroft[ \t]*\n { hfst::set_minimization_algorithm(hfst::HOPCROFT);};
-^[ \t]*\#use[ \t]*default[ \t]*\n { hfst::set_minimization_algorithm(hfst::BRZOZOWSKI);};
-
-
-#include BEGIN(incl);
-<incl>[ \t]* /* eat the whitespace */
-<incl>{FN}+ { HfstCompiler::error2("Missing quotes",yytext); }
-<incl>\"{FN}+\" { /* got the include file name */
- FILE *file;
- char *name=fst_strdup(yytext+1);
- name[strlen(name)-1] = 0;
- if ( Include_Stack_Ptr >= MAX_INCLUDE_DEPTH ) {
- fprintf( stderr, "Includes nested too deeply" );
- exit( 1 );
- }
- if (Verbose) fputc('\n', stderr);
- file = fopen( name, "rt" );
- if (!file)
- HfstCompiler::error2("Can't open include file", name);
- else {
- Name_Stack[Include_Stack_Ptr] = FileName;
- FileName = name;
- Lineno_Stack[Include_Stack_Ptr] = yylineno;
- yylineno = 1;
- Include_Stack[Include_Stack_Ptr++]=YY_CURRENT_BUFFER;
- yy_switch_to_buffer(yy_create_buffer(yyin, YY_BUF_SIZE));
- yyin = file;
- print_lineno();
- BEGIN(INITIAL);
- }
- }
-<<EOF>> {
- if (Verbose)
- fputc('\n', stderr);
- if ( --Include_Stack_Ptr < 0 )
- yyterminate();
- else {
- free(FileName);
- FileName = Name_Stack[Include_Stack_Ptr];
- yylineno = Lineno_Stack[Include_Stack_Ptr];
- yy_delete_buffer( YY_CURRENT_BUFFER );
- yy_switch_to_buffer(Include_Stack[Include_Stack_Ptr]);
- }
- }
-
-
-^[ \t]*\%.*\r?\n { print_lineno(); /* ignore comments */ }
-
-\%.*\\[ \t]*\r?\n { print_lineno(); /* ignore comments */ }
-
-\%.* { /* ignore comments */ }
-
-
-^[ \t]*ALPHABET[ \t]*= { return ALPHA; }
-
-\|\| { return COMPOSE; }
-"<=>" { yylval.type = twol_both; return ARROW; }
-"=>" { yylval.type = twol_right;return ARROW; }
-"<=" { yylval.type = twol_left; return ARROW; }
-"^->" { yylval.rtype = repl_up; return REPLACE; }
-"_->" { yylval.rtype = repl_down; return REPLACE; }
-"/->" { yylval.rtype = repl_right;return REPLACE; }
-"\\->" { yylval.rtype = repl_left; return REPLACE; }
-">>" { return PRINT; }
-"<<" { return INSERT; }
-"<<<" { return SUBSTITUTE; }
-"__" { return POS; }
-"^_" { return SWITCH; }
-
-[.,{}\[\]()&!?|*+:=_\^\-] { return yytext[0]; }
-
-\$=({C3}|(\\.))+\$ { yylval.name = fst_strdup(yytext); return RVAR; }
-
-\$({C3}|(\\.))+\$ { yylval.name = fst_strdup(yytext); return VAR; }
-
-#=({C4}|(\\.))+# { yylval.name = fst_strdup(yytext); return RSVAR; }
-
-#({C4}|(\\.))+# { yylval.name = fst_strdup(yytext); return SVAR; }
-
-\<({C5}|\\.)*\> { yylval.name = unquote(yytext,false); return SYMBOL; }
-
-\"<{FN}+>\" {
- yylval.value = fst_strdup(yytext+2);
- yylval.value[strlen(yylval.value)-2] = 0;
- return STRING2;
- }
-
-\"{FN}+\" {
- yylval.value = fst_strdup(yytext+1);
- yylval.value[strlen(yylval.value)-1] = 0;
- return STRING;
- }
-
-[ \t] { /* ignored */ }
-\\[ \t]*([ \t]\%.*)?\r?\n { print_lineno(); /* ignored */ }
-\r?\n { print_lineno(); return NEWLINE; }
-
-\\[0-9]+ { long l=atol(yytext+1);
- if (l <= 1114112) { yylval.value=fst_strdup(hfst_utf8::int2utf8((unsigned)l)); return UTF8CHAR; }
- yyerror(strdup("invalid expression"));
- }
-
-
-\\. { yylval.value=fst_strdup(yytext+1); return UTF8CHAR; }
-[\x00-\x7f] { yylval.value=fst_strdup(yytext); return UTF8CHAR; }
-[\xc0-\xdf]{CC} { yylval.value=fst_strdup(yytext); return UTF8CHAR; }
-[\xe0-\xef]{CC}{2} { yylval.value=fst_strdup(yytext); return UTF8CHAR; }
-[\xf0-\xff]{CC}{3} { yylval.value=fst_strdup(yytext); return UTF8CHAR; }
-
-%%
diff --git a/tools/src/hfst-shuffle.cc b/tools/src/hfst-shuffle.cc
index 390ffb1..106dfc3 100644
--- a/tools/src/hfst-shuffle.cc
+++ b/tools/src/hfst-shuffle.cc
@@ -86,7 +86,7 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_BINARY_SHORT,
long_options, &option_index);
if (-1 == c)
@@ -164,7 +164,7 @@ shuffle_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
char* secondname = hfst_get_name(*second, secondfilename);
if (transducer_n_first == 1)
{
- verbose_printf("Shuffling %s and %s...\n", firstname,
+ verbose_printf("Shuffling %s and %s...\n", firstname,
secondname);
}
else
@@ -206,13 +206,13 @@ shuffle_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
hfst_set_formula(*first, *first, *second, "shuffle");
outstream << *first;
- continueReading = firststream.is_good() &&
+ continueReading = firststream.is_good() &&
(secondstream.is_good() || transducer_n_second == 1);
delete first;
first=0;
// delete the transducer of second stream, unless we continue reading
- // the first stream and there is only one transducer in the second
+ // the first stream and there is only one transducer in the second
// stream
if ((continueReading && secondstream.is_good()) || not continueReading)
{
@@ -226,10 +226,10 @@ shuffle_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
if (firststream.is_good())
{
- error(EXIT_FAILURE, 0,
+ error(EXIT_FAILURE, 0,
"second input '%s' contains fewer transducers than first input"
" '%s'; this is only possible if the second input contains"
- " exactly one transducer",
+ " exactly one transducer",
secondfilename, firstfilename);
}
@@ -272,7 +272,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s and %s, writing to %s\n",
+ verbose_printf("Reading from %s and %s, writing to %s\n",
firstfilename, secondfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* firststream = NULL;
diff --git a/tools/src/hfst-split.cc b/tools/src/hfst-split.cc
index e1d4292..edf163b 100644
--- a/tools/src/hfst-split.cc
+++ b/tools/src/hfst-split.cc
@@ -66,7 +66,7 @@ print_usage()
" -e, --extension=EXT Use the extension EXT in "
"naming output files\n");
fprintf(message_out, "\n");
- fprintf(message_out,
+ fprintf(message_out,
"If INFILE is omitted or -, stdin is used.\n"
"If PRE is omitted, no prefix is used.\n"
"If EXT is omitted, .hfst is used.\n"
@@ -98,12 +98,12 @@ parse_options(int argc, char** argv)
{"input", required_argument, 0, 'i'},
{"prefix", required_argument, 0, 'p'},
{"extension", required_argument, 0, 'e'},
- // add tool-specific options here
+ // add tool-specific options here
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT "i:p:e:",
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT "i:p:e:",
long_options, &option_index);
if (-1 == c)
{
@@ -116,7 +116,7 @@ parse_options(int argc, char** argv)
case 'i':
inputfilename = hfst_strdup(optarg);
inputfile = hfst_fopen(inputfilename, "r");
- if (inputfile == stdin)
+ if (inputfile == stdin)
{
free(inputfilename);
inputfilename = hfst_strdup("<stdin>");
@@ -155,7 +155,7 @@ process_stream(HfstInputStream& instream)
sprintf(outfilename, "%s" SIZE_T_SPECIFIER "%s", prefix, transducer_n,
extension);
verbose_printf("Writing " SIZE_T_SPECIFIER " of %s to %s...\n", transducer_n,
- inputfilename, outfilename);
+ inputfilename, outfilename);
HfstOutputStream* outstream = new HfstOutputStream(outfilename,
instream.get_type());
//outstream->open();
@@ -191,7 +191,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s...%s\n",
+ verbose_printf("Reading from %s, writing to %s...%s\n",
inputfilename, prefix, extension);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-strings2fst.cc b/tools/src/hfst-strings2fst.cc
index 860b7f0..e68ac27 100644
--- a/tools/src/hfst-strings2fst.cc
+++ b/tools/src/hfst-strings2fst.cc
@@ -51,7 +51,7 @@ using std::pair;
#include "HfstTransducer.h"
#include "HfstInputStream.h"
#include "HfstOutputStream.h"
-#include "implementations/HfstTransitionGraph.h"
+#include "implementations/HfstBasicTransducer.h"
#include "hfst-commandline.h"
#include "hfst-program-options.h"
#include "hfst-tool-metadata.h"
@@ -127,9 +127,9 @@ print_usage()
// c.f. http://www.gnu.org/prep/standards/standards.html#g_t_002d_002dhelp
fprintf(message_out, "Usage: %s [OPTIONS...] [INFILE]\n"
"Compile string pairs and pair-strings into transducer(s)\n"
- "\n", program_name);
+ "\n", program_name);
print_common_program_options(message_out);
- fprintf(message_out,
+ fprintf(message_out,
"Input/Output options:\n"
" -i, --input=INFILE Read input strings from INFILE\n"
" -o, --output=OUTFILE Write output transducer to OUTFILE\n");
@@ -150,14 +150,14 @@ print_usage()
);
fprintf(message_out, "\n");
- fprintf(message_out,
+ fprintf(message_out,
"If OUTFILE or INFILE is missing or -, standard streams will be used.\n"
"FMT can be { foma, openfst-tropical, openfst-log, sfst, \n"
"optimized-lookup-weighted, optimized-lookup-unweighted }.\n"
"If EPS is not defined, the default representation of @0@ is used.\n"
"Option --norm precedes option --log.\n"
"The FILE of option -m lists all multichar-symbols, each symbol\n"
- "on its own line.\n"
+ "on its own line.\n"
"Backslash '\\' may be used to escape ':', tab and itself. For any\n"
"other symbol x '\\x' means x literally, i.e. is the same as 'x'.\n"
"The weight of a string can be given after the string separated\n"
@@ -200,7 +200,7 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "je:234pSm:f:",
long_options, &option_index);
if (-1 == c)
@@ -286,7 +286,7 @@ process_stream(HfstOutputStream& outstream)
if (tab == NULL)
{
string_end = line;
- while ((*string_end != '\0') && (*string_end != '\n') &&
+ while ((*string_end != '\0') && (*string_end != '\n') &&
(*string_end != '\r'))
{
string_end++;
@@ -319,7 +319,7 @@ process_stream(HfstOutputStream& outstream)
(line,has_spaces); }
}
catch (const hfst::UnescapedColsFound &e)
- {
+ {
if (pairstrings)
{
error_at_line
@@ -335,7 +335,7 @@ process_stream(HfstOutputStream& outstream)
"String `%s' contains unescaped `:'-symbols,\n"
"which are not pair separators. Use `\\:\' for literal `:'.\n"
"If you are compiling pair strings, use option -p.",
- line);
+ line);
}
}
catch (const IncorrectUtf8CodingException &e)
@@ -357,11 +357,11 @@ process_stream(HfstOutputStream& outstream)
{
HfstBasicTransducer tr;
- if (logarithmic_weights_e)
+ if (logarithmic_weights_e)
{
path_weight = take_negative_logarithm_e(weight);
}
- else if (logarithmic_weights_10)
+ else if (logarithmic_weights_10)
{
path_weight = take_negative_logarithm_10(weight);
}
@@ -384,7 +384,7 @@ process_stream(HfstOutputStream& outstream)
{
HfstTransducer res(disjunction, output_format);
- if (normalize_weights)
+ if (normalize_weights)
{
verbose_printf("Normalising weights...\n");
res.transform_weights(÷_by_sum_of_weights);
@@ -408,7 +408,7 @@ process_stream(HfstOutputStream& outstream)
}
-int main( int argc, char **argv )
+int main( int argc, char **argv )
{
#ifdef WINDOWS
_setmode(1, _O_BINARY);
@@ -423,7 +423,7 @@ int main( int argc, char **argv )
if (multichar_symbol_filename != NULL)
{
- verbose_printf("Reading multichar symbols from %s\n",
+ verbose_printf("Reading multichar symbols from %s\n",
multichar_symbol_filename);
std::ifstream multichar_in(multichar_symbol_filename);
(void)multichar_in.peek();
@@ -431,12 +431,12 @@ int main( int argc, char **argv )
{ error(EXIT_FAILURE, errno,"Multichar symbol file can't be read."); }
char multichar_line[1000];
while (multichar_in.good())
- {
+ {
multichar_in.getline(multichar_line,1000);
if (strlen(multichar_line) > 0)
- {
+ {
verbose_printf("Defining multichar symbol %s\n",multichar_line);
- multichar_symbols.push_back(multichar_line);
+ multichar_symbols.push_back(multichar_line);
}
}
}
@@ -446,7 +446,7 @@ int main( int argc, char **argv )
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstOutputStream* outstream = (outfile != stdout) ?
diff --git a/tools/src/hfst-strip-header.cc b/tools/src/hfst-strip-header.cc
index ed48c3b..389c84e 100644
--- a/tools/src/hfst-strip-header.cc
+++ b/tools/src/hfst-strip-header.cc
@@ -62,8 +62,8 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT,
long_options, &option_index);
if (-1 == c)
@@ -135,7 +135,7 @@ int main(int argc, char* argv[])
{
return retval;
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
retval = process_stream(inputfile, outfile);
diff --git a/tools/src/hfst-substitute.cc b/tools/src/hfst-substitute.cc
index 7347e40..2dd8f0d 100644
--- a/tools/src/hfst-substitute.cc
+++ b/tools/src/hfst-substitute.cc
@@ -37,7 +37,7 @@
#include "HfstTransducer.h"
#include "HfstInputStream.h"
#include "HfstOutputStream.h"
-#include "implementations/HfstTransitionGraph.h"
+#include "implementations/HfstBasicTransducer.h"
using hfst::HfstTransducer;
using hfst::HfstInputStream;
@@ -175,7 +175,7 @@ print_usage()
);
fprintf(message_out, "\n");
print_common_unary_program_parameter_instructions(message_out);
- fprintf(message_out,
+ fprintf(message_out,
"LABEL must be a symbol name in single arc in transducer,\n"
"or colon separated pair defining an arc.\n"
"If TFILE is specified, FLABEL must be a pair.\n"
@@ -208,7 +208,7 @@ parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
- // add tool-specific options here
+ // add tool-specific options here
{"from-label", required_argument, 0, 'f'},
{"from-file", required_argument, 0, 'F'},
{"to-label", required_argument, 0, 't'},
@@ -219,8 +219,8 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "f:F:t:T:R9C",
long_options, &option_index);
if (-1 == c)
@@ -302,7 +302,7 @@ parse_options(int argc, char** argv)
"Must state name of labels to rewrite with -f or -F");
return EXIT_FAILURE;
}
- if ((to_label == 0) && (to_transducer_filename == 0) &&
+ if ((to_label == 0) && (to_transducer_filename == 0) &&
(from_file_name == 0))
{
error(EXIT_FAILURE, 0,
@@ -361,15 +361,15 @@ do_substitute(HfstBasicTransducer& trans, size_t transducer_n)
}
if (transducer_n < 2)
{
- verbose_printf("Substituting pair %s:%s with transducer %s...\n",
+ verbose_printf("Substituting pair %s:%s with transducer %s...\n",
from_pair->first.c_str(),
from_pair->second.c_str(),
to_name);
}
else
{
- verbose_printf("Substituting pair %s:%s with transducer %s... " SIZE_T_SPECIFIER "\n",
- from_pair->first.c_str(),
+ verbose_printf("Substituting pair %s:%s with transducer %s... " SIZE_T_SPECIFIER "\n",
+ from_pair->first.c_str(),
from_pair->second.c_str(), to_name,
transducer_n);
}
@@ -385,12 +385,12 @@ do_substitute(HfstBasicTransducer& trans, size_t transducer_n)
}
if (transducer_n < 2)
{
- verbose_printf("Substituting id. label %s with transducer %s...\n",
+ verbose_printf("Substituting id. label %s with transducer %s...\n",
from_label, to_name);
}
else
{
- verbose_printf("Substituting id. label %s with transducer %s... " SIZE_T_SPECIFIER "\n",
+ verbose_printf("Substituting id. label %s with transducer %s... " SIZE_T_SPECIFIER "\n",
from_label, to_name,
transducer_n);
}
@@ -472,15 +472,15 @@ do_substitute(HfstTransducer& trans, size_t transducer_n)
}
if (transducer_n < 2)
{
- verbose_printf("Substituting pair %s:%s with transducer %s...\n",
+ verbose_printf("Substituting pair %s:%s with transducer %s...\n",
from_pair->first.c_str(),
from_pair->second.c_str(),
to_name);
}
else
{
- verbose_printf("Substituting pair %s:%s with transducer %s... " SIZE_T_SPECIFIER "\n",
- from_pair->first.c_str(),
+ verbose_printf("Substituting pair %s:%s with transducer %s... " SIZE_T_SPECIFIER "\n",
+ from_pair->first.c_str(),
from_pair->second.c_str(), to_name,
transducer_n);
}
@@ -496,12 +496,12 @@ do_substitute(HfstTransducer& trans, size_t transducer_n)
}
if (transducer_n < 2)
{
- verbose_printf("Substituting id. label %s with transducer %s...\n",
+ verbose_printf("Substituting id. label %s with transducer %s...\n",
from_label, to_name);
}
else
{
- verbose_printf("Substituting id. label %s with transducer %s... " SIZE_T_SPECIFIER "\n",
+ verbose_printf("Substituting id. label %s with transducer %s... " SIZE_T_SPECIFIER "\n",
from_label, to_name,
transducer_n);
}
@@ -551,7 +551,7 @@ process_stream(HfstInputStream& instream)
HfstInputStream tostream(to_transducer_filename);
to_transducer = new HfstTransducer(tostream);
tostream.close();
- } catch (NotTransducerStreamException ntse)
+ } catch (NotTransducerStreamException ntse)
{
error(EXIT_FAILURE, 0, "%s is not a valid transducer file",
to_transducer_filename);
@@ -688,7 +688,7 @@ process_stream(HfstInputStream& instream)
hfst::internal_epsilon.c_str());
}
- if (from_pair && to_pair)
+ if (from_pair && to_pair)
{
if (!in_order) {
pair_substitution_map->operator[](*from_pair) = *to_pair;
@@ -698,7 +698,7 @@ process_stream(HfstInputStream& instream)
do_substitute(trans, transducer_n);
}
}
- else if (from_label && to_label)
+ else if (from_label && to_label)
{
if (!in_order) {
label_substitution_map->operator[](std::string(from_label)) = std::string(to_label);
@@ -709,7 +709,7 @@ process_stream(HfstInputStream& instream)
}
}
else {
- try
+ try
{
do_substitute(trans, transducer_n);
}
@@ -719,7 +719,7 @@ process_stream(HfstInputStream& instream)
{
if (!silent) {
warning(0, 0, "substitution is not supported for this transducer type"
- " falling back to internal formats and trying...");
+ " falling back to internal formats and trying...");
}
fallback = new HfstBasicTransducer(trans);
warnedAlready = true;
@@ -734,14 +734,14 @@ process_stream(HfstInputStream& instream)
free(line);
// perform label-to-label substitution right away
- if (!in_order && symbol_map_in_use)
+ if (!in_order && symbol_map_in_use)
{
trans.substitute(*label_substitution_map);
symbol_map_in_use=false;
}
// perform symbol pair-to-symbol pair substitution right away
- if (!in_order && symbol_pair_map_in_use)
+ if (!in_order && symbol_pair_map_in_use)
{
trans.substitute(*pair_substitution_map);
symbol_pair_map_in_use=false;
@@ -780,9 +780,9 @@ process_stream(HfstInputStream& instream)
}
if (from_file)
{
- char* composed_name = static_cast<char*>(malloc(sizeof(char) *
+ char* composed_name = static_cast<char*>(malloc(sizeof(char) *
(strlen(from_file_name) +
- strlen("substitutions-from-%s"))
+ strlen("substitutions-from-%s"))
+ 1));
if (sprintf(composed_name, "substitute-from-%s",
from_file_name) > 0)
@@ -790,9 +790,9 @@ process_stream(HfstInputStream& instream)
hfst_set_name(trans, trans, composed_name);
free(composed_name);
}
- composed_name = static_cast<char*>(malloc(sizeof(char) *
+ composed_name = static_cast<char*>(malloc(sizeof(char) *
(strlen(from_file_name) +
- strlen("♲%s"))
+ strlen("♲%s"))
+ 1));
if (sprintf(composed_name, "♲%s",
from_file_name) > 0)
@@ -803,10 +803,10 @@ process_stream(HfstInputStream& instream)
}
else if (from_label && to_label)
{
- char* composed_name = static_cast<char*>(malloc(sizeof(char) *
+ char* composed_name = static_cast<char*>(malloc(sizeof(char) *
(strlen(from_label) +
strlen(to_label) +
- strlen("substitute-%s-with-%s"))
+ strlen("substitute-%s-with-%s"))
+ 1));
if (sprintf(composed_name, "substitute-%s-with-%s",
from_label, to_label) > 0)
@@ -814,10 +814,10 @@ process_stream(HfstInputStream& instream)
hfst_set_name(trans, trans, composed_name);
free(composed_name);
}
- composed_name = static_cast<char*>(malloc(sizeof(char) *
+ composed_name = static_cast<char*>(malloc(sizeof(char) *
(strlen(from_label) +
strlen(to_label) +
- strlen("%s ♲ %s"))
+ strlen("%s ♲ %s"))
+ 1));
if (sprintf(composed_name, "%s ♲ %s",
from_label, to_label) > 0)
@@ -832,10 +832,10 @@ process_stream(HfstInputStream& instream)
if (from_label == NULL) // make scan-build happy, this should not happen
throw "Error: from_label has a NULL value.";
- char* composed_name = static_cast<char*>(malloc(sizeof(char) *
+ char* composed_name = static_cast<char*>(malloc(sizeof(char) *
(strlen(from_label) +
strlen(to_transducer_filename) +
- strlen("substitute-%s-with-net-%s"))
+ strlen("substitute-%s-with-net-%s"))
+ 1));
if (sprintf(composed_name, "substitute-%s-with-net-%s",
from_label, to_transducer_filename) > 0)
@@ -843,10 +843,10 @@ process_stream(HfstInputStream& instream)
hfst_set_name(trans, trans, composed_name);
free(composed_name);
}
- composed_name = static_cast<char*>(malloc(sizeof(char) *
+ composed_name = static_cast<char*>(malloc(sizeof(char) *
(strlen(from_label) +
strlen(to_transducer_filename) +
- strlen("%s ♲ %s"))
+ strlen("%s ♲ %s"))
+ 1));
if (sprintf(composed_name, "%s ♲ %s",
from_label, to_transducer_filename) > 0)
@@ -876,7 +876,7 @@ process_stream(HfstInputStream& instream)
}
-int main( int argc, char **argv )
+int main( int argc, char **argv )
{
#ifdef WINDOWS
_setmode(0, _O_BINARY);
@@ -898,7 +898,7 @@ int main( int argc, char **argv )
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
if (from_file != NULL)
diff --git a/tools/src/hfst-subtract.cc b/tools/src/hfst-subtract.cc
index 198f626..0ccf814 100644
--- a/tools/src/hfst-subtract.cc
+++ b/tools/src/hfst-subtract.cc
@@ -96,7 +96,7 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_BINARY_SHORT "FH",
long_options, &option_index);
if (-1 == c)
@@ -181,7 +181,7 @@ subtract_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
char* secondname = hfst_get_name(*second, secondfilename);
if (transducer_n_first == 1)
{
- verbose_printf("Subtracting %s from %s...\n", secondname,
+ verbose_printf("Subtracting %s from %s...\n", secondname,
firstname);
}
else
@@ -195,14 +195,14 @@ subtract_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
warning(0, 0, "Warning: %s contains flag diacritics. The "
"result of subtraction may be incorrect.", secondfilename);
}
- if (first->has_flag_diacritics() and second->has_flag_diacritics())
+ if (first->has_flag_diacritics() and second->has_flag_diacritics())
{
if (not harmonize_flags)
{
- if (not silent)
+ if (not silent)
{
warning(0, 0, "The argumentes contain "
- "flag diacritics. Use -F to harmonize them.",
+ "flag diacritics. Use -F to harmonize them.",
secondname, firstname);
}
}
@@ -235,13 +235,13 @@ subtract_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
hfst_set_formula(*first, *first, *second, "−");
outstream << *first;
- continueReading = firststream.is_good() &&
+ continueReading = firststream.is_good() &&
(secondstream.is_good() || transducer_n_second == 1);
delete first;
first=0;
// delete the transducer of second stream, unless we continue reading
- // the first stream and there is only one transducer in the second
+ // the first stream and there is only one transducer in the second
// stream
if ((continueReading && secondstream.is_good()) || not continueReading)
{
@@ -255,10 +255,10 @@ subtract_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
if (firststream.is_good())
{
- error(EXIT_FAILURE, 0,
+ error(EXIT_FAILURE, 0,
"second input '%s' contains fewer transducers than first input"
" '%s'; this is only possible if the second input contains"
- " exactly one transducer",
+ " exactly one transducer",
secondfilename, firstfilename);
}
@@ -302,7 +302,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s and %s, writing to %s\n",
+ verbose_printf("Reading from %s and %s, writing to %s\n",
firstfilename, secondfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* firststream = NULL;
diff --git a/tools/src/hfst-summarize.cc b/tools/src/hfst-summarize.cc
index 73edaf4..9e7998e 100644
--- a/tools/src/hfst-summarize.cc
+++ b/tools/src/hfst-summarize.cc
@@ -35,7 +35,7 @@
#include "HfstTransducer.h"
#include "HfstInputStream.h"
#include "HfstOutputStream.h"
-#include "implementations/HfstTransitionGraph.h"
+#include "implementations/HfstBasicTransducer.h"
using std::map;
using std::string;
@@ -74,7 +74,7 @@ print_usage()
fprintf(message_out, "\n");
print_common_unary_program_parameter_instructions(message_out);
fprintf(message_out, "\n");
- fprintf(message_out,
+ fprintf(message_out,
"The parameter --verbose gives more extensive information on\n"
"the properties of a transducer.\n");
fprintf(message_out, "\n");
@@ -94,13 +94,13 @@ parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
- // add tool-specific options here
+ // add tool-specific options here
{"print-symbol-pair-statistics", optional_argument, 0, 'S'},
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "S::",
long_options, &option_index);
if (-1 == c)
@@ -251,7 +251,7 @@ process_stream(HfstInputStream& instream)
map<string,unsigned int> input_ambiguity;
map<string,unsigned int> output_ambiguity;
- for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
+ for (hfst::implementations::HfstBasicTransitions::const_iterator tr_it
= it->begin();
tr_it != it->end(); tr_it++)
{
@@ -270,7 +270,7 @@ process_stream(HfstInputStream& instream)
{
acceptor = false;
}
- if ( hfst::is_epsilon(tr_it->get_input_symbol()) &&
+ if ( hfst::is_epsilon(tr_it->get_input_symbol()) &&
hfst::is_epsilon(tr_it->get_output_symbol()) )
{
io_epsilons++;
@@ -356,15 +356,15 @@ process_stream(HfstInputStream& instream)
// count physical size
// average calculations
- double average_arcs_per_state =
+ double average_arcs_per_state =
static_cast<double>(arcs)/static_cast<float>(states);
- double average_input_epsilons =
+ double average_input_epsilons =
static_cast<double>(input_epsilons)/static_cast<double>(states);
- double average_input_ambiguity =
+ double average_input_ambiguity =
static_cast<double>(arcs)/static_cast<double>(uniq_input_arcs);
- double average_output_ambiguity =
+ double average_output_ambiguity =
static_cast<double>(arcs)/static_cast<double>(uniq_output_arcs);
- double expected_arcs_per_symbol =
+ double expected_arcs_per_symbol =
static_cast<double>(average_arcs_per_state)/
static_cast<float>(foundAlphabet.size());
@@ -420,8 +420,8 @@ process_stream(HfstInputStream& instream)
"# of ... coaccessible states: ???\n"
"# of ... connected states: ???\n"
"# of ... strongly conn components: ???\n",
- states, arcs,
- static_cast<long int>(initial_state),
+ states, arcs,
+ static_cast<long>(initial_state),
final_states, io_epsilons,
input_epsilons, output_epsilons);
// other names from properties...
@@ -480,7 +480,7 @@ process_stream(HfstInputStream& instream)
s != transducerAlphabet.end();
++s)
{
- if (!first)
+ if (!first)
{
fprintf(outfile, ", ");
}
@@ -499,7 +499,7 @@ process_stream(HfstInputStream& instream)
s != foundAlphabet.end();
++s)
{
- if (!first)
+ if (!first)
{
fprintf(outfile, ", ");
}
@@ -511,11 +511,11 @@ process_stream(HfstInputStream& instream)
if (transducerKnowsAlphabet)
{
StringSet transducerMinusSet;
- std::set_difference(transducerAlphabet.begin(),
+ std::set_difference(transducerAlphabet.begin(),
transducerAlphabet.end(),
- foundAlphabet.begin(),
+ foundAlphabet.begin(),
foundAlphabet.end(),
- std::inserter(transducerMinusSet,
+ std::inserter(transducerMinusSet,
transducerMinusSet.end()));
first = true;
@@ -523,7 +523,7 @@ process_stream(HfstInputStream& instream)
s != transducerMinusSet.end();
++s)
{
- if (!first)
+ if (!first)
{
fprintf(outfile, ", ");
}
@@ -546,7 +546,7 @@ process_stream(HfstInputStream& instream)
s != ss.end();
++s)
{
- if (!first)
+ if (!first)
{
fprintf(outfile, ", ");
}
@@ -559,7 +559,7 @@ process_stream(HfstInputStream& instream)
} // if verbose
// ADDED
- if (print_symbol_pair_statistics)
+ if (print_symbol_pair_statistics)
{
if (symbol_pair_threshold > -1)
{
@@ -603,7 +603,7 @@ int main( int argc, char **argv ) {
{
fclose(inputfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-tagger/src/Makefile.am b/tools/src/hfst-tagger/src/Makefile.am
index 61e3964..bd6dbe7 100644
--- a/tools/src/hfst-tagger/src/Makefile.am
+++ b/tools/src/hfst-tagger/src/Makefile.am
@@ -1,28 +1,27 @@
AM_CXXFLAGS= -O3 -g -Wall -Wextra -Wcast-qual -Wno-deprecated -Wfatal-errors -Wno-sign-compare\
- -I$(top_builddir)/tools/src/inc -I$(top_builddir)/tools/src
+ -I$(top_builddir)/tools/src/inc -I$(top_builddir)/tools/src
+if WANT_TRAIN_TAGGER
bin_PROGRAMS=hfst-build-tagger$(EXEEXT) hfst-open-input-file-for-tagger$(EXEEXT) hfst-tag \
hfst-reweight-tagger$(EXEEXT) hfst-train-tagger-system$(EXEEXT)
bin_SCRIPTS=hfst_tagger_compute_data_statistics.py hfst-train-tagger$(EXEEXT) \
hfst-train-tagger-loc$(EXEEXT)
EXTRA_DIST=hfst-train-tagger.sh hfst-train-tagger.bat
+else
+bin_PROGRAMS=hfst-tag hfst-reweight-tagger$(EXEEXT)
+bin_SCRIPTS=
+EXTRA_DIST=
+endif
+
BUILD_MODEL_SRC=$(srcdir)/build_model_src
USE_MODEL_SRC=$(srcdir)/use_model_src
AM_CPPFLAGS=-I${top_srcdir}/libhfst/src -Wno-sign-compare -Wno-unused-parameter
LDADD = $(top_builddir)/libhfst/src/libhfst.la
-AM_CPPFLAGS += -I${top_srcdir}/back-ends/foma \
- -I${top_srcdir}/back-ends
-
-if WANT_MINGW
- AM_CPPFLAGS += -I${top_srcdir}/back-ends/openfstwin/src/include \
- -I${top_srcdir}/back-ends/dlfcn -DWINDOWS
-else
- AM_CPPFLAGS += -I${top_srcdir}/back-ends/openfst/src/include
-endif
-
+if WANT_TRAIN_TAGGER
python_PYTHON=$(BUILD_MODEL_SRC)/tagger_aux.py \
hfst_tagger_compute_data_statistics.py
+endif
HFST_COMMON_SRC=$(top_srcdir)/tools/src/hfst-program-options.cc \
$(top_srcdir)/tools/src/hfst-commandline.cc \
@@ -34,6 +33,7 @@ AM_INSTALLCHECK_STD_OPTIONS_EXEMPT=hfst_tagger_compute_data_statistics.py \
hfst-train-tagger hfst-train-tagger-loc \
hfst-train-tagger-system
+if WANT_TRAIN_TAGGER
hfst_build_tagger_SOURCES=hfst-build-tagger.cc \
$(BUILD_MODEL_SRC)/FstBuilder.cc \
$(BUILD_MODEL_SRC)/LexicalModelBuilder.cc \
@@ -43,6 +43,7 @@ hfst_build_tagger_SOURCES=hfst-build-tagger.cc \
$(BUILD_MODEL_SRC)/WeightedStringVectorCollection.cc\
$(BUILD_MODEL_SRC)/string_handling.cc \
$(HFST_COMMON_SRC)
+endif
hfst_tag_SOURCES=hfst-tag.cc \
$(USE_MODEL_SRC)/DelayedSequenceModelComponent.cc \
@@ -58,6 +59,7 @@ hfst_tag_SOURCES=hfst-tag.cc \
hfst_reweight_tagger_SOURCES=hfst-reweight-tagger.cc \
$(HFST_COMMON_SRC)
+if WANT_TRAIN_TAGGER
hfst_train_tagger_system_SOURCES=hfst-train-tagger-system.cc
hfst-train-tagger$(EXEEXT): hfst-train-tagger.sh
@@ -70,11 +72,14 @@ hfst-train-tagger-loc$(EXEEXT): hfst-train-tagger.sh
clean-local:
-rm -f hfst-train-tagger$(EXEEXT) hfst-train-tagger-loc$(EXEEXT) hfst-train-tagger-system$(EXEEXT)
+endif
unit_test_aux.o:$(srcdir)/unit_test_aux.cc
+if WANT_TRAIN_TAGGER
hfst_open_input_file_for_tagger_SOURCES=hfst-open-input-file-for-tagger.cc \
$(HFST_COMMON_SRC)
+endif
check_PROGRAMS=FstBuilder LexicalModelBuilder ModelBuilder \
SequenceModelBuilder TaggerBuilder \
@@ -87,53 +92,53 @@ check_PROGRAMS=FstBuilder LexicalModelBuilder ModelBuilder \
FstBuilder_SOURCES=$(BUILD_MODEL_SRC)/FstBuilder.cc
FstBuilder_LDADD=string_handling.o FstBuilder.o \
$(top_srcdir)/libhfst/src/libhfst.la
-FstBuilder_CXXFLAGS=-DMAIN_TEST
+FstBuilder_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
LexicalModelBuilder_SOURCES=$(BUILD_MODEL_SRC)/LexicalModelBuilder.cc
LexicalModelBuilder_LDADD=string_handling.o FstBuilder.o ModelBuilder.o \
WeightedStringVectorCollection.o \
LexicalModelBuilder.o \
$(top_srcdir)/libhfst/src/libhfst.la
-LexicalModelBuilder_CXXFLAGS=-DMAIN_TEST
+LexicalModelBuilder_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
SequenceModelBuilder_SOURCES=$(BUILD_MODEL_SRC)/SequenceModelBuilder.cc
SequenceModelBuilder_LDADD=string_handling.o FstBuilder.o ModelBuilder.o \
WeightedStringVectorCollection.o \
SequenceModelBuilder.o \
$(top_srcdir)/libhfst/src/libhfst.la
-SequenceModelBuilder_CXXFLAGS=-DMAIN_TEST
+SequenceModelBuilder_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
ModelBuilder_SOURCES=$(BUILD_MODEL_SRC)/ModelBuilder.cc
ModelBuilder_LDADD=string_handling.o FstBuilder.o ModelBuilder.o \
WeightedStringVectorCollection.o \
$(top_srcdir)/libhfst/src/libhfst.la
-ModelBuilder_CXXFLAGS=-DMAIN_TEST
+ModelBuilder_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
WeightedStringVectorCollection_SOURCES=\
$(BUILD_MODEL_SRC)/WeightedStringVectorCollection.cc
WeightedStringVectorCollection_LDADD=string_handling.o \
WeightedStringVectorCollection.o \
$(top_srcdir)/libhfst/src/libhfst.la
-WeightedStringVectorCollection_CXXFLAGS=-DMAIN_TEST
+WeightedStringVectorCollection_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
TaggerBuilder_SOURCES=$(BUILD_MODEL_SRC)/TaggerBuilder.cc
TaggerBuilder_LDADD=string_handling.o TaggerBuilder.o \
LexicalModelBuilder.o SequenceModelBuilder.o \
WeightedStringVectorCollection.o ModelBuilder.o FstBuilder.o \
$(top_srcdir)/libhfst/src/libhfst.la
-TaggerBuilder_CXXFLAGS=-DMAIN_TEST
+TaggerBuilder_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
string_handling_SOURCES=$(BUILD_MODEL_SRC)/string_handling.cc
string_handling_LDADD=string_handling.o $(top_srcdir)/libhfst/src/libhfst.la \
unit_test_aux.o
-string_handling_CXXFLAGS=-DMAIN_TEST
+string_handling_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
AcyclicAutomaton_SOURCES=$(USE_MODEL_SRC)/AcyclicAutomaton.cc
AcyclicAutomaton_LDADD=AcyclicAutomaton.o $(top_srcdir)/libhfst/src/libhfst.la\
SequenceModelComponent.o unit_test_aux.o
-AcyclicAutomaton_CXXFLAGS=-DMAIN_TEST
+AcyclicAutomaton_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
DelayedSequenceModelComponent_SOURCES= \
$(USE_MODEL_SRC)/DelayedSequenceModelComponent.cc
@@ -141,13 +146,13 @@ DelayedSequenceModelComponent_SOURCES= \
DelayedSequenceModelComponent_LDADD=DelayedSequenceModelComponent.o \
$(top_srcdir)/libhfst/src/libhfst.la \
SequenceModelComponent.o unit_test_aux.o
-DelayedSequenceModelComponent_CXXFLAGS=-DMAIN_TEST
+DelayedSequenceModelComponent_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
NewLexicalModel_SOURCES=$(USE_MODEL_SRC)/NewLexicalModel.cc
NewLexicalModel_LDADD=NewLexicalModel.o $(top_srcdir)/libhfst/src/libhfst.la \
unit_test_aux.o
-NewLexicalModel_CXXFLAGS=-DMAIN_TEST
+NewLexicalModel_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
SentenceTagger_SOURCES=$(USE_MODEL_SRC)/SentenceTagger.cc
@@ -157,20 +162,20 @@ SentenceTagger_LDADD=SentenceTagger.o $(top_srcdir)/libhfst/src/libhfst.la \
SequenceModelComponentPair.o unit_test_aux.o \
SequenceTagger.o AcyclicAutomaton.o
-SentenceTagger_CXXFLAGS=-DMAIN_TEST
+SentenceTagger_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
SentenceTransducer_SOURCES=$(USE_MODEL_SRC)/SentenceTransducer.cc
SentenceTransducer_LDADD=SentenceTransducer.o \
$(top_srcdir)/libhfst/src/libhfst.la \
SequenceModelComponent.o unit_test_aux.o
-SentenceTransducer_CXXFLAGS=-DMAIN_TEST
+SentenceTransducer_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
SequenceModelComponent_SOURCES=$(USE_MODEL_SRC)/SequenceModelComponent.cc\
unit_test_aux.cc
SequenceModelComponent_LDADD=SequenceModelComponent.o \
$(top_srcdir)/libhfst/src/libhfst.la
-SequenceModelComponent_CXXFLAGS=-DMAIN_TEST
+SequenceModelComponent_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
SequenceModelComponentPair_SOURCES= \
$(USE_MODEL_SRC)/SequenceModelComponentPair.cc
@@ -179,7 +184,7 @@ SequenceModelComponentPair_LDADD=SequenceModelComponentPair.o \
$(top_srcdir)/libhfst/src/libhfst.la \
DelayedSequenceModelComponent.o \
SequenceModelComponent.o unit_test_aux.o
-SequenceModelComponentPair_CXXFLAGS=-DMAIN_TEST
+SequenceModelComponentPair_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
SequenceTagger_SOURCES=$(USE_MODEL_SRC)/SequenceTagger.cc
@@ -188,7 +193,7 @@ SequenceTagger_LDADD=SequenceTagger.o $(top_srcdir)/libhfst/src/libhfst.la\
unit_test_aux.o AcyclicAutomaton.o \
SequenceModelComponentPair.o \
DelayedSequenceModelComponent.o
-SequenceTagger_CXXFLAGS=-DMAIN_TEST
+SequenceTagger_CXXFLAGS=-DMAIN_TEST -Wno-deprecated
TESTS=$(check_PROGRAMS)
diff --git a/tools/src/hfst-tagger/src/build_model_src/FstBuilder.cc b/tools/src/hfst-tagger/src/build_model_src/FstBuilder.cc
index 186e515..9106f0a 100644
--- a/tools/src/hfst-tagger/src/build_model_src/FstBuilder.cc
+++ b/tools/src/hfst-tagger/src/build_model_src/FstBuilder.cc
@@ -13,8 +13,8 @@ FstBuilder::FstBuilder(ImplementationType type,
{}
HfstTransducer FstBuilder::get_model(void)
-{
- HfstTransducer model(model_fst,type);
+{
+ HfstTransducer model(model_fst,type);
model.set_name(name);
return model;
}
@@ -24,10 +24,10 @@ void FstBuilder::add_transition
HfstState target_state,
const std::string &symbol,
float weight)
-{
+{
model_fst.add_transition(initial_state,
HfstBasicTransition
- (target_state,symbol,symbol,weight));
+ (target_state,symbol,symbol,weight));
}
void FstBuilder::add_transition
@@ -39,7 +39,7 @@ void FstBuilder::add_transition
{
model_fst.add_transition(initial_state,
HfstBasicTransition
- (target_state,isymbol,osymbol,weight));
+ (target_state,isymbol,osymbol,weight));
}
bool FstBuilder::has_target(HfstState s, const std::string &symbol)
@@ -47,9 +47,9 @@ bool FstBuilder::has_target(HfstState s, const std::string &symbol)
if (s == START_STATE)
{ return start_state_targets.count(symbol) != 0; }
- const HfstBasicTransducer::HfstTransitions &transitions = model_fst[s];
+ const hfst::implementations::HfstBasicTransitions &transitions = model_fst[s];
- for (HfstBasicTransducer::HfstTransitions::const_iterator it =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator it =
transitions.begin();
it != transitions.end();
++it)
@@ -76,41 +76,41 @@ HfstState FstBuilder::get_target(HfstState s,
new_transition_required = true;
}
else
- {
- target = start_state_targets[symbol];
+ {
+ target = start_state_targets[symbol];
new_transition_required = false;
- }
+ }
}
else
{
- const HfstBasicTransducer::HfstTransitions &transitions = model_fst[s];
+ const hfst::implementations::HfstBasicTransitions &transitions = model_fst[s];
- for (HfstBasicTransducer::HfstTransitions::const_iterator it =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator it =
transitions.begin();
it != transitions.end();
++it)
{
if (it->get_input_symbol() == symbol)
{
- target = it->get_target_state();
+ target = it->get_target_state();
break;
}
}
if (target == NO_STATE)
- {
- target = model_fst.add_state();
+ {
+ target = model_fst.add_state();
new_transition_required = true;
}
else
{ new_transition_required = false; }
}
- model_fst.set_final_weight(target,default_final_weight);
+ model_fst.set_final_weight(target,default_final_weight);
return target;
}
-FstBuilder::StringVector
+FstBuilder::StringVector
FstBuilder::split_at_tabs(const std::string &str)
{
StringVector result;
@@ -124,7 +124,7 @@ FstBuilder::split_at_tabs(const std::string &str)
end_pos = str.find('\t',beg_pos);
std::string token = str.substr(beg_pos,end_pos - beg_pos);
result.push_back(token);
- }
+ }
while (end_pos != std::string::npos);
return result;
@@ -143,7 +143,7 @@ HfstState FstBuilder::add_sequence
{
target_state = get_target(initial_state,
*it,
- new_transition_required);
+ new_transition_required);
if (new_transition_required)
{ add_transition(initial_state,target_state,*it,0.0); }
diff --git a/tools/src/hfst-tagger/src/build_model_src/FstBuilder.h b/tools/src/hfst-tagger/src/build_model_src/FstBuilder.h
index bc60d1c..62152ec 100644
--- a/tools/src/hfst-tagger/src/build_model_src/FstBuilder.h
+++ b/tools/src/hfst-tagger/src/build_model_src/FstBuilder.h
@@ -80,7 +80,7 @@ class FstBuilder
const std::string &symbol);
HfstState get_target(HfstState s,
- const std::string &symbol,
+ const std::string &symbol,
bool &new_transition_required);
StringVector split_at_tabs(const std::string &str);
virtual void add_sequence(const WeightedStringVector &c,
diff --git a/tools/src/hfst-tagger/src/build_model_src/LexicalModelBuilder.cc b/tools/src/hfst-tagger/src/build_model_src/LexicalModelBuilder.cc
index 450820d..27edd6b 100644
--- a/tools/src/hfst-tagger/src/build_model_src/LexicalModelBuilder.cc
+++ b/tools/src/hfst-tagger/src/build_model_src/LexicalModelBuilder.cc
@@ -40,7 +40,7 @@ void LexicalModelBuilder::store(HfstOutputStream &out, const std::string &file_n
ModelBuilder::verbose_printf("Transforming the lexical model to "
"optimized format.\n");
- lexical_model.convert(HFST_OLW_TYPE);
+ lexical_model.convert(HFST_OLW_TYPE);
ModelBuilder::verbose_printf(std::string("Storing the lexical model in") +
file_name + ".\n");
diff --git a/tools/src/hfst-tagger/src/build_model_src/Makefile b/tools/src/hfst-tagger/src/build_model_src/Makefile
index 8b2a682..aeb11aa 100644
--- a/tools/src/hfst-tagger/src/build_model_src/Makefile
+++ b/tools/src/hfst-tagger/src/build_model_src/Makefile
@@ -65,7 +65,7 @@ TaggerBuilder:string_handling.cc WeightedStringVectorCollection.cc \
WeightedStringVectorCollection.cc
g++ -g -O3 -c -o FstBuilder.o FstBuilder.cc
g++ -g -O3 -c -o ModelBuilder.o ModelBuilder.cc
- g++ -g -O3 -c -o LexicalModelBuilder.o LexicalModelBuilder.cc
+ g++ -g -O3 -c -o LexicalModelBuilder.o LexicalModelBuilder.cc
g++ -g -O3 -c -o SequenceModelBuilder.o SequenceModelBuilder.cc
g++ -Wall -Wextra -g -O3 -c -o TaggerBuilder.o TaggerBuilder.cc
g++ -g -O3 -DMAIN_TEST -o TaggerBuilder ModelBuilder.o \
diff --git a/tools/src/hfst-tagger/src/build_model_src/ModelBuilder.cc b/tools/src/hfst-tagger/src/build_model_src/ModelBuilder.cc
index df537f6..af1cf1d 100644
--- a/tools/src/hfst-tagger/src/build_model_src/ModelBuilder.cc
+++ b/tools/src/hfst-tagger/src/build_model_src/ModelBuilder.cc
@@ -23,13 +23,13 @@ ModelBuilder::ModelBuilder
model_weights.get_name()),
depth(0)
{
- for (WeightedStringVectorCollection::const_iterator it =
+ for (WeightedStringVectorCollection::const_iterator it =
model_weights.begin();
it != model_weights.end();
++it)
- {
+ {
add_sequence(*it,
- (model_weights.is_lexical_model ? LEXICAL : SEQUENCE));
+ (model_weights.is_lexical_model ? LEXICAL : SEQUENCE));
}
if (not model_weights.is_lexical_model)
@@ -54,13 +54,13 @@ void ModelBuilder::add_sequence(const WeightedStringVector &v,
HfstState target_state =
FstBuilder::add_sequence(symbol_sequence.begin(),
- symbol_sequence.end() - 1);
+ symbol_sequence.end() - 1);
std::string last_output_symbol = symbol_sequence.back();
- std::string last_input_symbol =
+ std::string last_input_symbol =
((string_type == LEXICAL) ? internal_epsilon : symbol_sequence.back());
- HfstState final_state =
+ HfstState final_state =
((string_type == LEXICAL) ? model_fst.add_state() : START_STATE);
FstBuilder::add_transition(target_state,
@@ -77,9 +77,9 @@ void ModelBuilder::complete_model
StateVector::const_iterator default_state_vector_it,
float penalty_weight)
{
- const HfstBasicTransducer::HfstTransitions &transitions = model_fst[s];
+ const hfst::implementations::HfstBasicTransitions &transitions = model_fst[s];
- for (HfstBasicTransducer::HfstTransitions::const_iterator it =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator it =
transitions.begin();
it != transitions.end();
++it)
@@ -109,8 +109,8 @@ void ModelBuilder::complete_model(float penalty_weight)
StateVector default_states;
for (size_t i = 1; i < depth; ++i)
- {
- default_states.push_back(model_fst.add_state());
+ {
+ default_states.push_back(model_fst.add_state());
}
default_states.push_back(START_STATE);
@@ -121,7 +121,7 @@ void ModelBuilder::complete_model(float penalty_weight)
default_states[i+1],
DEFAULT_SYMBOL,
DEFAULT_SYMBOL,
- (default_states[i+1] == START_STATE ?
+ (default_states[i+1] == START_STATE ?
penalty_weight :
0.0));
}
diff --git a/tools/src/hfst-tagger/src/build_model_src/SequenceModelBuilder.cc b/tools/src/hfst-tagger/src/build_model_src/SequenceModelBuilder.cc
index cca50f7..878b660 100644
--- a/tools/src/hfst-tagger/src/build_model_src/SequenceModelBuilder.cc
+++ b/tools/src/hfst-tagger/src/build_model_src/SequenceModelBuilder.cc
@@ -19,8 +19,8 @@ SequenceModelBuilder::SequenceModelBuilder(std::istream &in)
model_vector.push_back(model_builder.get_model());
}
catch (const EmptyFile &e)
- {
- static_cast<void>(e);
+ {
+ static_cast<void>(e);
break;
}
}
@@ -28,7 +28,7 @@ SequenceModelBuilder::SequenceModelBuilder(std::istream &in)
void SequenceModelBuilder::store(const std::string &file_name)
{
- ModelBuilder::verbose_printf(std::string("Storing sequence models in ") +
+ ModelBuilder::verbose_printf(std::string("Storing sequence models in ") +
file_name + ".\n");
HfstOutputStream out(file_name,TROPICAL_OPENFST_TYPE);
diff --git a/tools/src/hfst-tagger/src/build_model_src/TaggerBuilder.cc b/tools/src/hfst-tagger/src/build_model_src/TaggerBuilder.cc
index c7e1562..8e477fc 100644
--- a/tools/src/hfst-tagger/src/build_model_src/TaggerBuilder.cc
+++ b/tools/src/hfst-tagger/src/build_model_src/TaggerBuilder.cc
@@ -8,8 +8,8 @@ TaggerBuilder::TaggerBuilder(void):
in(NULL),
lexical_model_builder(std::cin),
sequence_model_builder(std::cin)
-{
- delete in;
+{
+ delete in;
}
TaggerBuilder::TaggerBuilder(const std::string &file_name):
diff --git a/tools/src/hfst-tagger/src/build_model_src/TaggerBuilder.h b/tools/src/hfst-tagger/src/build_model_src/TaggerBuilder.h
index 2761a79..1610155 100644
--- a/tools/src/hfst-tagger/src/build_model_src/TaggerBuilder.h
+++ b/tools/src/hfst-tagger/src/build_model_src/TaggerBuilder.h
@@ -40,7 +40,7 @@ class TaggerBuilder
TaggerBuilder(const std::string &filename);
// Store the lexical model of this tagger in file_name_prefix + ".lex" and
- // store the sequence model in file_name_prefix + ".seq".
+ // store the sequence model in file_name_prefix + ".seq".
void store(const std::string &file_name_prefix);
// Store the lexical model and sequence model of this tagger in STDOUT.
diff --git a/tools/src/hfst-tagger/src/build_model_src/WeightedStringVectorCollection.cc b/tools/src/hfst-tagger/src/build_model_src/WeightedStringVectorCollection.cc
index 10fed0b..3beb246 100644
--- a/tools/src/hfst-tagger/src/build_model_src/WeightedStringVectorCollection.cc
+++ b/tools/src/hfst-tagger/src/build_model_src/WeightedStringVectorCollection.cc
@@ -20,8 +20,8 @@ WeightedStringVectorCollection::WeightedStringVectorCollection
this->name = read_model_start_tag(getline(in));
if (not is_lexical_model)
- {
- this->penalty_weight = read_model_penalty_weight(getline(in));
+ {
+ this->penalty_weight = read_model_penalty_weight(getline(in));
}
@@ -37,8 +37,8 @@ WeightedStringVectorCollection::WeightedStringVectorCollection
try
{
- WeightedStringVector entry =
- (this->is_lexical_model ?
+ WeightedStringVector entry =
+ (this->is_lexical_model ?
tokenize_lexicon_line(line,tokenizer) :
tokenize_grammar_line(line,tokenizer));
@@ -61,8 +61,8 @@ std::string WeightedStringVectorCollection::getline(std::istream &in)
std::string line;
if (in.peek() == EOF)
- {
- throw EmptyFile();
+ {
+ throw EmptyFile();
}
std::getline(in,line);
@@ -71,19 +71,19 @@ std::string WeightedStringVectorCollection::getline(std::istream &in)
return line;
}
-WeightedStringVectorCollection::const_iterator
+WeightedStringVectorCollection::const_iterator
WeightedStringVectorCollection::begin(void) const
{ return this->data.begin(); }
-WeightedStringVectorCollection::const_iterator
+WeightedStringVectorCollection::const_iterator
WeightedStringVectorCollection::end(void) const
{ return this->data.end(); }
-WeightedStringVectorCollection::iterator
+WeightedStringVectorCollection::iterator
WeightedStringVectorCollection::begin(void)
{ return this->data.begin(); }
-WeightedStringVectorCollection::iterator
+WeightedStringVectorCollection::iterator
WeightedStringVectorCollection::end(void)
{ return this->data.end(); }
@@ -111,7 +111,7 @@ int main(void)
std::string line7 = "DT\tNN\t1.5\n";
std::string line8 = "STOP SEQ MODEL";
- std::string file = line1 + line2 + line3 + line4 + line5 + line6 + line7 +
+ std::string file = line1 + line2 + line3 + line4 + line5 + line6 + line7 +
line8;
std::istringstream in(file);
@@ -151,8 +151,8 @@ int main(void)
assert(it == c2.end());
try
- {
- static_cast<void>(WeightedStringVectorCollection(in,SEQUENCE));
+ {
+ static_cast<void>(WeightedStringVectorCollection(in,SEQUENCE));
assert(false);
}
catch (...)
diff --git a/tools/src/hfst-tagger/src/build_model_src/WeightedStringVectorCollection.h b/tools/src/hfst-tagger/src/build_model_src/WeightedStringVectorCollection.h
index 7c2dda6..7a02c86 100644
--- a/tools/src/hfst-tagger/src/build_model_src/WeightedStringVectorCollection.h
+++ b/tools/src/hfst-tagger/src/build_model_src/WeightedStringVectorCollection.h
@@ -16,17 +16,17 @@
//! stored as string vectors with an associated weight.
//!
//! Two kinds of training data can be read:
-//!
-//! 1. Lexical training data lines have the form "WORDFORM\tTAG\tWEIGHT"
+//!
+//! 1. Lexical training data lines have the form "WORDFORM\tTAG\tWEIGHT"
//! e.g. "cat\tNN\t1.0", which is stored as (("c", "a", "t", "NN"),1.0).
//!
-//! 2. Sequence training data lines have the form
+//! 2. Sequence training data lines have the form
//! "WORDFORM1\tTAG1\tWORDFORM2\tTAG2\t...\tWORDFORMM\tTAGM\tWEIGHT", e.g.
-//! "the\tDT\tcar\tNN\t1.0" which is stored as
+//! "the\tDT\tcar\tNN\t1.0" which is stored as
//! (("the","DT","car","NN"),1.0).
//!
//! Lexical training data lines have to have 3 fields. Sequence training data
-//! lines have to have at least three fields and an odd number of fields.
+//! lines have to have at least three fields and an odd number of fields.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/tools/src/hfst-tagger/src/build_model_src/string_handling.cc b/tools/src/hfst-tagger/src/build_model_src/string_handling.cc
index 123ae52..71c0bb8 100644
--- a/tools/src/hfst-tagger/src/build_model_src/string_handling.cc
+++ b/tools/src/hfst-tagger/src/build_model_src/string_handling.cc
@@ -63,7 +63,7 @@ StringVector split(const std::string &line,char separator)
WeightedStringVector tokenize_lexicon_line(const std::string &line,
HfstTokenizer &tokenizer)
{
- StringVector split_line = split(line,'\t');
+ StringVector split_line = split(line,'\t');
if (split_line.size() != 2 and split_line.size() != 3)
{ throw InvalidLine(__FILE__ " " STR(__LINE__)); }
@@ -74,8 +74,8 @@ WeightedStringVector tokenize_lexicon_line(const std::string &line,
std::string weight_str = split_line[1];
if (split_line.size() == 3)
- {
- tag = split_line[1];
+ {
+ tag = split_line[1];
weight_str = split_line[2];
}
@@ -89,12 +89,13 @@ WeightedStringVector tokenize_lexicon_line(const std::string &line,
if (weight == -1)
{ throw InvalidFloat(weight_str); }
- return WeightedStringVector(entry, weight);
+ return WeightedStringVector(entry, weight);
}
WeightedStringVector tokenize_grammar_line(const std::string &line,
HfstTokenizer &tokenizer)
{
+ (void)tokenizer;
StringVector split_line = split(line, '\t');
if ((split_line.size() % 2) != 1 or split_line.size() < 3)
@@ -107,18 +108,18 @@ WeightedStringVector tokenize_grammar_line(const std::string &line,
float weight = get_non_negative_float(split_line.back());
if (weight == -1)
- {
- throw InvalidFloat(weight_str);
+ {
+ throw InvalidFloat(weight_str);
}
- return WeightedStringVector(entry, weight);
+ return WeightedStringVector(entry, weight);
}
std::string read_model_start_tag(const std::string &line)
{
if (line.find("START ") != 0)
{
- throw InvalidLine(__FILE__ " " STR(__LINE__));
+ throw InvalidLine(__FILE__ " " STR(__LINE__));
}
std::string model_name = line.substr(6);
@@ -165,43 +166,43 @@ int main(void)
/* Empty lines, strings without weights should throw an exception. */
try
- {
- static_cast<void>(tokenize_lexicon_line("", tokenizer));
+ {
+ static_cast<void>(tokenize_lexicon_line("", tokenizer));
fail_test(__FILE__,__LINE__);
}
catch (const InvalidLine &e)
{ static_cast<void>(e); }
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
try
{
- static_cast<void>(tokenize_lexicon_line("oof\tNN", tokenizer));
+ static_cast<void>(tokenize_lexicon_line("oof\tNN", tokenizer));
fail_test(__FILE__,__LINE__);
}
catch (const InvalidFloat &e)
{ static_cast<void>(e); }
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
/* Strings with erroneous weights should throw an exception. */
try
- {
- static_cast<void>(tokenize_lexicon_line("oof\tNN\tw", tokenizer));
+ {
+ static_cast<void>(tokenize_lexicon_line("oof\tNN\tw", tokenizer));
fail_test(__FILE__,__LINE__);
}
catch (const InvalidFloat &e)
{ static_cast<void>(e); }
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
/* Basic case. Should work. */
@@ -212,24 +213,24 @@ int main(void)
if (v.string_vector.size() != 4)
{ throw InvalidLine(__FILE__ " " STR(__LINE__)); }
- if (v.string_vector[0] != "o" or v.string_vector[1] != "o" or
+ if (v.string_vector[0] != "o" or v.string_vector[1] != "o" or
v.string_vector[2] != "f" or v.weight != 0.5)
{ throw InvalidLine(__FILE__ " " STR(__LINE__)); }
}
catch (const InvalidLine &e)
- {
+ {
std::cerr << "From line: " << e.representation << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
catch (const InvalidFloat &e)
- {
+ {
std::cerr << "From line: " << e.representation << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
/* Empty word. Should work. */
@@ -245,90 +246,90 @@ int main(void)
}
catch (const InvalidLine &e)
{
- std::cerr << "From line: " << e.representation << std::endl;
- fail_test(__FILE__,__LINE__);
+ std::cerr << "From line: " << e.representation << std::endl;
+ fail_test(__FILE__,__LINE__);
}
catch (const InvalidFloat &e)
- {
+ {
std::cerr << "From line: " << e.representation << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
/* utf-8 chars. Should work. */
try
{
- WeightedStringVector v = tokenize_lexicon_line("älä\tNN\t0.5",
+ WeightedStringVector v = tokenize_lexicon_line("älä\tNN\t0.5",
tokenizer);
if (v.string_vector.size() != 4)
{ throw InvalidLine(__FILE__ " " STR(__LINE__)); }
- if (v.string_vector[0] != "ä" and v.string_vector[1] != "l" and
+ if (v.string_vector[0] != "ä" and v.string_vector[1] != "l" and
v.string_vector[2] != "ä" and v.weight != 0.5)
{ throw InvalidLine(__FILE__ " " STR(__LINE__)); }
}
catch (const InvalidLine &e)
- {
+ {
std::cerr << "From line: " << e.representation << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
catch (const InvalidFloat &e)
- {
+ {
std::cerr << "From line: " << e.representation << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
/* Empty lines, lines with only weight and lines with an odd number
of tags should throw an excpetion. */
try
- {
- static_cast<void>(tokenize_lexicon_line("", tokenizer));
+ {
+ static_cast<void>(tokenize_lexicon_line("", tokenizer));
fail_test(__FILE__,__LINE__);
}
catch (const InvalidLine &e)
{ static_cast<void>(e); }
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
try
{
- static_cast<void>(tokenize_lexicon_line("0.5", tokenizer));
+ static_cast<void>(tokenize_lexicon_line("0.5", tokenizer));
fail_test(__FILE__,__LINE__);
}
catch (const InvalidLine &e)
{ static_cast<void>(e); }
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
try
{
- static_cast<void>(tokenize_lexicon_line("NN\tVBZ\tDT\t0.5", tokenizer));
+ static_cast<void>(tokenize_lexicon_line("NN\tVBZ\tDT\t0.5", tokenizer));
fail_test(__FILE__,__LINE__);
}
catch (const InvalidLine &e)
{ static_cast<void>(e); }
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
/* Basic case. Should work. */
@@ -340,24 +341,24 @@ int main(void)
if (v.string_vector.size() != 2)
{ throw InvalidLine(__FILE__ " " STR(__LINE__)); }
- if (v.string_vector[0] != "DT" or v.string_vector[1] != "NN" or
+ if (v.string_vector[0] != "DT" or v.string_vector[1] != "NN" or
v.weight != 0.5)
{ throw InvalidLine(__FILE__ " " STR(__LINE__)); }
}
catch (const InvalidLine &e)
- {
+ {
std::cerr << "From line: " << e.representation << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
catch (const InvalidFloat &e)
- {
+ {
std::cerr << "From line: " << e.representation << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
// Empty model name fails.
@@ -368,9 +369,9 @@ int main(void)
catch (const InvalidLine &e)
{ static_cast<void>(e); }
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
// Model name without START tag fails.
@@ -381,9 +382,9 @@ int main(void)
catch (const InvalidLine &e)
{ static_cast<void>(e); }
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
// Model name with tab fails.
@@ -394,9 +395,9 @@ int main(void)
catch (const InvalidLine &e)
{ static_cast<void>(e); }
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
// Regular model name works.
@@ -408,19 +409,19 @@ int main(void)
{ throw InvalidLine(__FILE__ " " STR(__LINE__)); }
}
catch (const InvalidLine &e)
- {
+ {
std::cerr << "From line: " << e.representation << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
catch (const InvalidFloat &e)
- {
+ {
std::cerr << "From line: " << e.representation << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
// Empty model name fails.
@@ -431,9 +432,9 @@ int main(void)
catch (const InvalidLine &e)
{ static_cast<void>(e); }
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
// Model name without STOP tag fails.
@@ -444,9 +445,9 @@ int main(void)
catch (const InvalidLine &e)
{ static_cast<void>(e); }
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
// Model name with tab fails.
@@ -457,9 +458,9 @@ int main(void)
catch (const InvalidLine &e)
{ static_cast<void>(e); }
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
// Regular model name works.
@@ -471,19 +472,19 @@ int main(void)
{ throw InvalidLine(__FILE__ " " STR(__LINE__)); }
}
catch (const InvalidLine &e)
- {
+ {
std::cerr << "From line: " << e.representation << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
catch (const InvalidFloat &e)
- {
+ {
std::cerr << "From line: " << e.representation << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
catch (const std::exception &e)
- {
+ {
std::cerr << "Caught: " << e.what() << std::endl;
- fail_test(__FILE__,__LINE__);
+ fail_test(__FILE__,__LINE__);
}
// TEST PASSED
diff --git a/tools/src/hfst-tagger/src/build_model_src/string_handling.h b/tools/src/hfst-tagger/src/build_model_src/string_handling.h
index 90c43b7..27b1235 100644
--- a/tools/src/hfst-tagger/src/build_model_src/string_handling.h
+++ b/tools/src/hfst-tagger/src/build_model_src/string_handling.h
@@ -13,7 +13,7 @@ using hfst::HfstTokenizer;
using hfst::StringVector;
// STR1 changes its argument into a string, i.e. surrounds it with
-// quotation marks. Since arguments of # aren't evaluated, two macro
+// quotation marks. Since arguments of # aren't evaluated, two macro
// levels are needed, to convert the result of a macro such as __LINE__
// into a string.
#define STR1(a) # a
@@ -39,7 +39,7 @@ struct InvalidFloat
std::string representation;
};
-// Hold a vector of strings (a future path in a transducer) and it's
+// Hold a vector of strings (a future path in a transducer) and it's
// associated weight, which is a positive float.
struct WeightedStringVector
{
@@ -69,7 +69,7 @@ std::string read_model_start_tag(const std::string &line);
// FOO". FOO can't contain tabs.
std::string read_model_stop_tag(const std::string &line);
-// Read the penalty weight X as a float from string "PENALTY_WEIGHT=X".
+// Read the penalty weight X as a float from string "PENALTY_WEIGHT=X".
float read_model_penalty_weight(const std::string &line);
#endif // HEADER_STRING_HANDLING_H
diff --git a/tools/src/hfst-tagger/src/build_model_src/tagger_aux.py b/tools/src/hfst-tagger/src/build_model_src/tagger_aux.py
index 25b15e1..90a10b2 100644
--- a/tools/src/hfst-tagger/src/build_model_src/tagger_aux.py
+++ b/tools/src/hfst-tagger/src/build_model_src/tagger_aux.py
@@ -44,7 +44,7 @@ class SequenceSimplifier:
for line in pattern:
if len(line) != 2 or (not line[0] in [0,1]) or \
- (not line[1] in [0,1]):
+ (not line[1] in [0,1]):
raise InvalidPattern()
self.pattern = pattern
@@ -76,7 +76,7 @@ class Pattern:
self.numerator = numerator
self.denominator = denominator
self.name = name
- self.order = len(numerator.pattern) - 1
+ self.order = len(numerator.pattern) - 1
# Read statistics patterns from file config_file_name.
def read_config_file(config_file_name):
@@ -158,7 +158,7 @@ def verbose_print(message, is_verbose):
# Return the utf-8 string str reversed. utf-8 symbols are not
# internally reversed.
-def reverse(str):
+def reverse(str):
return str.decode("utf-8")[::-1]
# Return a map for counting pairs e.g. word form and tag pairs.
@@ -199,12 +199,12 @@ def get_penalty_map(object_counter, total_count):
return penalty_map
-def print_conditional_penalties(pair_counter, object_counter,
+def print_conditional_penalties(pair_counter, object_counter,
appended_suffix,
invert_fields,
print_maximum_penalty):
- penalty_map = get_conditional_penalty_map(pair_counter, object_counter)
+ penalty_map = get_conditional_penalty_map(pair_counter, object_counter)
if print_maximum_penalty:
max_object_count = 0
@@ -231,14 +231,14 @@ def print_conditional_penalties(pair_counter, object_counter,
if not invert_fields:
- s = string.join([p[0] + appended_suffix,
- p[1],
+ s = string.join([p[0] + appended_suffix,
+ p[1],
str(penalty)],"\t")
print s.encode("utf-8")
else:
- s = string.join([p[1] + appended_suffix,
- p[0],
+ s = string.join([p[1] + appended_suffix,
+ p[0],
str(penalty)],"\t")
print s.encode("utf-8")
else:
@@ -262,7 +262,7 @@ def print_conditional_penalties(pair_counter, object_counter,
def print_penalties(object_counter, total_count, appended_suffix):
- penalty_map = get_penalty_map(object_counter, total_count)
+ penalty_map = get_penalty_map(object_counter, total_count)
for object, penalty in penalty_map.iteritems():
diff --git a/tools/src/hfst-tagger/src/hfst-build-tagger.cc b/tools/src/hfst-tagger/src/hfst-build-tagger.cc
index c7e84c0..5568f6a 100644
--- a/tools/src/hfst-tagger/src/hfst-build-tagger.cc
+++ b/tools/src/hfst-tagger/src/hfst-build-tagger.cc
@@ -63,8 +63,8 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "wDnf:",
long_options, &option_index);
if (-1 == c)
@@ -99,7 +99,7 @@ parse_options(int argc, char** argv)
case 'o':
outfilename = hfst_strdup(optarg);
outfile = hfst_fopen(outfilename, "w");
- if (outfile == stdout)
+ if (outfile == stdout)
{
free(outfilename);
outfilename = hfst_strdup("<stdout>");
@@ -134,7 +134,7 @@ int process_input_data(std::string output_file_prefix)
return EXIT_SUCCESS;
}
-int main( int argc, char **argv )
+int main( int argc, char **argv )
{
#ifdef WINDOWS
_setmode(1, _O_BINARY);
diff --git a/tools/src/hfst-tagger/src/hfst-open-input-file-for-tagger.cc b/tools/src/hfst-tagger/src/hfst-open-input-file-for-tagger.cc
index 5eed2fc..8bc4e4a 100644
--- a/tools/src/hfst-tagger/src/hfst-open-input-file-for-tagger.cc
+++ b/tools/src/hfst-tagger/src/hfst-open-input-file-for-tagger.cc
@@ -72,8 +72,8 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "wDnf:",
long_options, &option_index);
if (-1 == c)
@@ -104,7 +104,7 @@ void echo_stream_to_stdout(std::istream &in)
{ std::cout << line << std::endl; }
}
-int main( int argc, char **argv )
+int main( int argc, char **argv )
{
// This tool is used in a pipeline of commands, so help and version messages
// cannot be written to stdout.
@@ -123,15 +123,15 @@ int main( int argc, char **argv )
{ output_file_str += ".{lex,seq}"; }
verbose_printf("Reading training data from %s.\n"
- "Writing tagger to %s\n",
+ "Writing tagger to %s\n",
input_file_str.c_str(), output_file_str.c_str());
if (std::string(inputfilename) == "<stdin>")
{ echo_stream_to_stdout(std::cin); }
else
- {
+ {
std::ifstream in(inputfilename);
- echo_stream_to_stdout(in);
+ echo_stream_to_stdout(in);
}
free(inputfilename);
diff --git a/tools/src/hfst-tagger/src/hfst-reweight-tagger.cc b/tools/src/hfst-tagger/src/hfst-reweight-tagger.cc
index 150005c..38798a2 100644
--- a/tools/src/hfst-tagger/src/hfst-reweight-tagger.cc
+++ b/tools/src/hfst-tagger/src/hfst-reweight-tagger.cc
@@ -80,8 +80,8 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "wDnf:",
long_options, &option_index);
if (-1 == c)
@@ -181,7 +181,7 @@ float get_coefficient(const std::string &line)
}
FloatVector parse_coefficients(void)
-{
+{
std::ifstream coeff_in("hfst_tagger_config");
if (coeff_in.peek() == EOF)
diff --git a/tools/src/hfst-tagger/src/hfst-tag.cc b/tools/src/hfst-tagger/src/hfst-tag.cc
index 1c03d6c..b85379f 100644
--- a/tools/src/hfst-tagger/src/hfst-tag.cc
+++ b/tools/src/hfst-tagger/src/hfst-tag.cc
@@ -74,8 +74,8 @@ parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "wDnf:",
long_options, &option_index);
if (-1 == c)
@@ -160,7 +160,7 @@ void print_analysis(const WeightedStringPairVector &res,std::ostream * out)
if (std::string(it->first) == "||")
{ continue; }
- (out == NULL ? std::cout : *out) << it->first << "\t" << it->second
+ (out == NULL ? std::cout : *out) << it->first << "\t" << it->second
<< std::endl;
}
diff --git a/tools/src/hfst-tagger/src/hfst-train-tagger-system.cc b/tools/src/hfst-tagger/src/hfst-train-tagger-system.cc
index ea11c0c..58bcd89 100644
--- a/tools/src/hfst-tagger/src/hfst-train-tagger-system.cc
+++ b/tools/src/hfst-tagger/src/hfst-train-tagger-system.cc
@@ -1,5 +1,5 @@
//
-// An alternative for the tool hfst-train-tagger.sh.
+// An alternative for the tool hfst-train-tagger.sh.
// Uses system calls instead of shell commands.
//
@@ -16,12 +16,12 @@ int main(int argc, char* argv[]) {
arguments.append(std::string(" "));
}
- (void)sprintf(command,
+ (void)sprintf(command,
"hfst-open-input-file-for-tagger.exe %s | "
"hfst_tagger_compute_data_statistics.py %s | "
- "hfst-build-tagger %s",
- arguments.c_str(),
- arguments.c_str(),
+ "hfst-build-tagger %s",
+ arguments.c_str(),
+ arguments.c_str(),
arguments.c_str());
int retval = system(command);
diff --git a/tools/src/hfst-tagger/src/hfst_tagger_compute_data_statistics.py b/tools/src/hfst-tagger/src/hfst_tagger_compute_data_statistics.py
index f83b3ce..2d89d2d 100755
--- a/tools/src/hfst-tagger/src/hfst_tagger_compute_data_statistics.py
+++ b/tools/src/hfst-tagger/src/hfst_tagger_compute_data_statistics.py
@@ -54,7 +54,7 @@ tagger_aux.verbose_print("Parsing config file hfst_tagger_config.", verbose)
statistics_patterns = tagger_aux.read_config_file("hfst_tagger_config")
-counters = [ [tagger_aux.get_pair_counter(), tagger_aux.get_object_counter()]
+counters = [ [tagger_aux.get_pair_counter(), tagger_aux.get_object_counter()]
for pattern in statistics_patterns ]
# Read from stdin.
@@ -191,7 +191,7 @@ print "STOP P(WORD_FORM | TAG)"
tagger_aux.verbose_print("P(LOWER_SUFFIX_AND_TAG | LOWER_SUFFIX)",verbose)
print "START P(LOWER_SUFFIX_AND_TAG | LOWER_SUFFIX)"
-tagger_aux.print_conditional_penalties(lower_suffix_and_tag_count_map,
+tagger_aux.print_conditional_penalties(lower_suffix_and_tag_count_map,
lower_suffix_count_map,
"<lower_suffix_and_tag>",
True,
@@ -202,7 +202,7 @@ print "STOP P(LOWER_SUFFIX_AND_TAG | LOWER_SUFFIX)"
tagger_aux.verbose_print("P(LOWER_SUFFIX)",verbose)
print "START P(LOWER_SUFFIX)"
-tagger_aux.print_penalties(lower_suffix_count_map,
+tagger_aux.print_penalties(lower_suffix_count_map,
number_of_lower_suffixes,
"<lower_suffix>")
print "STOP P(LOWER_SUFFIX)"
@@ -211,7 +211,7 @@ print "STOP P(LOWER_SUFFIX)"
tagger_aux.verbose_print("P(LOWER_TAG)",verbose)
print "START P(LOWER_TAG)"
number_of_lower_tags = number_of_lower_suffixes
-tagger_aux.print_penalties(lower_tag_count_map,
+tagger_aux.print_penalties(lower_tag_count_map,
number_of_lower_suffixes,
"<lower_tag>")
print "STOP P(LOWER_TAG)"
@@ -219,7 +219,7 @@ print "STOP P(LOWER_TAG)"
# Compute and display the penalties for suffix and tag combinations.
print "START P(UPPER_SUFFIX_AND_TAG | UPPER_SUFFIX)"
-tagger_aux.print_conditional_penalties(upper_suffix_and_tag_count_map,
+tagger_aux.print_conditional_penalties(upper_suffix_and_tag_count_map,
upper_suffix_count_map,
"<upper_suffix_and_tag>",
True,
@@ -230,7 +230,7 @@ print "STOP P(UPPER_SUFFIX_AND_TAG | UPPER_SUFFIX)"
tagger_aux.verbose_print("P(UPPER_SUFFIX)",verbose)
print "START P(UPPER_SUFFIX)"
-tagger_aux.print_penalties(upper_suffix_count_map,
+tagger_aux.print_penalties(upper_suffix_count_map,
number_of_upper_suffixes,
"<upper_suffix>")
print "STOP P(UPPER_SUFFIX)"
@@ -239,7 +239,7 @@ print "STOP P(UPPER_SUFFIX)"
tagger_aux.verbose_print("P(UPPER_TAG)",verbose)
print "START P(UPPER_TAG)"
number_of_tags = number_of_upper_suffixes
-tagger_aux.print_penalties(upper_tag_count_map,
+tagger_aux.print_penalties(upper_tag_count_map,
number_of_upper_suffixes,
"<upper_tag>")
print "STOP P(UPPER_TAG)"
@@ -279,7 +279,7 @@ for i in range(len(statistics_patterns)):
print start_tag
- tagger_aux.print_conditional_penalties(counters[i][0],
+ tagger_aux.print_conditional_penalties(counters[i][0],
counters[i][1],
"",
False,
diff --git a/tools/src/hfst-tagger/src/use_model_src/AcyclicAutomaton.cc b/tools/src/hfst-tagger/src/use_model_src/AcyclicAutomaton.cc
index 53b6bed..0d450e8 100644
--- a/tools/src/hfst-tagger/src/use_model_src/AcyclicAutomaton.cc
+++ b/tools/src/hfst-tagger/src/use_model_src/AcyclicAutomaton.cc
@@ -116,10 +116,10 @@ void AcyclicAutomaton::clear(void)
for (AcyclicStateVector::iterator jt = it->begin();
jt != it->end();
++jt)
- {
+ {
if (not jt->is_initialized())
{ break; }
- jt->clear();
+ jt->clear();
}
}
@@ -187,7 +187,7 @@ int main(void)
acyclic_automaton1.finalize_position();
- WeightedSymbolVector weight_2_0_path_1_1 =
+ WeightedSymbolVector weight_2_0_path_1_1 =
acyclic_automaton1.get_best_path();
assert(weight_2_0_path_1_1.second.size() == 2);
diff --git a/tools/src/hfst-tagger/src/use_model_src/AcyclicAutomaton.h b/tools/src/hfst-tagger/src/use_model_src/AcyclicAutomaton.h
index 46b9de8..15fed28 100644
--- a/tools/src/hfst-tagger/src/use_model_src/AcyclicAutomaton.h
+++ b/tools/src/hfst-tagger/src/use_model_src/AcyclicAutomaton.h
@@ -18,7 +18,7 @@ class AcyclicState
public:
AcyclicState(size_t id);
void add_transition(size_t source_id,Symbol symbol,Weight weight);
- Symbol get_symbol(void) const;
+ Symbol get_symbol(void) const;
Weight get_weight(void) const;
void set_weight(Weight w);
size_t get_source(void) const;
diff --git a/tools/src/hfst-tagger/src/use_model_src/DelayedSequenceModelComponent.cc b/tools/src/hfst-tagger/src/use_model_src/DelayedSequenceModelComponent.cc
index f5d2815..6ef2ecf 100644
--- a/tools/src/hfst-tagger/src/use_model_src/DelayedSequenceModelComponent.cc
+++ b/tools/src/hfst-tagger/src/use_model_src/DelayedSequenceModelComponent.cc
@@ -23,14 +23,14 @@ TransitionData DelayedSequenceModelComponent::get_transition
TransitionData transition_data;
if (state < static_cast<int>(delay))
- {
+ {
transition_data.weight = 0.0;
transition_data.target = state + 1;
}
else
{
- TransitionData internal_transition_data =
- model.get_transition(state - delay, symbol);
+ TransitionData internal_transition_data =
+ model.get_transition(state - delay, symbol);
transition_data.weight = internal_transition_data.weight;
transition_data.target = internal_transition_data.target + delay;
@@ -91,11 +91,11 @@ int main(void)
get_transition(2,SequenceModelComponent::get_symbol("a")).
weight == static_cast<float>(0.0));
- SequenceModelComponent * smc =
+ SequenceModelComponent * smc =
&delayed_sequence_model_component_0;
assert(smc->get_final_weight(0) == infinity);
- smc =
+ smc =
&delayed_sequence_model_component_2;
assert(smc->get_final_weight(0) == 0.0);
}
diff --git a/tools/src/hfst-tagger/src/use_model_src/DelayedSequenceModelComponent.h b/tools/src/hfst-tagger/src/use_model_src/DelayedSequenceModelComponent.h
index f5ab7f8..fe1bbdd 100644
--- a/tools/src/hfst-tagger/src/use_model_src/DelayedSequenceModelComponent.h
+++ b/tools/src/hfst-tagger/src/use_model_src/DelayedSequenceModelComponent.h
@@ -14,7 +14,7 @@ class DelayedSequenceModelComponent : public SequenceModelComponent
size_t delay);
Weight get_final_weight(State state) const;
- TransitionData get_transition(State state, Symbol symbol);
+ TransitionData get_transition(State state, Symbol symbol);
protected:
SequenceModelComponent &model;
diff --git a/tools/src/hfst-tagger/src/use_model_src/NewLexicalModel.cc b/tools/src/hfst-tagger/src/use_model_src/NewLexicalModel.cc
index 50a5d46..b2c1678 100644
--- a/tools/src/hfst-tagger/src/use_model_src/NewLexicalModel.cc
+++ b/tools/src/hfst-tagger/src/use_model_src/NewLexicalModel.cc
@@ -4,6 +4,7 @@
#include <algorithm>
#include <queue>
+#include <math.h>
using hfst::StringVector;
using hfst::HfstOneLevelPaths;
@@ -21,10 +22,10 @@ NewLexicalModel::NewLexicalModel(const std::string &filename, std::istream * par
void NewLexicalModel::initialize_tag_probabilities(void)
{
- HfstOneLevelPaths * upper_case_tag_paths =
+ HfstOneLevelPaths * upper_case_tag_paths =
lexical_model.lookup("<upper_suffix_and_tag>");
- HfstOneLevelPaths * lower_case_tag_paths =
+ HfstOneLevelPaths * lower_case_tag_paths =
lexical_model.lookup("<lower_suffix_and_tag>");
if (upper_case_tag_paths->empty())
@@ -40,8 +41,8 @@ void NewLexicalModel::initialize_tag_probabilities(void)
float penalty = it->first;
if (it->second.empty())
- {
- lexical_model_is_broken = true;
+ {
+ lexical_model_is_broken = true;
break;
}
@@ -60,8 +61,8 @@ void NewLexicalModel::initialize_tag_probabilities(void)
float penalty = it->first;
if (it->second.empty())
- {
- lexical_model_is_broken = true;
+ {
+ lexical_model_is_broken = true;
break;
}
@@ -95,23 +96,23 @@ const WeightedStringVector &NewLexicalModel::get_first_word_analysis
delete paths;
if (word_not_in_lexicon)
- {
+ {
std::string lower_case_word = to_lower_case(word);
lexicon_o_o_v_words.insert(word);
if (not is_lexicon_oov(lower_case_word))
- {
+ {
o_o_v_words.insert(word);
- return operator[](lower_case_word);
+ return operator[](lower_case_word);
}
else
{ return operator[](word); }
}
else
- { return operator[](word); }
+ { return operator[](word); }
}
-const WeightedStringVector &NewLexicalModel::operator[]
+const WeightedStringVector &NewLexicalModel::operator[]
(const std::string &word)
{
if (lexical_model_is_broken)
@@ -120,9 +121,9 @@ const WeightedStringVector &NewLexicalModel::operator[]
AnalysisCache::const_iterator word_it = analysis_cache.find(word);
if (word_it == analysis_cache.end())
- {
+ {
const WeightedStringVector & v = cache_word_analyses(word);
- return v;
+ return v;
}
return word_it->second;
@@ -186,7 +187,7 @@ const WeightedStringVector &NewLexicalModel::cache_word_analyses
{
if (word[0] >= 65 and word[0] <= 90)
{ upper_case = true; }
- else if (word.find("Å") == 0 or word.find("Ä") == 0 or
+ else if (word.find("Å") == 0 or word.find("Ä") == 0 or
word.find("Ö") == 0)
{ upper_case = true; }
}
@@ -209,17 +210,17 @@ const WeightedStringVector &NewLexicalModel::cache_word_analyses
guesses = split(line);
}
if (line.find("+?") == std::string::npos)
- {
- assert((guesses.size() + 1) % 2 == 0);
- return paradigm_guess(word,guesses);
+ {
+ assert((guesses.size() + 1) % 2 == 0);
+ return paradigm_guess(word,guesses);
}
}
o_o_v_words.insert(word);
return guess(word,upper_case,rev_tokenized_word);
}
else
- {
- return cache_analyses(word,paths,analysis_cache);
+ {
+ return cache_analyses(word,paths,analysis_cache);
}
}
@@ -242,30 +243,30 @@ std::string NewLexicalModel::to_string(StringVector::const_iterator begin,
}
const WeightedStringVector &NewLexicalModel::guess
-(const std::string &word,bool upper_case,
+(const std::string &word,bool upper_case,
const StringVector &rev_tokenized_word)
{
WeightedStringVector &word_analyses = analysis_cache[word];
word_analyses = upper_case ? upper_case_empty_suffix_null : lower_case_empty_suffix_null;
- size_t suffix_length = std::min(rev_tokenized_word.size(),
+ size_t suffix_length = std::min(rev_tokenized_word.size(),
static_cast<size_t>(100));
size_t last_found_suffix_length = suffix_length;
for (size_t i = suffix_length; i > 0; --i)
{
- const WeightedStringVector &suffix_analyses =
+ const WeightedStringVector &suffix_analyses =
get_suffix_analyses(word,
rev_tokenized_word.begin(),
rev_tokenized_word.begin() + i,
upper_case);
if (suffix_analyses.empty())
- {
+ {
--last_found_suffix_length;
- continue;
+ continue;
}
merge_analyses(suffix_analyses,word_analyses,upper_case);
@@ -293,7 +294,7 @@ bool NewLexicalModel::is_oov(const std::string &word)
{ return o_o_v_words.find(word) != o_o_v_words.end(); }
bool NewLexicalModel::is_lexicon_oov(const std::string &word)
-{
+{
StringVector rev_tokenized_word = tokenizer.tokenize_one_level(word);
std::reverse(rev_tokenized_word.begin(),rev_tokenized_word.end());
@@ -366,8 +367,8 @@ const WeightedStringVector &NewLexicalModel::cache_suffix_analyses
}
const WeightedStringVector &NewLexicalModel::cache_analyses
-(const std::string &word,
- HfstOneLevelPaths * paths,
+(const std::string &word,
+ HfstOneLevelPaths * paths,
AnalysisCache &cache,
bool convert_to_probabilities)
{
@@ -383,9 +384,9 @@ const WeightedStringVector &NewLexicalModel::cache_analyses
it != paths->end();
++it)
{
- // Each entry in paths, is a weight analysis pair. The last entry in
+ // Each entry in paths, is a weight analysis pair. The last entry in
// the analysis is the tag.
- word_analyses[i].first =
+ word_analyses[i].first =
convert_to_probabilities ? get_prob(it->first) : it->first;
word_analyses[i].second = *(it->second.rbegin());
++i;
@@ -410,15 +411,15 @@ void NewLexicalModel::merge_analyses
for (WeightedStringVector::reverse_iterator it = word_analyses.rbegin();
it != word_analyses.rend();
++it)
- {
- const IdWeightPair &id_weight_pair =
+ {
+ const IdWeightPair &id_weight_pair =
suffix_tag_probability_hash[it->second];
// Check this.
- it->first =
- ((upper_case ? upper_case_interpolator : lower_case_interpolator) * it->first +
+ it->first =
+ ((upper_case ? upper_case_interpolator : lower_case_interpolator) * it->first +
(id == id_weight_pair.first ? id_weight_pair.second : 0.0)) /
- (1 + (upper_case ? upper_case_interpolator : lower_case_interpolator));
+ (1 + (upper_case ? upper_case_interpolator : lower_case_interpolator));
}
}
@@ -429,14 +430,14 @@ float NewLexicalModel::get_suffix_penalty(const std::string &suffix,
if (it == (upper_case ? upper_case_suffix_penalties.end() : lower_case_suffix_penalties.end()))
{
- HfstOneLevelPaths * suffix_penalty_paths =
+ HfstOneLevelPaths * suffix_penalty_paths =
lexical_model.lookup(suffix + (upper_case ? "<upper_suffix>" : "<lower_suffix>"));
if (suffix_penalty_paths->empty())
- {
+ {
std::cerr << suffix << std::endl;
delete suffix_penalty_paths;
- throw InvalidKey();
+ throw InvalidKey();
}
float penalty = suffix_penalty_paths->begin()->first;
@@ -451,10 +452,10 @@ float NewLexicalModel::get_suffix_penalty(const std::string &suffix,
float NewLexicalModel::get_tag_penalty(const std::string &tag,bool upper_case)
{
if (upper_case)
- {
+ {
if (upper_case_tag_penalties.find(tag) == upper_case_tag_penalties.end())
{
- throw InvalidKey();
+ throw InvalidKey();
}
}
else if(lower_case_tag_penalties.find(tag) == lower_case_tag_penalties.end())
@@ -484,7 +485,7 @@ void preserve_n_best_analyses(WeightedStringVector &word_analyses,size_t number_
word_analyses.clear();
for (size_t i = 0; i < number_of_guesses; ++i)
- {
+ {
if (analysis_queue.empty())
{ break; }
word_analyses.push_back(analysis_queue.top());
@@ -503,9 +504,9 @@ void NewLexicalModel::bayesian_invert(WeightedStringVector &word_analyses,
for (WeightedStringVector::iterator it = word_analyses.begin();
it != word_analyses.end();
++it)
- {
- it->first =
- suffix_penalty + get_penalty(it->first) - get_tag_penalty(it->second,upper_case);
+ {
+ it->first =
+ suffix_penalty + get_penalty(it->first) - get_tag_penalty(it->second,upper_case);
it->first *= 0.2;
}
}
diff --git a/tools/src/hfst-tagger/src/use_model_src/NewLexicalModel.h b/tools/src/hfst-tagger/src/use_model_src/NewLexicalModel.h
index 397e589..be28795 100644
--- a/tools/src/hfst-tagger/src/use_model_src/NewLexicalModel.h
+++ b/tools/src/hfst-tagger/src/use_model_src/NewLexicalModel.h
@@ -8,7 +8,7 @@
#include "HfstTransducer.h"
#ifdef USE_TR1_UNORDERED_MAP
- #include <tr1/unordered_map>
+ #include <tr1/unordered_map>
#else
#include <unordered_map>
#endif
@@ -34,7 +34,7 @@ using hfst::StringVector;
class NewLexicalModel
{
public:
- NewLexicalModel(const std::string &filename,
+ NewLexicalModel(const std::string &filename,
std::istream * paradigm_guess_stream = 0);
const WeightedStringVector &operator[] (const std::string &word);
const WeightedStringVector &get_first_word_analysis(const std::string &word);
@@ -42,16 +42,16 @@ class NewLexicalModel
bool is_lexicon_oov(const std::string &word);
private:
#ifdef USE_TR1_UNORDERED_MAP
- typedef std::tr1::unordered_map<std::string,WeightedStringVector>
+ typedef std::tr1::unordered_map<std::string,WeightedStringVector>
AnalysisCache;
#else
- typedef std::unordered_map<std::string,WeightedStringVector>
+ typedef std::unordered_map<std::string,WeightedStringVector>
AnalysisCache;
#endif
- AnalysisCache analysis_cache;
- AnalysisCache upper_case_suffix_cache;
- AnalysisCache lower_case_suffix_cache;
+ AnalysisCache analysis_cache;
+ AnalysisCache upper_case_suffix_cache;
+ AnalysisCache lower_case_suffix_cache;
HfstTokenizer tokenizer;
StringWeightMap tag_probability_hash;
ProbabilityMap upper_case_suffix_penalties;
diff --git a/tools/src/hfst-tagger/src/use_model_src/SentenceTagger.cc b/tools/src/hfst-tagger/src/use_model_src/SentenceTagger.cc
index 8fc747c..15ea3d5 100644
--- a/tools/src/hfst-tagger/src/use_model_src/SentenceTagger.cc
+++ b/tools/src/hfst-tagger/src/use_model_src/SentenceTagger.cc
@@ -10,8 +10,8 @@ SentenceTagger::SentenceTagger(const std::string &lexical_model_filename,
const std::string &sequence_model_filename,
std::istream * paradigm_guess_stream):
lexical_model(lexical_model_filename,paradigm_guess_stream)
-{
- init_sequence_model(sequence_model_filename);
+{
+ init_sequence_model(sequence_model_filename);
buffer_analyses.push_back(WeightedString(0.0,BUFFER));
}
@@ -63,8 +63,8 @@ WeightedStringPairVector SentenceTagger::operator[]
const std::string &word = *it;
const WeightedStringVector &unigram_analyses =
- (first ?
- lexical_model.get_first_word_analysis(word) :
+ (first ?
+ lexical_model.get_first_word_analysis(word) :
lexical_model[word]);
sentence_transducer.add_word(word,unigram_analyses);
@@ -101,7 +101,7 @@ SequenceModelComponent * SentenceTagger::read_model
for (size_t i = 1; i < n; ++i)
{
- DelayedSequenceModelComponent * delayed_model =
+ DelayedSequenceModelComponent * delayed_model =
new DelayedSequenceModelComponent(*model,2*i);
models.push_back(delayed_model);
@@ -126,7 +126,7 @@ size_t SentenceTagger::get_model_order(const HfstTransducer &model_fst)
assert(name.find("SEQUENCE-MODEL:N=") == 0);
#endif // OPTIMIZE_DANGEROUSLY
- std::string order_string =
+ std::string order_string =
name.substr(std::string("SEQUENCE-MODEL:N=").size());
std::istringstream in(order_string);
@@ -149,7 +149,7 @@ int main(int argc, char * argv[])
if (argc != 3 and argc != 4)
{
- std::cerr << "Usage: cat input | " << argv[0]
+ std::cerr << "Usage: cat input | " << argv[0]
<< " lexical_model sequence_model [ paradigm_guessed_input ]"
<< std::endl;
exit(1);
@@ -180,10 +180,10 @@ int main(int argc, char * argv[])
while (end_it != absolute_end_it)
{
if (*end_it == "||")
- {
+ {
++end_it;
++end_it;
- break;
+ break;
}
++end_it;
}
@@ -199,8 +199,8 @@ int main(int argc, char * argv[])
for (StringPairVector::const_iterator it = analysis.second.begin();
it != analysis.second.end() - 2;
++it)
- {
- std::cout << /*(sentence_tagger.is_lexicon_oov(it->first) ? (sentence_tagger.is_oov(it->first) ? "*U*" : "*P*" ) + it->first :*/ it->first /*)*/ << "\t" << it->second << std::endl;
+ {
+ std::cout << /*(sentence_tagger.is_lexicon_oov(it->first) ? (sentence_tagger.is_oov(it->first) ? "*U*" : "*P*" ) + it->first :*/ it->first /*)*/ << "\t" << it->second << std::endl;
}
start_it = end_it - 2;
diff --git a/tools/src/hfst-tagger/src/use_model_src/SentenceTransducer.cc b/tools/src/hfst-tagger/src/use_model_src/SentenceTransducer.cc
index d86ad75..962e1bf 100644
--- a/tools/src/hfst-tagger/src/use_model_src/SentenceTransducer.cc
+++ b/tools/src/hfst-tagger/src/use_model_src/SentenceTransducer.cc
@@ -17,13 +17,13 @@ void SentenceTransducer::add_word
State tag_target = add_state(std::numeric_limits<float>::infinity());
// Add the word transition.
- Symbol2TransitionDataMap &word_symbol_to_transition =
+ Symbol2TransitionDataMap &word_symbol_to_transition =
transition_map[current_max_state];
Symbol word_symbol = SequenceModelComponent::get_symbol(word);
add_transition_to_map(word_symbol_to_transition,word_symbol,0,word_target);
// Add analysis transitions.
- Symbol2TransitionDataMap &tag_symbol_to_transition =
+ Symbol2TransitionDataMap &tag_symbol_to_transition =
transition_map[word_target];
for (WeightedStringVector::const_iterator it = tags.begin();
it != tags.end();
@@ -92,47 +92,47 @@ int main(void)
assert(sentence_transducer.get_final_weight(4) == static_cast<float>(0));
assert(sentence_transducer.get_transition
- (0,SequenceModelComponent::get_symbol("a")).weight ==
+ (0,SequenceModelComponent::get_symbol("a")).weight ==
static_cast<float>(0));
assert(sentence_transducer.get_transition
- (0,SequenceModelComponent::get_symbol("a")).target == 1);
+ (0,SequenceModelComponent::get_symbol("a")).target == 1);
assert(sentence_transducer.get_transition
- (1,SequenceModelComponent::get_symbol("b")).weight ==
+ (1,SequenceModelComponent::get_symbol("b")).weight ==
static_cast<float>(1.0));
assert(sentence_transducer.get_transition
- (1,SequenceModelComponent::get_symbol("b")).target == 2);
+ (1,SequenceModelComponent::get_symbol("b")).target == 2);
assert(sentence_transducer.get_transition
- (1,SequenceModelComponent::get_symbol("c")).weight ==
+ (1,SequenceModelComponent::get_symbol("c")).weight ==
static_cast<float>(2.0));
assert(sentence_transducer.get_transition
- (1,SequenceModelComponent::get_symbol("c")).target == 2);
+ (1,SequenceModelComponent::get_symbol("c")).target == 2);
assert(sentence_transducer.get_transition
- (2,SequenceModelComponent::get_symbol("e")).weight ==
+ (2,SequenceModelComponent::get_symbol("e")).weight ==
static_cast<float>(0));
assert(sentence_transducer.get_transition
- (2,SequenceModelComponent::get_symbol("e")).target == 3);
+ (2,SequenceModelComponent::get_symbol("e")).target == 3);
assert(sentence_transducer.get_transition
- (3,SequenceModelComponent::get_symbol("f")).weight ==
+ (3,SequenceModelComponent::get_symbol("f")).weight ==
static_cast<float>(3.0));
assert(sentence_transducer.get_transition
- (3,SequenceModelComponent::get_symbol("f")).target == 4);
+ (3,SequenceModelComponent::get_symbol("f")).target == 4);
assert(sentence_transducer.get_transition
- (3,SequenceModelComponent::get_symbol("g")).weight ==
+ (3,SequenceModelComponent::get_symbol("g")).weight ==
static_cast<float>(4.0));
assert(sentence_transducer.get_transition
- (3,SequenceModelComponent::get_symbol("g")).target == 4);
+ (3,SequenceModelComponent::get_symbol("g")).target == 4);
const Symbol2TransitionDataMap &s = sentence_transducer[3];
diff --git a/tools/src/hfst-tagger/src/use_model_src/SequenceModelComponent.cc b/tools/src/hfst-tagger/src/use_model_src/SequenceModelComponent.cc
index f03bb92..703ad65 100644
--- a/tools/src/hfst-tagger/src/use_model_src/SequenceModelComponent.cc
+++ b/tools/src/hfst-tagger/src/use_model_src/SequenceModelComponent.cc
@@ -7,7 +7,7 @@
const State SequenceModelComponent::START_STATE = 0;
-typedef HfstBasicTransducer::HfstTransitionGraphAlphabet
+typedef HfstBasicTransducer::HfstAlphabet
HfstTransitionGraphAlphabet;
using hfst::internal_epsilon;
@@ -51,7 +51,7 @@ void SequenceModelComponent::add_symbols_from(const HfstBasicTransducer &fst)
void SequenceModelComponent::add_symbol(const std::string &string_symbol)
{
if (symbol_to_number_map.count(string_symbol) == 0)
- {
+ {
symbol_to_number_map[string_symbol] = number_to_symbol_map.size();
number_to_symbol_map.push_back(string_symbol);
}
@@ -90,15 +90,15 @@ void SequenceModelComponent::add_transitions_from
{
for (State s = 0; s <= static_cast<int>(fst.get_max_state()); ++s)
{
- const HfstBasicTransducer::HfstTransitions transitions = fst[s];
+ const hfst::implementations::HfstBasicTransitions transitions = fst[s];
Symbol2TransitionDataMap &symbol_to_transition = transition_map[s];
- for (HfstBasicTransducer::HfstTransitions::const_iterator it =
+ for (hfst::implementations::HfstBasicTransitions::const_iterator it =
transitions.begin();
it != transitions.end();
++it)
- {
+ {
Symbol symbol = symbol_to_number_map[it->get_input_symbol()];
add_transition_to_map(symbol_to_transition,
@@ -139,7 +139,7 @@ Weight SequenceModelComponent::get_final_weight(State state) const
return state_final_weight_map[state];
}
-TransitionData SequenceModelComponent::get_transition(State state,
+TransitionData SequenceModelComponent::get_transition(State state,
Symbol symbol)
{
#ifndef OPTIMIZE_DANGEROUSLY
@@ -147,14 +147,14 @@ TransitionData SequenceModelComponent::get_transition(State state,
{ throw InvalidState(); }
#endif // OPTIMIZE_DANGEROUSLY
- const Symbol2TransitionDataMap &symbol_to_transition =
- transition_map.at(state);
+ const Symbol2TransitionDataMap &symbol_to_transition =
+ transition_map.at(state);
if (symbol_to_transition.count(symbol) == 0)
- {
+ {
if (symbol_to_transition.count(DEFAULT) == 0)
- {
+ {
std::cerr << state << " " << symbol << std::endl;
- throw InvalidKey();
+ throw InvalidKey();
}
// Can't use the regular []-operator, since symbol_to_transition
@@ -218,77 +218,77 @@ int main(void)
// Check clockwise transitions.
assert(sequence_model_component.get_transition
- (0, SequenceModelComponent::get_symbol("a")).weight ==
+ (0, SequenceModelComponent::get_symbol("a")).weight ==
static_cast<float>(0.1));
assert(sequence_model_component.get_transition
- (0, SequenceModelComponent::get_symbol("a")).target ==
+ (0, SequenceModelComponent::get_symbol("a")).target ==
static_cast<float>(1));
assert(sequence_model_component.get_transition
- (1, SequenceModelComponent::get_symbol("a")).weight ==
+ (1, SequenceModelComponent::get_symbol("a")).weight ==
static_cast<float>(0.2));
assert(sequence_model_component.get_transition
- (1, SequenceModelComponent::get_symbol("a")).target ==
+ (1, SequenceModelComponent::get_symbol("a")).target ==
static_cast<float>(2));
assert(sequence_model_component.get_transition
- (2, SequenceModelComponent::get_symbol("a")).weight ==
+ (2, SequenceModelComponent::get_symbol("a")).weight ==
static_cast<float>(0.3));
assert(sequence_model_component.get_transition
- (2, SequenceModelComponent::get_symbol("a")).target ==
+ (2, SequenceModelComponent::get_symbol("a")).target ==
static_cast<float>(3));
assert(sequence_model_component.get_transition
- (3, SequenceModelComponent::get_symbol("a")).weight ==
+ (3, SequenceModelComponent::get_symbol("a")).weight ==
static_cast<float>(0.4));
assert(sequence_model_component.get_transition
- (3, SequenceModelComponent::get_symbol("a")).target ==
+ (3, SequenceModelComponent::get_symbol("a")).target ==
static_cast<float>(0));
// Check counter-clockwise transitions.
assert(sequence_model_component.get_transition
- (0, SequenceModelComponent::get_symbol("b")).weight ==
+ (0, SequenceModelComponent::get_symbol("b")).weight ==
static_cast<float>(1.1));
assert(sequence_model_component.get_transition
- (0, SequenceModelComponent::get_symbol("b")).target ==
+ (0, SequenceModelComponent::get_symbol("b")).target ==
static_cast<float>(3));
assert(sequence_model_component.get_transition
- (1, SequenceModelComponent::get_symbol("b")).weight ==
+ (1, SequenceModelComponent::get_symbol("b")).weight ==
static_cast<float>(1.4));
assert(sequence_model_component.get_transition
- (1, SequenceModelComponent::get_symbol("b")).target ==
+ (1, SequenceModelComponent::get_symbol("b")).target ==
static_cast<float>(0));
assert(sequence_model_component.get_transition
- (2, SequenceModelComponent::get_symbol("b")).weight ==
+ (2, SequenceModelComponent::get_symbol("b")).weight ==
static_cast<float>(1.3));
assert(sequence_model_component.get_transition
- (2, SequenceModelComponent::get_symbol("b")).target ==
+ (2, SequenceModelComponent::get_symbol("b")).target ==
static_cast<float>(1));
assert(sequence_model_component.get_transition
- (3, SequenceModelComponent::get_symbol("b")).weight ==
+ (3, SequenceModelComponent::get_symbol("b")).weight ==
static_cast<float>(1.2));
assert(sequence_model_component.get_transition
- (3, SequenceModelComponent::get_symbol("b")).target ==
+ (3, SequenceModelComponent::get_symbol("b")).target ==
static_cast<float>(2));
// Check default transition
assert(sequence_model_component.get_transition
- (1, SequenceModelComponent::get_symbol("c")).target ==
+ (1, SequenceModelComponent::get_symbol("c")).target ==
static_cast<float>(2));
assert(sequence_model_component.get_transition
- (1, SequenceModelComponent::get_symbol("c")).weight ==
- static_cast<float>(10.0));
+ (1, SequenceModelComponent::get_symbol("c")).weight ==
+ static_cast<float>(10.0));
}
#endif // MAIN_TEST
diff --git a/tools/src/hfst-tagger/src/use_model_src/SequenceModelComponent.h b/tools/src/hfst-tagger/src/use_model_src/SequenceModelComponent.h
index f6cc246..f15e1c0 100644
--- a/tools/src/hfst-tagger/src/use_model_src/SequenceModelComponent.h
+++ b/tools/src/hfst-tagger/src/use_model_src/SequenceModelComponent.h
@@ -23,7 +23,7 @@ class SequenceModelComponent
static const State START_STATE;
virtual Weight get_final_weight(State state) const;
- virtual TransitionData get_transition(State state, Symbol symbol);
+ virtual TransitionData get_transition(State state, Symbol symbol);
static Symbol get_symbol(const std::string &string_symbol);
static std::string get_string_symbol(Symbol symbol);
diff --git a/tools/src/hfst-tagger/src/use_model_src/SequenceModelComponentPair.cc b/tools/src/hfst-tagger/src/use_model_src/SequenceModelComponentPair.cc
index 625bca7..cebe4c7 100644
--- a/tools/src/hfst-tagger/src/use_model_src/SequenceModelComponentPair.cc
+++ b/tools/src/hfst-tagger/src/use_model_src/SequenceModelComponentPair.cc
@@ -2,7 +2,7 @@
#ifndef MAIN_TEST
-size_t
+size_t
SequenceModelComponentPair::StatePair2StateMap::count(const StatePair &p)
{
if (static_cast<int>(state_pair_to_state_map_.size()) <= p.first)
@@ -10,17 +10,17 @@ SequenceModelComponentPair::StatePair2StateMap::count(const StatePair &p)
if (static_cast<int>(state_pair_to_state_map_[p.first].size()) <= p.second)
{ return 0; }
- return state_pair_to_state_map_[p.first][p.second] == -1 ? 0 : 1;
+ return state_pair_to_state_map_[p.first][p.second] == -1 ? 0 : 1;
}
State &
SequenceModelComponentPair::StatePair2StateMap::operator[](const StatePair &p)
{
while (static_cast<int>(state_pair_to_state_map_.size()) <= p.first)
- {
+ {
state_pair_to_state_map_.insert(state_pair_to_state_map_.end(),
state_pair_to_state_map_.size() + 1,
- StateVector());
+ StateVector());
}
StateVector &state_vector = state_pair_to_state_map_[p.first];
@@ -47,7 +47,7 @@ void SequenceModelComponentPair::clear(void)
state_pair_to_state_map.clear();
state_to_state_pair_map.clear();
- (void)get_state(StatePair(0,0));
+ (void)get_state(StatePair(0,0));
}
SequenceModelComponentPair::SequenceModelComponentPair
@@ -67,9 +67,9 @@ Weight SequenceModelComponentPair::get_final_weight(State state) const
StatePair state_pair = state_to_state_pair_map[state];
- return
- component1.get_final_weight(state_pair.first) +
- component2.get_final_weight(state_pair.second);
+ return
+ component1.get_final_weight(state_pair.first) +
+ component2.get_final_weight(state_pair.second);
}
TransitionData SequenceModelComponentPair::get_transition
@@ -82,10 +82,10 @@ TransitionData SequenceModelComponentPair::get_transition
StatePair state_pair = state_to_state_pair_map[state];
- TransitionData transition_data1 =
+ TransitionData transition_data1 =
component1.get_transition(state_pair.first,symbol);
- TransitionData transition_data2 =
+ TransitionData transition_data2 =
component2.get_transition(state_pair.second,symbol);
StatePair target_pair(transition_data1.target, transition_data2.target);
@@ -102,7 +102,7 @@ TransitionData SequenceModelComponentPair::get_transition
State SequenceModelComponentPair::get_state(const StatePair &state_pair)
{
if (state_pair_to_state_map.count(state_pair) == 0)
- {
+ {
state_pair_to_state_map[state_pair] = state_to_state_pair_map.size();
state_to_state_pair_map.push_back(state_pair);
diff --git a/tools/src/hfst-tagger/src/use_model_src/SequenceModelComponentPair.h b/tools/src/hfst-tagger/src/use_model_src/SequenceModelComponentPair.h
index 4c2e325..8934e7d 100644
--- a/tools/src/hfst-tagger/src/use_model_src/SequenceModelComponentPair.h
+++ b/tools/src/hfst-tagger/src/use_model_src/SequenceModelComponentPair.h
@@ -17,7 +17,7 @@ class SequenceModelComponentPair : public SequenceModelComponent
SequenceModelComponent &component2);
Weight get_final_weight(State state) const;
- TransitionData get_transition(State state, Symbol symbol);
+ TransitionData get_transition(State state, Symbol symbol);
virtual void clear(void);
diff --git a/tools/src/hfst-tagger/src/use_model_src/SequenceTagger.cc b/tools/src/hfst-tagger/src/use_model_src/SequenceTagger.cc
index 982cd7a..a9b7739 100644
--- a/tools/src/hfst-tagger/src/use_model_src/SequenceTagger.cc
+++ b/tools/src/hfst-tagger/src/use_model_src/SequenceTagger.cc
@@ -14,8 +14,8 @@ State sentence_state = 0;
SequenceTagger::SequenceTagger
(SequenceModelComponent &sequence_model_component):
sequence_model_component(sequence_model_component),
- states(0),
- sentences(0)
+ // states(0),
+ sentences(0)
{}
SequenceTagger::~SequenceTagger(void)
@@ -64,7 +64,7 @@ WeightedStringPairVector SequenceTagger::operator[]
inline void SequenceTagger::process_configuration
(const SentenceTransducer &sentence_transducer,
- const StatePair &configuration,
+ const StatePair &configuration,
State result_state,
AcyclicAutomaton &result)
{
@@ -74,12 +74,12 @@ inline void SequenceTagger::process_configuration
// Get the sentence transducer transitions for state sentence_state.
- const Symbol2TransitionDataMap &sentence_transitions =
+ const Symbol2TransitionDataMap &sentence_transitions =
sentence_transducer[sentence_state];
// Iterate throught the sentence transitions, get corresponding model
// transitions and construct the composition of the transitions in result.
- for (Symbol2TransitionDataMap::const_iterator it =
+ for (Symbol2TransitionDataMap::const_iterator it =
sentence_transitions.begin();
it != sentence_transitions.end();
++it)
@@ -89,14 +89,14 @@ inline void SequenceTagger::process_configuration
Weight sentence_weight = it->second.weight;
State sentence_target = it->second.target;
- TransitionData model_transition =
+ TransitionData model_transition =
sequence_model_component.get_transition(model_state,symbol);
State model_target = model_transition.target;
Weight model_weight = model_transition.weight;
StatePair new_configuration(sentence_target,model_target);
- State result_target =
+ State result_target =
get_state(new_configuration,sentence_transducer,result);
if (result_target == -1)
@@ -113,12 +113,12 @@ inline State SequenceTagger::get_state
AcyclicAutomaton &result)
{
if (sentence_state != configuration.first)
- {
- result.finalize_position();
+ {
+ result.finalize_position();
sentence_state = configuration.first;
}
- StatePair2StateMap::iterator configuration_it =
+ StatePair2StateMap::iterator configuration_it =
state_pair_to_state_map.find(configuration);
if (configuration_it == state_pair_to_state_map.end())
@@ -130,7 +130,7 @@ inline State SequenceTagger::get_state
state_pair_to_state_map[configuration] = result_state;
state_to_state_pair_map.push_back(configuration);
- Weight sentence_final_weight =
+ Weight sentence_final_weight =
sentence_transducer.get_final_weight(configuration.first);
Weight model_final_weight =
@@ -145,7 +145,7 @@ inline State SequenceTagger::get_state
return configuration_it->second;
}
-WeightedStringPairVector
+WeightedStringPairVector
SequenceTagger::get_weighted_analysis(AcyclicAutomaton &result) const
{
WeightedStringPairVector tagging;
@@ -158,12 +158,12 @@ SequenceTagger::get_weighted_analysis(AcyclicAutomaton &result) const
// We know that each word should have exactly one tag, so there should
// be an even number or entries.
- // Go from format
+ // Go from format
// word1:word1 tag1:tag1 word2:word2 tag2:tag2 ...
// to format
// word1:tag1 word2:tag2 ...
for (size_t i = 0; i < path.second.size(); ++i)
- {
+ {
if (path.second[i] == -1)
{ continue; }
diff --git a/tools/src/hfst-tagger/src/use_model_src/SequenceTagger.h b/tools/src/hfst-tagger/src/use_model_src/SequenceTagger.h
index 62c01b7..bee545a 100644
--- a/tools/src/hfst-tagger/src/use_model_src/SequenceTagger.h
+++ b/tools/src/hfst-tagger/src/use_model_src/SequenceTagger.h
@@ -34,7 +34,7 @@ class SequenceTagger
Agenda agenda;
- size_t states;
+ //size_t states;
size_t sentences;
void process_configuration(const SentenceTransducer &sentence_transducer,
@@ -42,7 +42,7 @@ class SequenceTagger
State result_state,
AcyclicAutomaton &result);
- WeightedStringPairVector
+ WeightedStringPairVector
get_weighted_analysis(AcyclicAutomaton &result) const;
State get_state(const StatePair &configuration,
diff --git a/tools/src/hfst-tail.cc b/tools/src/hfst-tail.cc
index f4ed4ed..de59350 100644
--- a/tools/src/hfst-tail.cc
+++ b/tools/src/hfst-tail.cc
@@ -92,12 +92,12 @@ parse_options(int argc, char** argv)
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
{"n-last", required_argument, 0, 'n'},
- // add tool-specific options here
+ // add tool-specific options here
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "n:",
long_options, &option_index);
if (-1 == c)
@@ -205,7 +205,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-tokenize.cc b/tools/src/hfst-tokenize.cc
index 369b999..91a0075 100644
--- a/tools/src/hfst-tokenize.cc
+++ b/tools/src/hfst-tokenize.cc
@@ -21,8 +21,10 @@
#endif
+#include <iterator>
#include <iostream>
#include <fstream>
+#include <iterator>
#include <vector>
#include <map>
@@ -55,18 +57,29 @@ using hfst::HfstTransducer;
#include "inc/globals-common.h"
#include "inc/globals-unary.h"
-static bool blankline_separated = true;
+static bool superblanks = false; // Input is apertium-style superblanks (overrides blankline_separated)
+static bool blankline_separated = true; // Input is separated by blank lines (as opposed to single newlines)
+static bool keep_newlines = false;
static bool print_all = false;
static bool print_weights = false;
static bool tokenize_multichar = false;
+static string tag_separator = "+"; // + and # are hardcoded in cg-conv at least
+static string subreading_separator = "#";
+static string wtag = "W"; // TODO: cg-conv has an argument --wtag, allow changing here as well?
static double time_cutoff = 0.0;
+static int token_number = 1;
+static int max_weight_classes = std::numeric_limits<int>::max();
+static bool dedupe = false;
std::string tokenizer_filename;
static hfst::ImplementationType default_format = hfst::TROPICAL_OPENFST_TYPE;
enum OutputFormat {
tokenize,
+ space_separated,
xerox,
cg,
- finnpos
+ finnpos,
+ giellacg,
+ conllu
};
OutputFormat output_format = tokenize;
@@ -78,7 +91,7 @@ void
print_usage()
{
// c.f. http://www.gnu.org/prep/standards/standards.html#g_t_002d_002dhelp
- fprintf(message_out, "Usage: %s [--segment | --xerox | --cg] [OPTIONS...] RULESET\n"
+ fprintf(message_out, "Usage: %s [--segment | --xerox | --cg | --giella-cg] [OPTIONS...] RULESET\n"
"perform matching/lookup on text streams\n"
"\n", program_name);
print_common_program_options(message_out);
@@ -89,16 +102,25 @@ print_usage()
" -m, --tokenize-multichar Tokenize multicharacter symbols\n"
" (by default only one utf-8 character is tokenized at a time\n"
" regardless of what is present in the alphabet)\n"
- " -t, --time-cutoff=S Limit search after having used S seconds per input\n"
+ " -tS, --time-cutoff=S Limit search after having used S seconds per input\n"
+ " -lN, --weight-classes=N Output no more than N best weight classes\n"
+ " (where analyses with equal weight constitute a class\n"
+ " -u, --unique Remove duplicate analyses\n"
" -z, --segment Segmenting / tokenization mode (default)\n"
+ " -i, --space-separated Tokenization with one sentence per line, space-separated tokens\n"
" -x, --xerox Xerox output\n"
- " -c, --cg cg output\n"
+ " -c, --cg Constraint Grammar output\n"
+ " -S, --superblanks Ignore contents of unescaped [] (cf. apertium-destxt); flush on NUL\n"
+ " -g, --giella-cg CG format used in Giella infrastructe (implies -l2,\n"
+ " treats @PMATCH_INPUT_MARK@ as subreading separator,\n"
+ " expects tags to start or end with +, flush on NUL)\n"
+ " -C --conllu CoNLL-U format\n"
" -f, --finnpos FinnPos output\n");
- fprintf(message_out,
+ fprintf(message_out,
"Use standard streams for input and output (for now).\n"
"\n"
);
-
+
print_report_bugs();
fprintf(message_out, "\n");
print_more_info();
@@ -107,25 +129,41 @@ print_usage()
void print_no_output(std::string const & input, std::ostream & outstream)
{
- if (output_format == tokenize) {
+ if (output_format == tokenize || output_format == space_separated) {
outstream << input;
} else if (output_format == xerox) {
outstream << input << "\t" << input << "+?";
- } else if (output_format == cg) {
+ } else if (output_format == cg || output_format == giellacg) {
outstream << "\"<" << input << ">\"" << std::endl << "\t\"" << input << "\" ?";
}
// std::cerr << "from print_no_output\n";
outstream << "\n\n";
}
+void print_escaping_newlines(std::string const & str, std::ostream & outstream)
+{
+ // TODO: inline?
+ size_t i = 0, j = 0;
+ while((j = str.find("\n", i)) != std::string::npos) {
+ outstream << str.substr(i, j-i) << "\\n";
+ i = j+1;
+ }
+ outstream << str.substr(i, j-i);
+}
+
void print_nonmatching_sequence(std::string const & str, std::ostream & outstream)
{
- if (output_format == tokenize) {
+ if (output_format == tokenize || output_format == space_separated) {
outstream << str;
} else if (output_format == xerox) {
outstream << str << "\t" << str << "+?";
} else if (output_format == cg) {
outstream << "\"<" << str << ">\"" << std::endl << "\t\"" << str << "\" ?";
+ } else if (output_format == giellacg) {
+ outstream << ":";
+ print_escaping_newlines(str, outstream);
+ } else if (output_format == conllu) {
+ outstream << str;
} else if (output_format == finnpos) {
outstream << str << "\t_\t_\t_\t_";
}
@@ -210,7 +248,471 @@ hfst_ol::PmatchContainer make_naive_tokenizer(HfstTransducer & dictionary)
return retval;
}
-void print_location_vector(LocationVector const & locations, std::ostream & outstream)
+bool location_compare(const Location& lhs, const Location& rhs) {
+ if (lhs.weight == rhs.weight) {
+ if(lhs.tag == rhs.tag) {
+ if(lhs.start == rhs.start){
+ if(lhs.length == rhs.length) {
+ return lhs.output < rhs.output;
+ }
+ else {
+ return lhs.length < rhs.length;
+ }
+ }
+ else {
+ return lhs.start < rhs.start;
+ }
+ }
+ else {
+ return lhs.tag < rhs.tag;
+ }
+ }
+ else {
+ return lhs.weight < rhs.weight;
+ }
+};
+
+/**
+ * Keep only the max_weight_classes best weight classes
+ */
+const LocationVector dedupe_locations(LocationVector const & locations) {
+ if(!dedupe) {
+ return locations;
+ }
+ std::set<Location, bool(*)(const Location& lhs, const Location& rhs)> ls(&location_compare);
+ ls.insert(locations.begin(), locations.end());
+ LocationVector uniq;
+ std::copy(ls.begin(), ls.end(), std::back_inserter(uniq));
+ return uniq;
+}
+/**
+ * Keep only the max_weight_classes best weight classes
+ */
+const LocationVector keep_n_best_weight(LocationVector const & locations)
+{
+ if(locations.size() <= max_weight_classes) {
+ // We know we won't trim anything, no need to copy the vector:
+ return locations;
+ }
+ int classes_found = -1;
+ hfst_ol::Weight last_weight_class = 0.0;
+ LocationVector goodweight;
+ for (LocationVector::const_iterator it = locations.begin();
+ it != locations.end(); ++it) {
+ if(it->output.empty()) {
+ goodweight.push_back(*it);
+ continue;
+ }
+ hfst_ol::Weight current_weight = it->weight;
+ if (classes_found == -1) // we're just starting
+ {
+ classes_found = 1;
+ last_weight_class = current_weight;
+ }
+ else if (last_weight_class != current_weight)
+ {
+ last_weight_class = current_weight;
+ ++classes_found;
+ }
+ if (classes_found > max_weight_classes)
+ {
+ break;
+ }
+ else {
+ goodweight.push_back(*it);
+ }
+ }
+ return goodweight;
+}
+
+/**
+ * Return empty string if it wasn't a tag, otherwise the tag without the initial/final +
+ */
+const string as_cg_tag(const string & str) {
+ size_t len = str.size();
+ if(len > 1) {
+ if (str.at(0) == '+') {
+ return str.substr(1);
+ }
+ else if(str.at(len - 1) == '+') {
+ return str.substr(0, len - 1);
+ }
+ }
+ return "";
+}
+
+void print_cg_subreading(size_t const & indent,
+ hfst::StringVector::const_iterator & out_beg,
+ hfst::StringVector::const_iterator & out_end,
+ hfst_ol::Weight const & weight,
+ hfst::StringVector::const_iterator & in_beg,
+ hfst::StringVector::const_iterator & in_end,
+ std::ostream & outstream)
+{
+ outstream << string(indent, '\t');
+ bool in_lemma = false;
+ bool want_spc = false;
+ for(hfst::StringVector::const_iterator it = out_beg;
+ it != out_end; ++it) {
+ if(it->compare("@PMATCH_BACKTRACK@") == 0) {
+ continue;
+ }
+ const string & tag = as_cg_tag(*it);
+ if(in_lemma) {
+ if(tag.empty()) {
+ outstream << (*it);
+ }
+ else {
+ in_lemma = false;
+ outstream << "\" " << tag;
+ want_spc = true;
+ }
+ }
+ else {
+ if(want_spc) {
+ outstream << " ";
+ }
+ if(tag.empty()) {
+ in_lemma = true;
+ outstream << "\"" << (*it);
+ }
+ else {
+ outstream << tag;
+ want_spc = true;
+ }
+ }
+ }
+ if(in_lemma) {
+ outstream << "\"";
+ }
+
+ if (print_weights) {
+ outstream << " <" << wtag << ":" << weight << ">";
+ }
+ if (in_beg != in_end) {
+ std::ostringstream form;
+ std::copy(in_beg, in_end, std::ostream_iterator<string>(form, ""));
+ outstream << " \"<" << form.str() << ">\"";
+ }
+ outstream << std::endl;
+}
+
+typedef std::set<size_t> SplitPoints;
+
+pair<SplitPoints, size_t>
+print_reading_giellacg(const Location *loc,
+ size_t indent,
+ const bool always_wftag,
+ std::ostream & outstream)
+{
+ SplitPoints bt_its;
+ if(loc->output.empty()) {
+ return make_pair(bt_its, indent);
+ }
+ typedef hfst::StringVector::const_iterator PartIt;
+ PartIt
+ out_beg = loc->output_symbol_strings.begin(),
+ out_end = loc->output_symbol_strings.end(),
+ in_beg = loc->input_symbol_strings.begin(),
+ in_end = loc->input_symbol_strings.end();
+ if(!always_wftag) {
+ // don't print input wordform tag unless we've seen a subreading/input mark
+ in_beg = in_end;
+ }
+ size_t part = loc->input_parts.size();
+ while(true) {
+ string inpart;
+ bool sub_found = false;
+ size_t out_part = part > 0 ? loc->output_parts.at(part-1) : 0;
+ while(out_part > 0 && loc->output_symbol_strings.at(out_part-1) == "@PMATCH_BACKTRACK@") {
+ bt_its.insert(loc->input_parts.at(part-1));
+ --part;
+ out_part = part > 0 ? loc->output_parts.at(part-1) : 0;
+ }
+ for(PartIt it = out_end-1;
+ it > loc->output_symbol_strings.begin() + out_part;
+ --it) {
+ if(subreading_separator.compare(*it) == 0) {
+ // Found a sub-reading mark
+ out_beg = ++it;
+ sub_found = true;
+ break;
+ }
+ }
+ if(!sub_found) {
+ if(out_part > 0) {
+ // Found an input mark
+ out_beg = loc->output_symbol_strings.begin() + out_part;
+ in_beg = loc->input_symbol_strings.begin() + loc->input_parts.at(part-1);
+ --part;
+ }
+ else {
+ // No remaining sub-marks or input-marks to the left
+ out_beg = loc->output_symbol_strings.begin();
+ if(in_end != loc->input_symbol_strings.end()) {
+ // We've seen at least one input-mark, so we need to output the remaining input as well
+ in_beg = loc->input_symbol_strings.begin();
+ }
+ }
+ }
+ print_cg_subreading(indent,
+ out_beg,
+ out_end,
+ loc->weight,
+ in_beg,
+ in_end,
+ outstream);
+ if(out_beg == loc->output_symbol_strings.begin()) {
+ break;
+ }
+ else {
+ ++indent;
+ out_end = out_beg;
+ in_end = in_beg;
+ if(sub_found) {
+ --out_end; // skip the subreading separator symbol
+ }
+ }
+ }
+ if(!bt_its.empty()) {
+ bt_its.insert(0);
+ bt_its.insert(loc->input_symbol_strings.size());
+ }
+ return make_pair(bt_its, indent);
+}
+
+/**
+ * Treat syms as "characters" to concatenate and split at indices
+ * given by splitpoints to create a new string vector. Assumes
+ * splitpoints includes both ends of syms.
+ */
+const hfst::StringVector split_at(const hfst::StringVector & syms,
+ const SplitPoints * splitpoints)
+{
+ hfst::StringVector subs;
+ if(splitpoints->size() < 2) {
+ std::cerr << "split_at called with " << std::endl;
+ return subs;
+ }
+ // Loop to next-to-last
+ for(SplitPoints::const_iterator it = splitpoints->begin(); std::next(it) != splitpoints->end(); ++it) {
+ std::ostringstream ss;
+ // Copy the substring between this point and the next:
+ std::copy(syms.begin() + *(it),
+ syms.begin() + *(std::next(it)),
+ std::ostream_iterator<string>(ss, ""));
+ subs.push_back(ss.str());
+ }
+ return subs;
+}
+
+/*
+ * Look up form, filtering out empties and those that don't cover the
+ * full string.
+ */
+const LocationVector locate_fullmatch(hfst_ol::PmatchContainer & container,
+ string & form)
+{
+ LocationVectorVector sublocs = container.locate(form, time_cutoff);
+ LocationVector loc_filtered;
+ // TODO: Worth noticing about? Is this as safe as checking that input.length != form.length?
+ // if(sublocs.size() != 1) {
+ // std::cerr << "Warning: '" << form << "' only tokenisable by further splitting."<<std::endl;
+ // }
+ for(LocationVectorVector::const_iterator it = sublocs.begin();
+ it != sublocs.end(); ++it) {
+ if (it->empty()
+ || (it->size() == 1 && it->at(0).output.compare("@_NONMATCHING_@") == 0)
+ // keep only those that cover the full form
+ || it->at(0).input.length() != form.length()) {
+ continue;
+ }
+ LocationVector loc = keep_n_best_weight(dedupe_locations(*it));
+ for (LocationVector::const_iterator loc_it = loc.begin();
+ loc_it != loc.end(); ++loc_it) {
+ if(!loc_it->output.empty()
+ && loc_it->weight < std::numeric_limits<float>::max()) {
+ // TODO: why aren't the <W:inf> excluded earlier?
+ loc_filtered.push_back(*loc_it);
+ }
+ }
+ }
+ return loc_filtered;
+}
+
+void print_location_vector_giellacg(hfst_ol::PmatchContainer & container,
+ LocationVector const & locations,
+ std::ostream & outstream)
+{
+ outstream << "\"<" << locations.at(0).input << ">\"" << std::endl;
+ if(locations.size() == 1 && locations.at(0).output.empty()) {
+ // Treat empty analyses as unknown-but-tokenised:
+ outstream << "\t\"" << locations.at(0).input << "\" ?" << std::endl;
+ return;
+ }
+ // Output regular analyses first, making a note of backtracking points.
+ std::set<SplitPoints> backtrack;
+ for (LocationVector::const_iterator loc_it = locations.begin();
+ loc_it != locations.end(); ++loc_it) {
+ SplitPoints bt_points = print_reading_giellacg(&(*loc_it), 1, false, outstream).first;
+ if(!bt_points.empty()) {
+ backtrack.insert(bt_points);
+ }
+ }
+ if(backtrack.empty()) {
+ return;
+ }
+ // The rest of the function handles possible backtracking:
+ hfst::StringVector in_syms = locations.at(0).input_symbol_strings;
+
+ for(std::set<SplitPoints>::const_iterator bt_points = backtrack.begin();
+ bt_points != backtrack.end(); ++bt_points) {
+
+ // First, for every set of backtrack points, we split on every
+ // point in that N+1-sized set (the backtrack points include
+ // start/end points), and create an N-sized vector splitlocs of
+ // resulting analyses
+ LocationVectorVector splitlocs;
+ hfst::StringVector words = split_at(in_syms, &*(bt_points));
+ for(hfst::StringVector::const_iterator it = words.begin(); it != words.end(); ++it) {
+ // Trim left/right spaces:
+ const size_t first = it->find_first_not_of(' ');
+ const size_t last = it->find_last_not_of(' ') + 1;
+ string form = it->substr(first, last-first);
+ LocationVector loc = locate_fullmatch(container, form);
+ if(loc.size() == 0 && verbose) {
+ std::cerr << "Warning: The analysis of \"<" << locations.at(0).input << ">\" has backtracking around the substring \"<" << form << ">\", but that substring has no analyses." << std::endl;
+ // but push it anyway, since we want exactly one subvector per splitpoint
+ }
+ if(form.length() != it->length()) { // Ensure the spaces we ignored when looking up are output in the form:
+ vector<string> lspace = vector<string>(first, " ");
+ vector<string> rspace = vector<string>(it->length()-last, " ");
+ for(LocationVector::iterator lvit = loc.begin(); lvit != loc.end(); ++lvit) {
+ lvit->input = form;
+ vector<string>& syms = lvit->input_symbol_strings;
+ syms.insert(syms.begin(), lspace.begin(), lspace.end());
+ syms.insert(syms.end(), rspace.begin(), rspace.end());
+ for(vector<size_t>::iterator ip = lvit->input_parts.begin(); ip != lvit->input_parts.end(); ++ip) {
+ *ip += first;
+ }
+ }
+ }
+ splitlocs.push_back(loc);
+ }
+ if(splitlocs.empty()) {
+ continue;
+ }
+ // Second, we reorder splitlocs so we can output as a
+ // cohort of non-branching CG subreadings; first word as leaf
+ // nodes. This means that splitlocs = [[A,B],[C,D]] should
+ // end up as the sequence
+ // (C,0),(A,1),(C,0),(B,1),(D,0),(A,1),(D,0),(B,1)
+ // (where the number is the initial indentation).
+ size_t depth = 0;
+ const size_t bottom = splitlocs.size()-1;
+ vector<std::ostringstream> out(splitlocs.size());
+ vector<pair<LocationVector, size_t > > stack;
+ // In CG the *last* word is the least indented, so start from
+ // the end of splitlocs, indentation being 1 tab:
+ stack.push_back(make_pair(splitlocs.at(bottom),
+ 0));
+ while(!stack.empty() && !stack.back().first.empty()) {
+ LocationVector & locs = stack.back().first;
+ const Location loc = locs.back();
+ locs.pop_back();
+ const size_t indent = 1 + stack.back().second;
+ out.at(depth).clear();
+ out.at(depth).str(string());
+ // (ignore splitpoints of splitpoints)
+ const size_t new_indent = print_reading_giellacg(&loc, indent, true, out.at(depth)).second;
+ if(depth == bottom) {
+ for(vector<std::ostringstream>::const_iterator it = out.begin(); it != out.end(); ++it) {
+ outstream << it->str();
+ }
+ }
+ if(depth < bottom) {
+ ++depth;
+ if(depth > 0) {
+ stack.push_back(make_pair(splitlocs.at(bottom-depth),
+ new_indent));
+ }
+ }
+ else if(locs.empty()) {
+ depth--;
+ stack.pop_back();
+ }
+ }
+ }
+}
+
+// Omorfi-specific at this time
+std::string fetch_and_kill_between(std::string left, std::string right, std::string & analysis)
+{
+ size_t start = analysis.find(left);
+ size_t stop = analysis.find(right, start + 1);
+ if (start == std::string::npos || stop == std::string::npos) {
+ return "";
+ }
+ std::string retval = analysis.substr(start + left.size(), stop - start - left.size());
+ analysis.erase(start, stop - start + right.size());
+ return retval;
+}
+
+std::string fetch_and_kill_feats(std::string & analysis)
+{
+ std::string retval;
+ std::string tmp;
+ tmp = fetch_and_kill_between("[ANIMACY=", "]", analysis);
+ retval += (tmp != "" ? ("Animacy=" + tmp + "|") : "");
+ tmp = fetch_and_kill_between("[ASPECT=", "]", analysis);
+ retval += (tmp != "" ? ("Aspect=" + tmp + "|") : "");
+ tmp = fetch_and_kill_between("[CASE=", "]", analysis);
+ retval += (tmp != "" ? ("Case=" + tmp + "|") : "");
+ tmp = fetch_and_kill_between("[DEFINITE=", "]", analysis);
+ retval += (tmp != "" ? ("Definite=" + tmp + "|") : "");
+ tmp = fetch_and_kill_between("[CMP=", "]", analysis);
+ retval += (tmp != "" ? ("Degree=" + tmp + "|") : "");
+ tmp = fetch_and_kill_between("[GENDER=", "]", analysis);
+ retval += (tmp != "" ? ("Gender=" + tmp + "|") : "");
+ tmp = fetch_and_kill_between("[MOOD=", "]", analysis);
+ retval += (tmp != "" ? ("Mood=" + tmp + "|") : "");
+ tmp = fetch_and_kill_between("[NEGATIVE=", "]", analysis);
+ retval += (tmp != "" ? ("Negative=" + tmp + "|") : "");
+ tmp = fetch_and_kill_between("[NUMTYPE=", "]", analysis);
+ retval += (tmp != "" ? ("Numtype=" + tmp + "|") : "");
+ tmp = fetch_and_kill_between("[NUM=", "]", analysis);
+ retval += (tmp != "" ? ("Number=" + tmp + "|") : "");
+ tmp = fetch_and_kill_between("[PERS=", "]", analysis);
+ retval += (tmp != "" ? ("Person=" + tmp + "|") : "");
+ tmp = fetch_and_kill_between("[POSS=", "]", analysis);
+ retval += (tmp != "" ? ("Poss=" + tmp + "|") : "");
+ tmp = fetch_and_kill_between("[PRONTYPE=", "]", analysis);
+ retval += (tmp != "" ? ("PronType=" + tmp + "|") : "");
+ tmp = fetch_and_kill_between("[REFLEX=", "]", analysis);
+ retval += (tmp != "" ? ("Reflex=" + tmp + "|") : "");
+ tmp = fetch_and_kill_between("[TENSE=", "]", analysis);
+ retval += (tmp != "" ? ("Tense=" + tmp + "|") : "");
+ tmp = fetch_and_kill_between("[VERBFORM=", "]", analysis);
+ retval += (tmp != "" ? ("VerbForm=" + tmp + "|") : "");
+ tmp = fetch_and_kill_between("[VOICE=", "]", analysis);
+ retval += (tmp != "" ? ("Voice=" + tmp + "|") : "");
+ if (retval.size() != 0) {
+ retval.erase(retval.size() - 1);
+ }
+ return retval;
+}
+
+std::string empty_to_underscore(std::string to_test)
+{
+ if (to_test.size() == 0) {
+ return "_";
+ }
+ return to_test;
+}
+
+void print_location_vector(hfst_ol::PmatchContainer & container,
+ LocationVector const & locations,
+ std::ostream & outstream)
{
if (output_format == tokenize && locations.size() != 0) {
outstream << locations.at(0).input;
@@ -218,6 +720,18 @@ void print_location_vector(LocationVector const & locations, std::ostream & outs
outstream << "\t" << locations.at(0).weight;
}
outstream << std::endl;
+ if (locations.at(0).tag == "<Boundary=Sentence>") {
+ outstream << std::endl;
+ }
+ } else if (output_format == space_separated && locations.size() != 0) {
+ outstream << locations.at(0).input;
+ if (print_weights) {
+ outstream << "\t" << locations.at(0).weight;
+ }
+ outstream << " ";
+ if (locations.at(0).tag == "<Boundary=Sentence>") {
+ outstream << std::endl;
+ }
} else if (output_format == cg && locations.size() != 0) {
// Print the cg cohort header
outstream << "\"<" << locations.at(0).input << ">\"" << std::endl;
@@ -238,6 +752,9 @@ void print_location_vector(LocationVector const & locations, std::ostream & outs
}
outstream << std::endl;
}
+ outstream << std::endl;
+ } else if (output_format == giellacg && locations.size() != 0) {
+ print_location_vector_giellacg(container, locations, outstream);
} else if (output_format == xerox) {
for (LocationVector::const_iterator loc_it = locations.begin();
loc_it != locations.end(); ++loc_it) {
@@ -247,6 +764,33 @@ void print_location_vector(LocationVector const & locations, std::ostream & outs
}
outstream << std::endl;
}
+ outstream << std::endl;
+ } else if (output_format == conllu) {
+ hfst_ol::Weight lowest_weight = hfst_ol::INFINITE_WEIGHT;
+ hfst_ol::Location best_location;
+ for (LocationVector::const_iterator loc_it = locations.begin();
+ loc_it != locations.end(); ++loc_it) {
+ if (loc_it->weight < lowest_weight) {
+ best_location = *loc_it;
+ lowest_weight = loc_it->weight;
+ }
+// if (loc_it->tag == "@MULTIWORD@"
+// outstream << loc_it->input << "\t" << loc_it->output;
+ }
+ outstream << token_number
+ << "\t" << best_location.input;
+ outstream << "\t" << empty_to_underscore(fetch_and_kill_between("[WORD_ID=", "]", best_location.output));
+ outstream << "\t" << empty_to_underscore(fetch_and_kill_between("[UPOS=", "]", best_location.output));
+ outstream << "\t" << empty_to_underscore(fetch_and_kill_between("[XPOS=", "]", best_location.output));
+ outstream << "\t" << empty_to_underscore(fetch_and_kill_feats(best_location.output))
+ << "\t" << "_" // HEAD
+ << "\t" << "_" // DEPREL
+ << "\t" << "_"; // DEPS
+ outstream << "\t" << empty_to_underscore(best_location.output); // MISC
+ if (print_weights) {
+ outstream << "\t" << best_location.weight;
+ }
+ outstream << std::endl;
} else if (output_format == finnpos) {
std::set<std::string> tags;
std::set<std::string> lemmas;
@@ -292,24 +836,23 @@ void print_location_vector(LocationVector const & locations, std::ostream & outs
}
outstream << accumulator.substr(0, accumulator.size() - 1);
}
- outstream << "\t_";
+ outstream << "\t_" << std::endl;
+ if (locations.at(0).tag == "<Boundary=Sentence>") {
+ outstream << std::endl;
+ }
}
// std::cerr << "from print_location_vector\n";
- outstream << std::endl;
}
void match_and_print(hfst_ol::PmatchContainer & container,
std::ostream & outstream,
- std::string & input_text)
+ const string & input_text)
{
- if (input_text.size() > 0 && input_text.at(input_text.size() - 1) == '\n') {
- // Remove final newline
- input_text.erase(input_text.size() -1, 1);
- }
LocationVectorVector locations = container.locate(input_text, time_cutoff);
if (locations.size() == 0 && print_all) {
print_no_output(input_text, outstream);
}
+ token_number = 1;
for(LocationVectorVector::const_iterator it = locations.begin();
it != locations.end(); ++it) {
if ((it->size() == 1 && it->at(0).output.compare("@_NONMATCHING_@") == 0)) {
@@ -319,41 +862,149 @@ void match_and_print(hfst_ol::PmatchContainer & container,
continue;
// All nonmatching cases have been handled
}
- print_location_vector(*it, outstream);
+ print_location_vector(container,
+ keep_n_best_weight(dedupe_locations(*it)),
+ outstream);
+ ++token_number;
}
if (output_format == finnpos) {
outstream << std::endl;
}
}
-
-int process_input(hfst_ol::PmatchContainer & container,
- std::ostream & outstream)
+// TODO: lambda this when C++11 available everywhere
+inline void process_input_0delim_print(hfst_ol::PmatchContainer & container,
+ std::ostream & outstream,
+ std::ostringstream& cur)
+{
+ string input_text(cur.str());
+ if(!input_text.empty()) {
+ match_and_print(container, outstream, input_text);
+ }
+ cur.clear();
+ cur.str(string());
+}
+
+template<bool do_superblank>
+int process_input_0delim(hfst_ol::PmatchContainer & container,
+ std::ostream & outstream)
{
- std::string input_text;
char * line = NULL;
- size_t len = 0;
- while (hfst_getline(&line, &len, inputfile) > 0) {
- if (!blankline_separated) {
- // newline separated
- input_text = line;
- match_and_print(container, outstream, input_text);
- } else if (line[0] == '\n') {
- match_and_print(container, outstream, input_text);
- input_text.clear();
- } else {
- input_text.append(line);
+ size_t bufsize = 0;
+ bool in_blank = false;
+ std::ostringstream cur;
+ ssize_t len = -1;
+ while ((len = hfst_getdelim(&line, &bufsize, '\0', inputfile)) > 0) {
+ bool escaped = false; // beginning of line is necessarily unescaped
+ for(size_t i = 0; i < len; ++i) {
+ if(escaped) {
+ cur << line[i];
+ escaped = false;
+ continue;
+ }
+ else if(do_superblank && !in_blank && line[i] == '[') {
+ process_input_0delim_print(container, outstream, cur);
+ cur << line[i];
+ in_blank = true;
+ }
+ else if(do_superblank && in_blank && line[i] == ']') {
+ cur << line[i];
+ if(i+1 < len && line[i+1] == '[') {
+ // Join consecutive superblanks
+ ++i;
+ cur << line[i];
+ }
+ else {
+ in_blank = false;
+ print_nonmatching_sequence(cur.str(), outstream);
+ cur.clear();
+ cur.str(string());
+ }
+ }
+ else if(!in_blank && line[i] == '\n') {
+ cur << line[i];
+ process_input_0delim_print(container, outstream, cur);
+ }
+ else if(line[i] == '\0') {
+ process_input_0delim_print(container, outstream, cur);
+ outstream << "<STREAMCMD:FLUSH>" << std::endl; // CG format uses this instead of \0
+ outstream.flush();
+ if(outstream.bad()) {
+ std::cerr << "hfst-tokenize: Could not flush file" << std::endl;
+ }
+ }
+ else {
+ cur << line[i];
+ }
+ escaped = (line[i] == '\\');
}
free(line);
line = NULL;
+ if(std::feof(inputfile)) {
+ break;
+ }
}
-
- if (blankline_separated && !input_text.empty()) {
- match_and_print(container, outstream, input_text);
+ if(in_blank) {
+ print_nonmatching_sequence(cur.str(), outstream);
+ }
+ else {
+ process_input_0delim_print(container, outstream, cur);
}
return EXIT_SUCCESS;
}
+inline void maybe_erase_newline(string& input_text)
+{
+ if(!keep_newlines && input_text.size() > 0 && input_text.at(input_text.size() - 1) == '\n') {
+ // Remove final newline
+ input_text.erase(input_text.size() -1, 1);
+ }
+}
+
+int process_input(hfst_ol::PmatchContainer & container,
+ std::ostream & outstream)
+{
+ if(output_format == giellacg || superblanks) {
+ if(superblanks) {
+ return process_input_0delim<true>(container, outstream);
+ }
+ else {
+ return process_input_0delim<false>(container, outstream);
+ }
+ }
+ string input_text;
+ char * line = NULL;
+ size_t bufsize = 0;
+ if(blankline_separated) {
+ while (hfst_getline(&line, &bufsize, inputfile) > 0) {
+ if (line[0] == '\n') {
+ maybe_erase_newline(input_text);
+ match_and_print(container, outstream, input_text);
+ input_text.clear();
+ } else {
+ input_text.append(line);
+ }
+ free(line);
+ line = NULL;
+ }
+ if (!input_text.empty()) {
+ maybe_erase_newline(input_text);
+ match_and_print(container, outstream, input_text);
+ }
+ }
+ else {
+ // newline or non-separated
+ while (hfst_getline(&line, &bufsize, inputfile) > 0) {
+ input_text = line;
+ maybe_erase_newline(input_text);
+ match_and_print(container, outstream, input_text);
+ free(line);
+ line = NULL;
+ }
+ }
+
+ return EXIT_SUCCESS;
+}
int parse_options(int argc, char** argv)
{
@@ -365,18 +1016,26 @@ int parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
{"newline", no_argument, 0, 'n'},
+ {"keep-newline", no_argument, 0, 'k'},
{"print-all", no_argument, 0, 'a'},
{"print-weights", no_argument, 0, 'w'},
{"tokenize-multichar", no_argument, 0, 'm'},
{"time-cutoff", required_argument, 0, 't'},
+ {"weight-classes", required_argument, 0, 'l'},
+ {"unique", required_argument, 0, 'u'},
{"segment", no_argument, 0, 'z'},
+ {"space-separated", no_argument, 0, 'd'},
{"xerox", no_argument, 0, 'x'},
{"cg", no_argument, 0, 'c'},
+ {"superblanks", no_argument, 0, 'S'},
+ {"giella-cg", no_argument, 0, 'g'},
+ {"gtd", no_argument, 0, 'g'},
+ {"conllu", no_argument, 0, 'C'},
{"finnpos", no_argument, 0, 'f'},
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT "nawmt:zxcf",
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT "nkawmut:l:zixcSgCf",
long_options, &option_index);
if (-1 == c)
{
@@ -387,6 +1046,10 @@ int parse_options(int argc, char** argv)
switch (c)
{
#include "inc/getopt-cases-common.h"
+ case 'k':
+ keep_newlines = true;
+ blankline_separated = false;
+ break;
case 'n':
blankline_separated = false;
break;
@@ -407,23 +1070,52 @@ int parse_options(int argc, char** argv)
return EXIT_FAILURE;
}
break;
+ case 'u':
+ dedupe = true;
+ break;
+ case 'l':
+ max_weight_classes = atoi(optarg);
+ if (max_weight_classes < 1)
+ {
+ std::cerr << "Invalid or no argument --weight-classes count\n";
+ return EXIT_FAILURE;
+ }
+ break;
case 'z':
output_format = tokenize;
break;
+ case 'i':
+ output_format = space_separated;
+ break;
case 'x':
output_format = xerox;
break;
case 'c':
output_format = cg;
break;
+ case 'C':
+ output_format = conllu;
+ break;
+ case 'S':
+ superblanks = true;
+ break;
+ case 'g':
+ output_format = giellacg;
+ print_weights = true;
+ print_all = true;
+ dedupe = true;
+ if(max_weight_classes == std::numeric_limits<int>::max()) {
+ max_weight_classes = 2;
+ }
+ break;
case 'f':
output_format = finnpos;
break;
#include "inc/getopt-cases-error.h"
}
-
-
+
+
}
// if (!inputNamed)
@@ -431,7 +1123,7 @@ int parse_options(int argc, char** argv)
// inputfile = stdin;
// inputfilename = hfst_strdup("<stdin>");
// }
-
+
// no more options, we should now be at the input filename
if ( (optind + 1) < argc)
{
@@ -453,49 +1145,13 @@ int parse_options(int argc, char** argv)
#include "inc/check-params-common.h"
-
+
return EXIT_FAILURE;
}
-bool first_transducer_is_called_TOP(std::istream & f)
+bool first_transducer_is_called_TOP(const HfstTransducer & dictionary)
{
- const char* header1 = "HFST";
- unsigned int header_loc = 0;
- int c;
- while(header_loc < strlen(header1) + 1) {
- c = f.get();
- ++header_loc;
- if(c != header1[header_loc]) {
- return false;
- }
- }
- if(header_loc == strlen(header1) + 1) {
- unsigned short remaining_header_len;
- f.read((char*) &remaining_header_len, sizeof(remaining_header_len));
- if (f.get() != '\0') {
- return false;
- }
- char * headervalue = new char[remaining_header_len];
- f.read(headervalue, remaining_header_len);
- if (headervalue[remaining_header_len - 1] != '\0') {
- delete[] headervalue;
- return false;
- }
- int i = 0;
- while (i < remaining_header_len) {
- if (strcmp(headervalue + i, "name") == 0) {
- bool retval = strcmp(headervalue + i + strlen("name") + 1, "TOP") == 0;
- delete[] headervalue;
- return retval;
- }
- while (headervalue[i] != '\0') {
- ++i;
- }
- ++i;
- }
- delete[] headervalue;
- }
- return false;
+ return dictionary.get_name() == "TOP";
}
int main(int argc, char ** argv)
@@ -513,7 +1169,9 @@ int main(int argc, char ** argv)
return EXIT_FAILURE;
}
try {
- if (first_transducer_is_called_TOP(instream)) {
+ hfst::HfstInputStream is(tokenizer_filename);
+ HfstTransducer dictionary(is);
+ if (first_transducer_is_called_TOP(dictionary)) {
instream.seekg(0);
instream.clear();
hfst_ol::PmatchContainer container(instream);
@@ -522,19 +1180,19 @@ int main(int argc, char ** argv)
return process_input(container, std::cout);
} else {
instream.close();
- hfst::HfstInputStream is(tokenizer_filename);
- HfstTransducer dictionary(is);
hfst_ol::PmatchContainer container = make_naive_tokenizer(dictionary);
container.set_verbose(verbose);
container.set_single_codepoint_tokenization(!tokenize_multichar);
return process_input(container, std::cout);
}
} catch(HfstException & e) {
- std::cerr << "The archive in " << tokenizer_filename << " doesn't look right."
- "\nDid you make it with hfst-pmatch2fst or make sure it's in weighted optimized-lookup format?\n";
+ std::cerr << "The archive in " << tokenizer_filename <<
+ " doesn't look right.\nDid you make it with hfst-pmatch2fst"
+ " or make sure it's in weighted optimized-lookup format?\n"
+ "Exception thrown:\n" << e.what() << std::endl;
return 1;
}
-
+
// if (outfile != stdout) {
// std::filebuf fb;
// fb.open(outfilename, std::ios::out);
diff --git a/tools/src/hfst-tool-metadata.cc b/tools/src/hfst-tool-metadata.cc
index d3d1d96..858f618 100644
--- a/tools/src/hfst-tool-metadata.cc
+++ b/tools/src/hfst-tool-metadata.cc
@@ -1,7 +1,7 @@
/**
* @file hfst-tool-metadata.cc
*
- * @brief
+ * @brief
*/
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -68,17 +68,18 @@ hfst_set_name(hfst::HfstTransducer& dest, const hfst::HfstTransducer& lhs,
{
throw std::logic_error("!(a && b) || (!a && b) || (a && !b) || (!a && !b)");
}
- }
+ }
void
hfst_set_formula(hfst::HfstTransducer& dest, const string& src,
const string& op)
{
- if ((0 < src.at(0)) && (src.at(0) < 128))
+ int c = (int)src.at(0);
+ if ((0 < c) && (c < 128))
{
dest.set_property("formulaic-definition", op + " " + src.substr(0, 1));
}
- else
+ else
{
dest.set_property("formulaic-definition", op + " U8");
}
@@ -91,7 +92,7 @@ hfst_set_formula(hfst::HfstTransducer& dest, const hfst::HfstTransducer& src,
if (src.get_property("formulaic-definition") != "")
{
- dest.set_property("formulaic-definition",
+ dest.set_property("formulaic-definition",
op + " " + src.get_property("formulaic-definition"));
}
else
@@ -109,7 +110,7 @@ hfst_set_formula(hfst::HfstTransducer& dest, const hfst::HfstTransducer& lhs,
(rhs.get_property("formulaic-definition") != ""))
{
dest.set_property("formulaic-definition",
- lhs.get_property("formulaic-definition") +
+ lhs.get_property("formulaic-definition") +
" " + op + " " +
rhs.get_property("formulaic-definition"));
}
@@ -117,14 +118,14 @@ hfst_set_formula(hfst::HfstTransducer& dest, const hfst::HfstTransducer& lhs,
(rhs.get_property("formulaic-definition") != ""))
{
dest.set_property("formulaic-definition",
- ". " + op + " " +
+ ". " + op + " " +
rhs.get_property("formulaic-definition"));
}
else if ((lhs.get_property("formulaic-definition") != "") &&
(rhs.get_property("formulaic-definition") == ""))
{
dest.set_property("formulaic-definition",
- lhs.get_property("formulaic-definition") +
+ lhs.get_property("formulaic-definition") +
" " + op + " .");
}
else
@@ -134,7 +135,7 @@ hfst_set_formula(hfst::HfstTransducer& dest, const hfst::HfstTransducer& lhs,
}
}
-void
+void
hfst_set_commandline_def(hfst::HfstTransducer& dest,
int argc, const char** argv)
{
@@ -165,7 +166,7 @@ hfst_set_commandline_def(hfst::HfstTransducer& dest,
dest.set_property("commandline-definition", cmdline);
}
-void
+void
hfst_set_commandline_def(hfst::HfstTransducer& dest,
const hfst::HfstTransducer& src,
int argc, const char** argv)
diff --git a/tools/src/hfst-tool-metadata.h b/tools/src/hfst-tool-metadata.h
index 6fbff49..dcaa6e1 100644
--- a/tools/src/hfst-tool-metadata.h
+++ b/tools/src/hfst-tool-metadata.h
@@ -39,10 +39,10 @@ void hfst_set_name(hfst::HfstTransducer& dest, const hfst::HfstTransducer& lhs,
void hfst_set_formula(hfst::HfstTransducer& dest, const std::string& src,
const std::string& op);
-void hfst_set_formula(hfst::HfstTransducer& dest,
+void hfst_set_formula(hfst::HfstTransducer& dest,
const hfst::HfstTransducer& src,
const std::string& op);
-void hfst_set_formula(hfst::HfstTransducer& dest,
+void hfst_set_formula(hfst::HfstTransducer& dest,
const hfst::HfstTransducer& lhs,
const hfst::HfstTransducer& rhs,
const std::string& op);
diff --git a/tools/src/hfst-traverse.cc b/tools/src/hfst-traverse.cc
index bf00938..1b1d063 100644
--- a/tools/src/hfst-traverse.cc
+++ b/tools/src/hfst-traverse.cc
@@ -131,13 +131,13 @@ parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
- // add tool-specific options here
+ // add tool-specific options here
{"cave", no_argument, 0, 'X'},
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT,
long_options, &option_index);
if (-1 == c)
@@ -164,7 +164,7 @@ parse_options(int argc, char** argv)
int
main_loop(HfstBasicTransducer trans)
{
- fprintf(message_out,
+ fprintf(message_out,
"Enter labels to seek all paths\n");
// record current paths with their end states
multimap<string, HfstState> paths;
@@ -192,18 +192,18 @@ main_loop(HfstBasicTransducer trans)
++p)
{
_rl_arcs.clear();
- fprintf(message_out, "On path `%s' are continuations:\n",
+ fprintf(message_out, "On path `%s' are continuations:\n",
p->first.c_str());
if (trans[p->second].begin() == trans[p->second].end())
{
fprintf(message_out,
"<Nothing, you've hit a dead end here>\n");
}
- for (HfstBasicTransducer::HfstTransitions::const_iterator arc = trans[p->second].begin();
+ for (hfst::implementations::HfstBasicTransitions::const_iterator arc = trans[p->second].begin();
arc != trans[p->second].end();
++arc)
{
- fprintf(message_out, "%s\t%s\n",
+ fprintf(message_out, "%s\t%s\n",
arc->get_input_symbol().c_str(),
arc->get_output_symbol().c_str());
_rl_arcs.push_back(arc->get_input_symbol());
@@ -219,7 +219,7 @@ main_loop(HfstBasicTransducer trans)
p != paths.end();
++p)
{
- for (HfstBasicTransducer::HfstTransitions::const_iterator arc = trans[p->second].begin();
+ for (hfst::implementations::HfstBasicTransitions::const_iterator arc = trans[p->second].begin();
arc != trans[p->second].end();
++arc)
{
@@ -274,7 +274,7 @@ process_stream(HfstInputStream& instream)
HfstBasicTransducer walkable(trans);
if (cave_mode)
{
- fprintf(message_out,
+ fprintf(message_out,
"WELCOME TO ADVENTURE!! WOULD YOU LIKE INSTRUCTIONS?\n\n");
char* yesno = hfst_readline("");
if ((strcmp(yesno, "YES") == 0) || (strcmp(yesno, "yes") == 0))
@@ -301,7 +301,7 @@ process_stream(HfstInputStream& instream)
}
else
{
- fprintf(message_out, "Traversing automaton %s\n\n",
+ fprintf(message_out, "Traversing automaton %s\n\n",
trans_name.c_str());
}
if (walkable.begin() == walkable.end())
@@ -332,7 +332,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-twolc/src/Makefile.am b/tools/src/hfst-twolc/src/Makefile.am
index 8ba7698..84de2e0 100644
--- a/tools/src/hfst-twolc/src/Makefile.am
+++ b/tools/src/hfst-twolc/src/Makefile.am
@@ -1,222 +1,41 @@
-AM_YFLAGS= -d -o y.tab.c
AM_CXXFLAGS= -Wcast-qual -Wno-deprecated -Wfatal-errors
-bin_PROGRAMS=htwolcpre1 htwolcpre2 htwolcpre3 hfst-twolc-system
-bin_SCRIPTS=hfst-twolc$(EXEEXT) hfst-twolc-loc$(EXEEXT)
-EXTRA_DIST=hfst-twolc.sh hfst-twolc.bat common_globals.h
-IO_SRC=$(srcdir)/io_src
-STRING_SRC=$(srcdir)/string_src
-VARIABLE_SRC=$(srcdir)/variable_src
-RULE_SRC=$(srcdir)/rule_src
-ALPHABET_SRC=$(srcdir)/alphabet_src
-CL_SRC=$(srcdir)/commandline_src
-AM_CPPFLAGS = -I${top_srcdir}/libhfst/src
-LDADD = $(top_builddir)/libhfst/src/libhfst.la
-
-AM_CPPFLAGS += -I${top_srcdir}/back-ends/foma \
- -I${top_srcdir}/back-ends
-if WANT_MINGW
- AM_CPPFLAGS += -I${top_srcdir}/back-ends/openfstwin/src/include \
- -I${top_srcdir}/back-ends/dlfcn -DWINDOWS
+if WANT_TWOLC_SCRIPT
+ bin_PROGRAMS=hfst-twolc htwolcpre1 htwolcpre2 htwolcpre3
+ bin_SCRIPTS=hfst-twolc-script hfst-twolc-local-script
else
- AM_CPPFLAGS += -I${top_srcdir}/back-ends/openfst/src/include
+ bin_PROGRAMS=hfst-twolc
+ bin_SCRIPTS=
endif
+AM_CPPFLAGS = -I${top_srcdir}/libhfst/src -I${top_srcdir}/libhfst/src/parsers
+LDADD = $(top_builddir)/libhfst/src/libhfst.la
+
# the preprocessor scripts do not support options per se, so we whitelist
# them here
-AM_INSTALLCHECK_STD_OPTIONS_EXEMPT=htwolcpre1 htwolcpre2 htwolcpre3 \
- hfst-twolc$(EXEEXT) hfst-twolc-loc$(EXEEXT) hfst-twolc-system$(EXEEXT)
-
-htwolcpre1_SOURCES=htwolcpre1.yy scanner1.ll HfstTwolcDefs.h \
- $(CL_SRC)/CommandLine.h $(CL_SRC)/CommandLine.cc \
- $(IO_SRC)/input_defs.h grammar_defs.h \
- $(IO_SRC)/InputReader.h $(IO_SRC)/InputReader.cc \
- $(STRING_SRC)/string_manipulation.cc \
- $(STRING_SRC)/string_manipulation.h \
- $(VARIABLE_SRC)/VariableValues.h \
- $(VARIABLE_SRC)/VariableValues.cc \
- $(VARIABLE_SRC)/VariableValueIterator.h \
- $(VARIABLE_SRC)/VariableDefs.h \
- $(VARIABLE_SRC)/VariableContainerBase.h \
- $(VARIABLE_SRC)/VariableContainer.h \
- $(VARIABLE_SRC)/VariableBlockContainer.h \
- $(VARIABLE_SRC)/VariableBlock.h \
- $(VARIABLE_SRC)/RuleVariablesConstIterator.cc \
- $(VARIABLE_SRC)/RuleVariablesConstIterator.h \
- $(VARIABLE_SRC)/RuleVariables.cc \
- $(VARIABLE_SRC)/RuleVariables.h \
- $(VARIABLE_SRC)/RuleSymbolVector.cc \
- $(VARIABLE_SRC)/RuleSymbolVector.h \
- $(VARIABLE_SRC)/MatchedConstContainerIterator.h \
- $(VARIABLE_SRC)/MixedConstContainerIterator.h \
- $(VARIABLE_SRC)/ConstContainerIterator.h
-
-htwolcpre2_SOURCES=htwolcpre2.yy scanner2.ll HfstTwolcDefs.h \
- $(CL_SRC)/CommandLine.h $(CL_SRC)/CommandLine.cc \
- $(IO_SRC)/input_defs.h grammar_defs.h \
- $(IO_SRC)/InputReader.h $(IO_SRC)/InputReader.cc
-
-htwolcpre3_SOURCES=htwolcpre3.yy scanner3.ll HfstTwolcDefs.h \
- $(CL_SRC)/CommandLine.h $(CL_SRC)/CommandLine.cc \
- $(IO_SRC)/input_defs.h grammar_defs.h \
- $(IO_SRC)/InputReader.cc $(IO_SRC)/InputReader.h \
- $(STRING_SRC)/string_manipulation.cc \
- $(STRING_SRC)/string_manipulation.h \
- $(ALPHABET_SRC)/Alphabet.cc $(ALPHABET_SRC)/Alphabet.h \
- $(RULE_SRC)/ConflictResolvingLeftArrowRule.cc \
- $(RULE_SRC)/ConflictResolvingLeftArrowRule.h \
- $(RULE_SRC)/ConflictResolvingRightArrowRule.cc \
- $(RULE_SRC)/ConflictResolvingRightArrowRule.h \
- $(RULE_SRC)/LeftArrowRule.cc \
- $(RULE_SRC)/LeftArrowRule.h \
- $(RULE_SRC)/LeftArrowRuleContainer.cc \
- $(RULE_SRC)/LeftArrowRuleContainer.h \
- $(RULE_SRC)/LeftRestrictionArrowRule.cc \
- $(RULE_SRC)/LeftRestrictionArrowRule.h \
- $(RULE_SRC)/OtherSymbolTransducer.cc \
- $(RULE_SRC)/OtherSymbolTransducer.h \
- $(RULE_SRC)/RightArrowRule.cc \
- $(RULE_SRC)/RightArrowRule.h \
- $(RULE_SRC)/RightArrowRuleContainer.cc \
- $(RULE_SRC)/RightArrowRuleContainer.h \
- $(RULE_SRC)/Rule.cc \
- $(RULE_SRC)/Rule.h \
- $(RULE_SRC)/RuleContainer.cc \
- $(RULE_SRC)/RuleContainer.h \
- $(RULE_SRC)/TwolCGrammar.cc \
- $(RULE_SRC)/TwolCGrammar.h
-
-hfst_twolc_system_SOURCES=hfst-twolc-system.cc
+AM_INSTALLCHECK_STD_OPTIONS_EXEMPT=htwolcpre1 htwolcpre2 htwolcpre3
-# some systems might require all these
-BUILT_SOURCES=htwolcpre1.cc htwolcpre2.cc htwolcpre3.cc
-
-if YACC_USE_PARSER_H_EXTENSION
- HEADER=h
-else
- HEADER=hh
+hfst_twolc_SOURCES=hfst-twolc.cc
+if WANT_TWOLC_SCRIPT
+ htwolcpre1_SOURCES=htwolcpre1-main.cc
+ htwolcpre2_SOURCES=htwolcpre2-main.cc
+ htwolcpre3_SOURCES=htwolcpre3-main.cc
endif
-CLEANFILES=htwolcpre1.$(HEADER) htwolcpre2.$(HEADER) htwolcpre3.$(HEADER) \
- htwolcpre1.cc htwolcpre2.cc htwolcpre3.cc
-
-hfst-twolc$(EXEEXT): hfst-twolc.sh
+if WANT_TWOLC_SCRIPT
+hfst-twolc-script: hfst-twolc.sh
cat $< | sed -e 's|BINDIR|$(bindir)|' > $@
- chmod ogu+rx hfst-twolc$(EXEEXT)
+ chmod ogu+rx hfst-twolc-script
-hfst-twolc-loc$(EXEEXT): hfst-twolc.sh
+hfst-twolc-local-script: hfst-twolc.sh
cat $< | sed -e 's|BINDIR|$(abs_builddir)|' > $@
- chmod ogu+rx hfst-twolc-loc$(EXEEXT)
+ chmod ogu+rx hfst-twolc-local-script
clean-local:
- -rm -f hfst-twolc$(EXEEXT) hfst-twolc-loc$(EXEEXT) hfst-twolc-system$(EXEEXT)
-
-CLEANFILES+=htwolcpre1.$(HEADER) htwolcpre2.$(HEADER) htwolcpre3.$(HEADER)
-
-scanner1.ll: htwolcpre1.$(HEADER)
-scanner2.ll: htwolcpre2.$(HEADER)
-scanner3.ll: htwolcpre3.$(HEADER)
-
-HFST_TWOLC_TESTS=string_manipulation InputReader Alphabet \
- OtherSymbolTransducer Rule LeftArrowRule RightArrowRule \
- RuleContainer LeftArrowRuleContainer RightArrowRuleContainer \
- ConflictResolvingLeftArrowRule \
- ConflictResolvingRightArrowRule TwolCGrammar \
- LeftRestrictionArrowRule
-
-check_PROGRAMS=$(HFST_TWOLC_TESTS)
-
-string_manipulation_SOURCES=$(STRING_SRC)/string_manipulation.cc
-string_manipulation_CXXFLAGS=-DSTRING_MANIPULATION_TEST
-
-InputReader_SOURCES=$(IO_SRC)/InputReader.cc
-InputReader_CXXFLAGS=-DINPUT_READER_TEST
-
-Alphabet_SOURCES=$(ALPHABET_SRC)/Alphabet.cc \
- $(RULE_SRC)/OtherSymbolTransducer.cc
-Alphabet_CXXFLAGS=-DTEST_ALPHABET
-
-OtherSymbolTransducer_SOURCES=$(RULE_SRC)/OtherSymbolTransducer.cc
-OtherSymbolTransducer_CXXFLAGS=-DTEST_OTHER_SYMBOL_TRANSDUCER
-
-Rule_SOURCES=$(RULE_SRC)/OtherSymbolTransducer.cc $(RULE_SRC)/Rule.cc \
- $(STRING_SRC)/string_manipulation.cc
-Rule_CXXFLAGS=-DTEST_RULE
-
-LeftArrowRule_SOURCES=$(RULE_SRC)/OtherSymbolTransducer.cc \
- $(STRING_SRC)/string_manipulation.cc \
- $(RULE_SRC)/Rule.cc $(RULE_SRC)/LeftArrowRule.cc
-LeftArrowRule_CXXFLAGS=-DTEST_LEFT_ARROW_RULE
-
-LeftRestrictionArrowRule_SOURCES=$(RULE_SRC)/OtherSymbolTransducer.cc \
- $(STRING_SRC)/string_manipulation.cc \
- $(RULE_SRC)/Rule.cc $(RULE_SRC)/LeftArrowRule.cc \
- $(RULE_SRC)/LeftRestrictionArrowRule.cc
-LeftRestrictionArrowRule_CXXFLAGS=-DTEST_LEFT_RESTRICTION_ARROW_RULE
-
-RightArrowRule_SOURCES=$(RULE_SRC)/OtherSymbolTransducer.cc \
- $(STRING_SRC)/string_manipulation.cc \
- $(RULE_SRC)/Rule.cc $(RULE_SRC)/RightArrowRule.cc
-RightArrowRule_CXXFLAGS=-DTEST_RIGHT_ARROW_RULE
-
-RuleContainer_SOURCES=$(RULE_SRC)/OtherSymbolTransducer.cc \
- $(RULE_SRC)/Rule.cc $(RULE_SRC)/RuleContainer.cc \
- $(STRING_SRC)/string_manipulation.cc
-RuleContainer_CXXFLAGS=-DTEST_RULE_CONTAINER
-
-LeftArrowRuleContainer_SOURCES=$(RULE_SRC)/OtherSymbolTransducer.cc \
- $(RULE_SRC)/Rule.cc \
- $(RULE_SRC)/RuleContainer.cc \
- $(STRING_SRC)/string_manipulation.cc \
- $(RULE_SRC)/ConflictResolvingLeftArrowRule.cc \
- $(RULE_SRC)/LeftArrowRuleContainer.cc \
- $(RULE_SRC)/LeftArrowRule.cc
-
-LeftArrowRuleContainer_CXXFLAGS=-DTEST_LEFT_ARROW_RULE_CONTAINER
-
-RightArrowRuleContainer_SOURCES=$(RULE_SRC)/OtherSymbolTransducer.cc \
- $(RULE_SRC)/Rule.cc \
- $(RULE_SRC)/RuleContainer.cc \
- $(STRING_SRC)/string_manipulation.cc \
- $(RULE_SRC)/ConflictResolvingRightArrowRule.cc\
- $(RULE_SRC)/RightArrowRuleContainer.cc \
- $(RULE_SRC)/RightArrowRule.cc
-RightArrowRuleContainer_CXXFLAGS=-DTEST_RIGHT_ARROW_RULE_CONTAINER
-
-ConflictResolvingLeftArrowRule_SOURCES=$(RULE_SRC)/OtherSymbolTransducer.cc \
- $(RULE_SRC)/Rule.cc \
- $(RULE_SRC)/RuleContainer.cc \
- $(STRING_SRC)/string_manipulation.cc \
- $(RULE_SRC)/ConflictResolvingLeftArrowRule.cc\
- $(RULE_SRC)/LeftArrowRule.cc
-ConflictResolvingLeftArrowRule_CXXFLAGS=\
--DTEST_CONFLICT_RESOLVING_LEFT_ARROW_RULE
-
-ConflictResolvingRightArrowRule_SOURCES=$(RULE_SRC)/OtherSymbolTransducer.cc \
- $(RULE_SRC)/Rule.cc \
- $(RULE_SRC)/RuleContainer.cc \
- $(STRING_SRC)/string_manipulation.cc \
- $(RULE_SRC)/ConflictResolvingRightArrowRule.cc\
- $(RULE_SRC)/RightArrowRule.cc
-ConflictResolvingRightArrowRule_CXXFLAGS=\
--DTEST_CONFLICT_RESOLVING_RIGHT_ARROW_RULE
-
-TwolCGrammar_SOURCES=$(RULE_SRC)/OtherSymbolTransducer.cc \
- $(RULE_SRC)/Rule.cc $(RULE_SRC)/RuleContainer.cc \
- $(STRING_SRC)/string_manipulation.cc \
- $(RULE_SRC)/ConflictResolvingLeftArrowRule.cc \
- $(RULE_SRC)/LeftArrowRule.cc \
- $(RULE_SRC)/ConflictResolvingRightArrowRule.cc\
- $(RULE_SRC)/RightArrowRule.cc \
- $(RULE_SRC)/RightArrowRuleContainer.cc \
- $(RULE_SRC)/LeftArrowRuleContainer.cc \
- $(RULE_SRC)/TwolCGrammar.cc \
- $(RULE_SRC)/LeftRestrictionArrowRule.cc \
- $(ALPHABET_SRC)/Alphabet.cc
-TwolCGrammar_CXXFLAGS=-DTEST_TWOL_C_GRAMMAR
-
-TESTS=$(HFST_TWOLC_TESTS)
+ -rm -f hfst-twolc-script hfst-twolc-loc-script
+endif
-EXTRA_DIST += htwolcpre1.$(HEADER) htwolcpre2.$(HEADER) htwolcpre3.$(HEADER) \
- htwolcpre1.cc htwolcpre2.cc htwolcpre3.cc \
- scanner1.cc scanner2.cc scanner3.cc
+# Distribute all twolc implementations
+EXTRA_DIST=hfst-twolc.sh hfst-twolc.bat \
+ htwolcpre1-main.cc htwolcpre2-main.cc htwolcpre3-main.cc \
+ hfst-twolc.cc
diff --git a/tools/src/hfst-twolc/src/doxygen_src/hfst_twolc_doxygen_header.h b/tools/src/hfst-twolc/src/doxygen_src/hfst_twolc_doxygen_header.h
index 1ba68e4..df71b3e 100644
--- a/tools/src/hfst-twolc/src/doxygen_src/hfst_twolc_doxygen_header.h
+++ b/tools/src/hfst-twolc/src/doxygen_src/hfst_twolc_doxygen_header.h
@@ -1,57 +1,57 @@
//! @file hfst_twolc_doxygen_header.h
//!
//! @author Miikka Silfverberg
-//!
+//!
//! @brief This file contains only the Doxygen documentation main page.
#ifndef HFST_TWOLC_DOXYGEN_HEADER_H
#define HFST_TWOLC_DOXYGEN_HEADER_H
-//! \mainpage hfst-twolc documentation (under construction)
-//! hfst-twolc is a compiler for two level morphologies (see
+//! \mainpage hfst-twolc documentation (under construction)
+//! hfst-twolc is a compiler for two level morphologies (see
//! [Koskenniemi 1983]). The compiler should be able to compile all grammars
-//! which follow the Xerox twolc documentation (see [Beesley and Karttunen
+//! which follow the Xerox twolc documentation (see [Beesley and Karttunen
//! 2003], [Karttunen et al. 1987]). There is some difficulty in obtaining full
//! compliance with Xerox, since different documents give slightly different
-//! information about the Xerox twolc compiler (e.g. use of the keywords
+//! information about the Xerox twolc compiler (e.g. use of the keywords
//! FREELY and MIXED varies).
//!
//! hfst-twolc is open source and is released under the GPL license ([see Free
//! Software Foundation 2007]).
//!
-//! This documentation is mainly intended for developers of hfst-twolc. It
+//! This documentation is mainly intended for developers of hfst-twolc. It
//! documents the C++ classes used in compiling twolc grammars. The compiler
//! additionally includes flex scanners (see [The Flex Project 2008]) and bison
-//! compilers (see [Free Software Foundation 2010]) which have in file
+//! compilers (see [Free Software Foundation 2010]) which have in file
//! documentation.
//!
-//! Documentation for the end-user of hfst-twolc is available
+//! Documentation for the end-user of hfst-twolc is available
//! <a href="https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstTwolC">here</a>.
//!
-//! Beesely and Karttunen 2003:
+//! Beesely and Karttunen 2003:
//! <a href="http://www.stanford.edu/~laurik/.book2software/twolc.pdf">
//! Two-Level Rule Compiler</a>
//!
-//! The Flex Project 2008:
+//! The Flex Project 2008:
//! <a href="http://flex.sourceforge.net/manual/">Lexical Analysis With Flex
//! </a>
//!
-//! Free SoftWare Foundation 2010:
+//! Free SoftWare Foundation 2010:
//! <a href="http://www.gnu.org/software/bison/manual/bison.html">
//! Bison 2.4.3</a>
//!
-//! Free SoftWare Foundation 2007:
-//! <a href="http://www.gnu.org/licenses/gpl-3.0.html">GNU GENERAL PUBLIC
+//! Free SoftWare Foundation 2007:
+//! <a href="http://www.gnu.org/licenses/gpl-3.0.html">GNU GENERAL PUBLIC
//! LICENSE</a>
//!
-//! Karttunen, Koskenniemi and Kaplan 1987:
+//! Karttunen, Koskenniemi and Kaplan 1987:
//! <a href=
//! "http://www2.parc.com/istl/members/karttune/publications/archive/
//! twolcomp.pdf">A Compiler for Two-level Phonological Rules</a>
-//!
+//!
//! Koskenniemi 1983: <a
//! href="http://www.ling.helsinki.fi/~koskenni/doc/Two-LevelMorphology.pdf">
//! Two-level Morphology:
//! A General Computational Model for Word-Form Recognition and Production</a>,
-//! University of Helsinki.
+//! University of Helsinki.
#endif
diff --git a/tools/src/hfst-twolc/src/hfst-twolc-system.cc b/tools/src/hfst-twolc/src/hfst-twolc-system.cc
index c2d57d0..8fd4ca1 100644
--- a/tools/src/hfst-twolc/src/hfst-twolc-system.cc
+++ b/tools/src/hfst-twolc/src/hfst-twolc-system.cc
@@ -16,12 +16,12 @@ int main(int argc, char* argv[]) {
arguments.append(std::string(" "));
}
- (void)sprintf(command,
+ (void)sprintf(command,
"htwolcpre1.exe %s | "
"htwolcpre2.exe %s | "
- "htwolcpre3.exe %s",
- arguments.c_str(),
- arguments.c_str(),
+ "htwolcpre3.exe %s",
+ arguments.c_str(),
+ arguments.c_str(),
arguments.c_str());
int retval = system(command);
diff --git a/tools/src/hfst-twolc/src/hfst-twolc.cc b/tools/src/hfst-twolc/src/hfst-twolc.cc
new file mode 100644
index 0000000..e3e7691
--- /dev/null
+++ b/tools/src/hfst-twolc/src/hfst-twolc.cc
@@ -0,0 +1,164 @@
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 3 of the Licence.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+#include "commandline_src/CommandLine.h"
+#include "HfstTwolcDefs.h"
+#include "io_src/InputReader.h"
+#include "grammar_defs.h"
+#include "rule_src/TwolCGrammar.h"
+#include "rule_src/OtherSymbolTransducer.h"
+
+namespace hfst {
+ namespace twolcpre1 {
+ int parse();
+ void set_input(std::istream & istr);
+ void set_output(std::ostream & ostr);
+ }
+}
+
+namespace hfst {
+ namespace twolcpre2 {
+ int parse();
+ void set_input(std::istream & istr);
+ void complete_alphabet(void);
+ const HandyDeque<std::string> & get_total_alphabet_symbol_queue();
+ const HandyDeque<std::string> & get_non_alphabet_symbol_queue();
+ }
+}
+
+namespace hfst {
+ namespace twolcpre3 {
+ int parse();
+ void set_input(std::istream & istr);
+ void set_grammar(TwolCGrammar * grammar);
+ TwolCGrammar * get_grammar();
+ void set_silent(bool val);
+ void set_verbose(bool val);
+ }
+}
+
+bool silent=false;
+bool verbose=false;
+
+int main(int argc, char * argv[])
+{
+#ifdef WINDOWS
+ _setmode(0, _O_BINARY);
+ _setmode(1, _O_BINARY);
+#endif
+
+ CommandLine command_line(argc,argv);
+
+ if (command_line.help || command_line.version)
+ {
+ if (command_line.version)
+ { command_line.print_version(); }
+ if (command_line.help)
+ { command_line.print_help(); }
+ exit(0);
+ }
+ if (command_line.usage)
+ {
+ command_line.print_usage();
+ exit(0);
+ }
+ if (! command_line.be_quiet)
+ {
+ if (! command_line.has_input_file)
+ { std::cerr << "Reading input from STDIN." << std::endl; }
+ else
+ { std::cerr << "Reading input from " << command_line.input_file_name
+ << "." << std::endl; }
+ if (! command_line.has_output_file)
+ { std::cerr << "Writing output to STDOUT." << std::endl; }
+ else
+ { std::cerr << "Writing output to " << command_line.output_file_name
+ << "." << std::endl; }
+ }
+ if (command_line.be_verbose)
+ { std::cerr << "Verbose mode." << std::endl; }
+
+ hfst::twolcpre1::set_input(command_line.set_input_file());
+
+ // Test that the output file is okay.
+ (void)command_line.set_output_file();
+ std::ostringstream oss1;
+ hfst::twolcpre1::set_output(oss1);
+ if (hfst::twolcpre1::parse() != 0)
+ {
+ exit(1);
+ }
+
+ std::istringstream iss1(oss1.str());
+ hfst::twolcpre2::set_input(iss1);
+ if (hfst::twolcpre2::parse() != 0)
+ {
+ exit(1);
+ }
+ hfst::twolcpre2::complete_alphabet();
+
+ std::ostringstream oss2;
+ oss2 << hfst::twolcpre2::get_total_alphabet_symbol_queue() << " ";
+ oss2 << hfst::twolcpre2::get_non_alphabet_symbol_queue();
+
+#ifdef DEBUG_TWOLC_3_GRAMMAR
+ hfst::twolcpre3debug = 1;
+#endif
+
+ try
+ {
+ std::istringstream iss2(oss2.str());
+ hfst::twolcpre3::set_input(iss2);
+
+ OtherSymbolTransducer::set_transducer_type(command_line.format);
+ silent = command_line.be_quiet;
+ hfst::twolcpre3::set_silent(silent);
+ verbose = command_line.be_verbose;
+ hfst::twolcpre3::set_verbose(verbose);
+
+ TwolCGrammar twolc_grammar(command_line.be_quiet,
+ command_line.be_verbose,
+ command_line.resolve_left_conflicts,
+ command_line.resolve_right_conflicts);
+ hfst::twolcpre3::set_grammar(&twolc_grammar);
+ int exit_code = hfst::twolcpre3::parse();
+ if (exit_code != 0)
+ { exit(exit_code); }
+
+ if (! command_line.has_output_file)
+ {
+ HfstOutputStream stdout_(command_line.format);
+ hfst::twolcpre3::get_grammar()->compile_and_store(stdout_);
+ }
+ else
+ {
+ HfstOutputStream out
+ (command_line.output_file_name,command_line.format);
+ hfst::twolcpre3::get_grammar()->compile_and_store(out);
+ }
+ exit(0);
+ }
+ catch (const HfstException e)
+ {
+ std::cerr << "This is an hfst interface bug:" << std::endl
+ << e() << std::endl;
+ exit(1);
+ }
+ catch (const char * s)
+ {
+ std::cerr << "This is an a bug probably from sfst:" << std::endl
+ << s << std::endl;
+ exit(1);
+ }
+}
+
+
diff --git a/tools/src/hfst-twolc/src/htwolcpre1-main.cc b/tools/src/hfst-twolc/src/htwolcpre1-main.cc
new file mode 100644
index 0000000..a384a9b
--- /dev/null
+++ b/tools/src/hfst-twolc/src/htwolcpre1-main.cc
@@ -0,0 +1,70 @@
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 3 of the Licence.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+#include "commandline_src/CommandLine.h"
+
+namespace hfst {
+ namespace twolcpre1 {
+ int parse();
+ void set_input(std::istream & istr);
+ void set_output(std::ostream & ostr);
+ void set_warning_stream(std::ostream & ostr);
+ void set_error_stream(std::ostream & ostr);
+ }
+}
+
+int main(int argc, char * argv[])
+{
+#ifdef WINDOWS
+ _setmode(1, _O_BINARY);
+#endif
+
+ CommandLine command_line(argc,argv);
+
+ if (command_line.help || command_line.version)
+ {
+ if (command_line.version)
+ { command_line.print_version(); }
+ if (command_line.help)
+ { command_line.print_help(); }
+ exit(0);
+ }
+ if (command_line.usage)
+ {
+ command_line.print_usage();
+ exit(0);
+ }
+ if (! command_line.be_quiet)
+ {
+ if (! command_line.has_input_file)
+ { std::cerr << "Reading input from STDIN." << std::endl; }
+ else
+ { std::cerr << "Reading input from " << command_line.input_file_name
+ << "." << std::endl; }
+ if (! command_line.has_output_file)
+ { std::cerr << "Writing output to STDOUT." << std::endl; }
+ else
+ { std::cerr << "Writing output to " << command_line.output_file_name
+ << "." << std::endl; }
+ }
+ if (command_line.be_verbose)
+ { std::cerr << "Verbose mode." << std::endl; }
+
+ hfst::twolcpre1::set_input(command_line.set_input_file());
+
+ // Test that the output file is okay.
+ (void)command_line.set_output_file();
+ hfst::twolcpre1::set_output(std::cout);
+ hfst::twolcpre1::set_warning_stream(std::cerr);
+ hfst::twolcpre1::set_error_stream(std::cerr);
+ return hfst::twolcpre1::parse();
+}
diff --git a/tools/src/hfst-twolc/src/htwolcpre2-main.cc b/tools/src/hfst-twolc/src/htwolcpre2-main.cc
new file mode 100644
index 0000000..6cea812
--- /dev/null
+++ b/tools/src/hfst-twolc/src/htwolcpre2-main.cc
@@ -0,0 +1,49 @@
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 3 of the Licence.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+#include "HfstTwolcDefs.h"
+#include "io_src/InputReader.h"
+#include "commandline_src/CommandLine.h"
+#include "grammar_defs.h"
+
+namespace hfst {
+ namespace twolcpre2 {
+ int parse();
+ void set_input(std::istream & istr);
+ void complete_alphabet(void);
+ const HandyDeque<std::string> & get_total_alphabet_symbol_queue();
+ const HandyDeque<std::string> & get_non_alphabet_symbol_queue();
+ void set_warning_stream(std::ostream & ostr);
+ void set_error_stream(std::ostream & ostr);
+ }
+}
+
+int main(int argc, char * argv[])
+{
+#ifdef WINDOWS
+ _setmode(0, _O_BINARY);
+ _setmode(1, _O_BINARY);
+#endif
+
+ CommandLine command_line(argc,argv);
+ if (command_line.help || command_line.usage || command_line.version)
+ { exit(0); }
+ hfst::twolcpre2::set_input(std::cin);
+ hfst::twolcpre2::set_warning_stream(std::cerr);
+ hfst::twolcpre2::set_error_stream(std::cerr);
+
+ int exit_code = hfst::twolcpre2::parse();
+ hfst::twolcpre2::complete_alphabet();
+ std::cout << hfst::twolcpre2::get_total_alphabet_symbol_queue() << " ";
+ std::cout << hfst::twolcpre2::get_non_alphabet_symbol_queue();
+ return exit_code;
+}
diff --git a/tools/src/hfst-twolc/src/htwolcpre3-main.cc b/tools/src/hfst-twolc/src/htwolcpre3-main.cc
new file mode 100644
index 0000000..61f3d8e
--- /dev/null
+++ b/tools/src/hfst-twolc/src/htwolcpre3-main.cc
@@ -0,0 +1,102 @@
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 3 of the Licence.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+#include "io_src/InputReader.h"
+#include "rule_src/TwolCGrammar.h"
+#include "rule_src/OtherSymbolTransducer.h"
+#include "commandline_src/CommandLine.h"
+
+namespace hfst {
+ namespace twolcpre3 {
+ int parse();
+ void set_input(std::istream & istr);
+ void set_grammar(TwolCGrammar * grammar);
+ TwolCGrammar * get_grammar();
+ void set_silent(bool val);
+ void set_verbose(bool val);
+ void message(const std::string &);
+ void set_warning_stream(std::ostream & ostr);
+ void set_error_stream(std::ostream & ostr);
+ }
+}
+
+bool silent=false;
+bool verbose=false;
+
+int main(int argc, char * argv[])
+{
+#ifdef WINDOWS
+ _setmode(0, _O_BINARY);
+ _setmode(1, _O_BINARY);
+#endif
+
+#ifdef DEBUG_TWOLC_3_GRAMMAR
+ htwolcpre3debug = 1;
+#endif
+
+ try
+ {
+ CommandLine command_line(argc,argv);
+ if (command_line.help || command_line.usage || command_line.version)
+ { exit(0); }
+ if (command_line.has_debug_file)
+ { hfst::twolcpre3::set_input(command_line.set_input_file()); }
+ else
+ { hfst::twolcpre3::set_input(std::cin); }
+
+ hfst::twolcpre3::set_warning_stream(std::cerr);
+ hfst::twolcpre3::set_error_stream(std::cerr);
+
+ OtherSymbolTransducer::set_transducer_type(command_line.format);
+ silent = command_line.be_quiet;
+ hfst::twolcpre3::set_silent(silent);
+ verbose = command_line.be_verbose;
+ hfst::twolcpre3::set_verbose(verbose);
+
+ TwolCGrammar twolc_grammar(command_line.be_quiet,
+ command_line.be_verbose,
+ command_line.resolve_left_conflicts,
+ command_line.resolve_right_conflicts);
+ hfst::twolcpre3::set_grammar(&twolc_grammar);
+ int exit_code = hfst::twolcpre3::parse();
+ if (exit_code != 0)
+ { exit(exit_code); }
+
+ hfst::twolcpre3::message("Compiling and storing rules.");
+ if (! command_line.has_output_file)
+ {
+ HfstOutputStream stdout_(command_line.format);
+ hfst::twolcpre3::get_grammar()->compile_and_store(stdout_);
+ }
+ else
+ {
+ HfstOutputStream out
+ (command_line.output_file_name,command_line.format);
+ hfst::twolcpre3::get_grammar()->compile_and_store(out);
+ }
+ exit(0);
+ }
+ catch (const HfstException e)
+ {
+ std::cerr << "This is an hfst interface bug:" << std::endl
+ << e() << std::endl;
+ exit(1);
+ }
+ catch (const char * s)
+ {
+ std::cerr << "This is an a bug probably from sfst:" << std::endl
+ << s << std::endl;
+ exit(1);
+ }
+}
+
+
diff --git a/tools/src/hfst-twolc/src/scanner2.ll b/tools/src/hfst-twolc/src/scanner2.ll
deleted file mode 100644
index 254f055..0000000
--- a/tools/src/hfst-twolc/src/scanner2.ll
+++ /dev/null
@@ -1,294 +0,0 @@
-%option 8Bit batch yylineno noyywrap nounput
-
-%{
- // This program is free software: you can redistribute it and/or modify
- // it under the terms of the GNU General Public License as published by
- // the Free Software Foundation, version 3 of the Licence.
- //
- // This program is distributed in the hope that it will be useful,
- // but WITHOUT ANY WARRANTY; without even the implied warranty of
- // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- // GNU General Public License for more details.
- //
- // You should have received a copy of the GNU General Public License
- // along with this program. If not, see <http://www.gnu.org/licenses/>.
-
- // Autotools stuff
-#ifdef HAVE_CONFIG_H
-# include <config.h>
-#endif
-
-
- // The parser. Supplies flex with its symbol tables.
-#ifdef YACC_USE_PARSER_H_EXTENSION
- #include "htwolcpre2.h"
-#else
- #include "htwolcpre2.hh"
-#endif
-
- // input_defs.h declares the inputHandler, which is
- // an object that overrides flex' default input reading.
- #include "io_src/input_defs.h"
-
- // non_alphabet_symbol_queue is used to store the grammar symbols which are
- // not located in the Alphabet section of the grammar.
- #include "HfstTwolcDefs.h"
- extern HandyDeque<std::string> non_alphabet_symbol_queue;
-
- // alphabet_symbol_queue is used to store the symbols in the Alphabet section
- // of the grammar.
- extern HandyDeque<std::string> alphabet_symbol_queue;
-
- // Tells whether the Alphabet section ended. It ends when the first
- // __HFST_TWOLC_; is seen.
- bool alphabet_ended = false;
-%}
-
-%%
-
-__HFST_TWOLC_Alphabet {
- // Alphabet declaration.
- alphabet_symbol_queue.push_back("__HFST_TWOLC_Alphabet");
- return ALPHABET_DECLARATION;
-}
-
-__HFST_TWOLC_Diacritics {
- // Diacritics declaration.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_Diacritics");
- return DIACRITICS_DECLARATION;
-}
-__HFST_TWOLC_Definitions {
- // Definitions declaration.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_Definitions");
- return DEFINITION_DECLARATION;
-}
-__HFST_TWOLC_Sets {
- // Sets declaration.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_Sets");
- return SETS_DECLARATION;
-}
-__HFST_TWOLC_Rules {
- // Rules declaration.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_Rules");
- return RULES_DECLARATION;
-}
-__HFST_TWOLC_except {
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_except");
- return EXCEPT;
-}
-
-__HFST_TWOLC_KILL_SYMBOL {
- // Signifies a syntax error in the first compilation phase.
- // Just die quietly, since syntax error msgs have been issued by
- // the first compilation phase.
- exit(1);
-}
-[ ] { /* space: ignore */ }
-__HFST_TWOLC_RULE_NAME=\"[^\"]+\" {
- // Rule name.
- non_alphabet_symbol_queue.push_back(yytext);
- return RULE_NAME;
-}
-__HFST_TWOLC_[*] {
- // Kleene star operator.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_*");
- return STAR;
-}
-__HFST_TWOLC_[+] {
- // Kleene plus operator.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_+");
- return PLUS;
-}
-__HFST_TWOLC_[/] {
- // Freely insert operator.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_/");
- return FREELY_INSERT;
-}
-__HFST_TWOLC_[~] {
- // Complement operator.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_~");
- return COMPLEMENT;
-}
-__HFST_TWOLC_[\\] {
- // Term complement operator.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_\\");
- return TERM_COMPLEMENT;
-}
-__HFST_TWOLC_[$][.] {
- // Containment once operator.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_$.");
- return CONTAINMENT_ONCE;
-}
-__HFST_TWOLC_[$] {
- // Containment at least once operator.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_$");
- return CONTAINMENT;
-}
-__HFST_TWOLC_[?] {
- // Any symbol.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_?");
- return QUESTION_MARK;
-}
-__HFST_TWOLC_[0] {
- if (alphabet_ended)
- { non_alphabet_symbol_queue.push_back(yytext); }
- else
- { alphabet_symbol_queue.push_back(yytext); }
- return SYMBOL;
-}
-__HFST_TWOLC_[|] {
- // Or.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_|");
- return UNION;
-}
-__HFST_TWOLC_[&] {
- // And.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_&");
- return INTERSECTION;
-}
-__HFST_TWOLC_[\^] {
- // Power operator.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_^");
- return POWER;
-}
-__HFST_TWOLC_[\-] {
- // Difference operator.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_-");
- return DIFFERENCE;
-}
-__HFST_TWOLC_NUMBER=[0-9]+,[0-9]+ {
- // Number.
- non_alphabet_symbol_queue.push_back(yytext);
- return NUMBER;
-}
-__HFST_TWOLC_NUMBER=[0-9]+ {
- // Number.
- non_alphabet_symbol_queue.push_back(yytext);
- return NUMBER;
-}
-__HFST_TWOLC_[.][#][.] {
- // Word boundary.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_.#.");
- return SYMBOL;
-}
-__HFST_TWOLC_\[ {
- // Beginning of a bracketed regex.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_[");
- return LEFT_SQUARE_BRACKET;
-}
-__HFST_TWOLC_\] {
- // End of a bracketed regex.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_]");
- return RIGHT_SQUARE_BRACKET;
-}
-__HFST_TWOLC_\[\[ {
- // Beginning of a bracketed regex.
- //
- // For some bizarre reason [[ can't occur in the string-literal here...
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_[" "[");
- return RE_LEFT_SQUARE_BRACKET;
-}
-__HFST_TWOLC_\]\] {
- // End of a bracketed regex.
- //
- // For some bizarre reason ]] can't occur in the string-literal here...
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_]" "]");
- return RE_RIGHT_SQUARE_BRACKET;
-}
-__HFST_TWOLC_\( {
- // Beginning of an optional bracketed regex.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_(");
- return LEFT_PARENTHESIS;
-}
-__HFST_TWOLC_\) {
- // End of an optional bracketed regex.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_)");
- return RIGHT_PARENTHESIS;
-}
-__HFST_TWOLC_[/][<][=] {
- // Restriction rule operator.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_/<=");
- return LEFT_RESTRICTION_ARROW;
-}
-__HFST_TWOLC_[<][=] {
- // Left rule operator.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_<=");
- return LEFT_ARROW;
-}
-__HFST_TWOLC_[=][>] {
- // Right rule operator.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_=>");
- return RIGHT_ARROW;
-}
-__HFST_TWOLC_[<][=][>] {
- // Equivalence rule operator.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_<=>");
- return LEFT_RIGHT_ARROW;
-}
-__HFST_TWOLC_[/][<][=][=] {
- // Restriction rule operator.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_/<==");
- return RE_LEFT_RESTRICTION_ARROW;
-}
-__HFST_TWOLC_[<][=][=] {
- // Left rule operator.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_<==");
- return RE_LEFT_ARROW;
-}
-__HFST_TWOLC_[=][=][>] {
- // Right rule operator.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_==>");
- return RE_RIGHT_ARROW;
-}
-__HFST_TWOLC_[<][=][=][>] {
- // Equivalence rule operator.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_<==>");
- return RE_LEFT_RIGHT_ARROW;
-}
-__HFST_TWOLC_[:] {
- if (alphabet_ended)
- { non_alphabet_symbol_queue.push_back("__HFST_TWOLC_:"); }
- else
- { alphabet_symbol_queue.push_back("__HFST_TWOLC_:"); }
- // Pair separator in expressions like "[a:]".
- return PAIR_SEPARATOR;
-}
-__HFST_TWOLC_[;] {
- // End-of-line symbol.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_;");
- // The first __HFST_TWOLC_; ends the alphabet.
- alphabet_ended = true;
- return SEMI_COLON;
-}
-__HFST_TWOLC_[=] {
- // Equals sign used when defining sets and making definitions.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC_=");
- return EQUALS;
-}
-__HFST_TWOLC__ {
- // Center-marker in a rule context.
- non_alphabet_symbol_queue.push_back("__HFST_TWOLC__");
- return CENTER_MARKER;
-}
-__HFST_TWOLC_DIE {
- // If this symbol is seen, pass it on and exit quietly.
- std::cout << "__HFST_TWOLC_DIE";
- exit(1);
-}
-__HFST_TWOLC_SET_NAME=[^ ]+ {
- non_alphabet_symbol_queue.push_back(yytext);
- return SET_NAME;
-}
-__HFST_TWOLC_DEFINITION_NAME=[^ ]+ {
- non_alphabet_symbol_queue.push_back(yytext);
- return DEFINITION_NAME;
-}
-[^ ]+ {
- if (alphabet_ended)
- { non_alphabet_symbol_queue.push_back(yytext); }
- else
- { alphabet_symbol_queue.push_back(yytext); }
- return SYMBOL;
-}
-
-%%
diff --git a/tools/src/hfst-twolc/test/test b/tools/src/hfst-twolc/test/test
index 540b521..dc17682 100755
--- a/tools/src/hfst-twolc/test/test
+++ b/tools/src/hfst-twolc/test/test
@@ -21,59 +21,84 @@ echo "" | ../../hfst-format --test-format sfst > /dev/null
SFST_EXISTS=$?
WINDOWS=1
-if ( uname | egrep "MINGW|mingw" 2>1 > /dev/null); then
+if ( uname | egrep "MINGW|mingw" 2> /dev/null > /dev/null); then
WINDOWS=0
fi
-
+
+if [ -e "../src/hfst-twolc" ]; then
+ USE_HFST_TWOLC=0
+else
+ USE_HFST_TWOLC=1
+fi
for f in $(ls -d $srcdir/* | egrep "test[0-9][0-9]*$" | sort -n)
do
if [ $FOMA_EXISTS -eq 0 ]
then
- if [ $WINDOWS -eq 0 ]
- then
- cat "$f" | ../src/htwolcpre1 -R -s -f foma | \
- ../src/htwolcpre2 -R -s -f foma | \
- ../src/htwolcpre3 -R -s -f foma > temp.twolc.hfst0
- else
- cat "$f" | ../src/hfst-twolc-loc -R -s -f foma > temp.twolc.hfst0
- fi
+ if [ $USE_HFST_TWOLC -eq 0 ]
+ then
+ cat "$f" | ../src/hfst-twolc -R -s -f foma > temp.twolc.hfst0
+ else
+ if [ $WINDOWS -eq 0 ]
+ then
+ cat "$f" | ../src/htwolcpre1 -R -s -f foma | \
+ ../src/htwolcpre2 -R -s -f foma | \
+ ../src/htwolcpre3 -R -s -f foma > temp.twolc.hfst0
+ else
+ cat "$f" | ../src/hfst-twolc-loc -R -s -f foma > temp.twolc.hfst0
+ fi
+ fi
fi
if [ $LOG_OPENFST_EXISTS -eq 0 ]
then
- if [ $WINDOWS -eq 0 ]
- then
- cat "$f" | ../src/htwolcpre1 -R -s -f openfst-log | \
- ../src/htwolcpre2 -R -s -f openfst-log | \
- ../src/htwolcpre3 -R -s -f openfst-log > temp.twolc.hfst1
- else
- cat "$f" | ../src/hfst-twolc-loc -R -s -f openfst-log > temp.twolc.hfst1
- fi
+ if [ $USE_HFST_TWOLC -eq 0 ]
+ then
+ cat "$f" | ../src/hfst-twolc -R -s -f openfst-log > temp.twolc.hfst1
+ else
+ if [ $WINDOWS -eq 0 ]
+ then
+ cat "$f" | ../src/htwolcpre1 -R -s -f openfst-log | \
+ ../src/htwolcpre2 -R -s -f openfst-log | \
+ ../src/htwolcpre3 -R -s -f openfst-log > temp.twolc.hfst1
+ else
+ cat "$f" | ../src/hfst-twolc-loc -R -s -f openfst-log > temp.twolc.hfst1
+ fi
+ fi
fi
if [ $TROPICAL_OPENFST_EXISTS -eq 0 ]
then
- if [ $WINDOWS -eq 0 ]
- then
- cat "$f" | ../src/htwolcpre1 -R -s -f openfst-tropical | \
- ../src/htwolcpre2 -R -s -f openfst-tropical | \
- ../src/htwolcpre3 -R -s -f openfst-tropical > temp.twolc.hfst2
- else
- cat "$f" | ../src/hfst-twolc-loc -R -s -f openfst-tropical > temp.twolc.hfst2
- fi
+ if [ $USE_HFST_TWOLC -eq 0 ]
+ then
+ cat "$f" | ../src/hfst-twolc -R -s -f openfst-tropical > temp.twolc.hfst2
+ else
+ if [ $WINDOWS -eq 0 ]
+ then
+ cat "$f" | ../src/htwolcpre1 -R -s -f openfst-tropical | \
+ ../src/htwolcpre2 -R -s -f openfst-tropical | \
+ ../src/htwolcpre3 -R -s -f openfst-tropical > temp.twolc.hfst2
+ else
+ cat "$f" | ../src/hfst-twolc-loc -R -s -f openfst-tropical > temp.twolc.hfst2
+ fi
+ fi
fi
if [ $SFST_EXISTS -eq 0 ]
then
- if [ $WINDOWS -eq 0 ]
- then
- cat "$f" | ../src/htwolcpre1 -R -s -f sfst | \
- ../src/htwolcpre2 -R -s -f sfst | \
- ../src/htwolcpre3 -R -s -f sfst > temp.twolc.hfst3
- else
- cat "$f" | ../src/hfst-twolc-loc -R -s -f sfst > temp.twolc.hfst3
- fi
+ if [ $USE_HFST_TWOLC -eq 0 ]
+ then
+ cat "$f" | ../src/hfst-twolc -R -s -f sfst > temp.twolc.hfst3
+ else
+ if [ $WINDOWS -eq 0 ]
+ then
+ cat "$f" | ../src/htwolcpre1 -R -s -f sfst | \
+ ../src/htwolcpre2 -R -s -f sfst | \
+ ../src/htwolcpre3 -R -s -f sfst > temp.twolc.hfst3
+ else
+ cat "$f" | ../src/hfst-twolc-loc -R -s -f sfst > temp.twolc.hfst3
+ fi
+ fi
fi
if [ $? -ne 0 ]
@@ -88,14 +113,19 @@ do
if [ $? -eq 0 ]
then
cat temp.twolc.hfst"$n" | ../../hfst-fst2txt | ../../hfst-txt2fst -e"@_EPSILON_SYMBOL_@" > temp.twolc.hfst
- if [ $WINDOWS -eq 0 ]
- then
- cat "$f" | ../src/htwolcpre1 -R -s | \
- ../src/htwolcpre2 -R -s | \
- ../src/htwolcpre3 -R -s > /dev/null 2>&1
- else
- cat "$f" | ../src/hfst-twolc-loc -R -s > /dev/null 2>&1
- fi
+ if [ $USE_HFST_TWOLC -eq 0 ]
+ then
+ cat "$f" | ../src/hfst-twolc -R -s > /dev/null 2>&1
+ else
+ if [ $WINDOWS -eq 0 ]
+ then
+ cat "$f" | ../src/htwolcpre1 -R -s | \
+ ../src/htwolcpre2 -R -s | \
+ ../src/htwolcpre3 -R -s > /dev/null 2>&1
+ else
+ cat "$f" | ../src/hfst-twolc-loc -R -s > /dev/null 2>&1
+ fi
+ fi
if ! ../../hfst-compare -q -1 temp.hfst -2 temp.twolc.hfst > /dev/null 2>&1
then
echo "hfst-twolc sub$(basename $f) failed."
diff --git a/tools/src/hfst-txt2fst.cc b/tools/src/hfst-txt2fst.cc
index 1c60826..723d2db 100644
--- a/tools/src/hfst-txt2fst.cc
+++ b/tools/src/hfst-txt2fst.cc
@@ -81,7 +81,7 @@ print_usage()
" -C, --check-negative-epsilon-cycles Issue a warning if there are epsilon cycles\n"
" with a negative weight in the transducer\n");
fprintf(message_out, "\n");
- fprintf(message_out,
+ fprintf(message_out,
"If OUTFILE or INFILE is missing or -, standard streams will be used.\n"
"If FMT is not given, OpenFst's tropical format will be used.\n"
"The possible values for FMT are { foma, openfst-tropical, openfst-log,\n"
@@ -111,14 +111,14 @@ parse_options(int argc, char** argv)
// add tool-specific options here
{"epsilon", required_argument, 0, 'e'},
{"number", no_argument, 0, 'n'},
- {"format", required_argument, 0, 'f'},
- {"prolog", no_argument, 0, 'p'},
- {"check-negative-epsilon-cycles", no_argument, 0, 'C'},
+ {"format", required_argument, 0, 'f'},
+ {"prolog", no_argument, 0, 'p'},
+ {"check-negative-epsilon-cycles", no_argument, 0, 'C'},
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT "e:nf:pC",
long_options, &option_index);
if (-1 == c)
@@ -155,7 +155,7 @@ break;
if (epsilonname == NULL)
{
epsilonname = hfst_strdup("@0@");
- verbose_printf("Using default epsilon representation %s\n",
+ verbose_printf("Using default epsilon representation %s\n",
epsilonname);
}
if (output_format == hfst::UNSPECIFIED_TYPE)
@@ -213,7 +213,7 @@ process_stream(HfstOutputStream& outstream)
}
try {
- HfstBasicTransducer fsm =
+ HfstBasicTransducer fsm =
HfstBasicTransducer::read_in_prolog_format(inputfile, linecount);
if (check_negative_epsilon_cycles)
@@ -242,7 +242,7 @@ process_stream(HfstOutputStream& outstream)
return EXIT_FAILURE;
}
}
- else
+ else
{
try {
HfstTransducer t(inputfile,
@@ -280,7 +280,7 @@ process_stream(HfstOutputStream& outstream)
}
-int main( int argc, char **argv )
+int main( int argc, char **argv )
{
#ifdef WINDOWS
_setmode(1, _O_BINARY);
@@ -298,7 +298,7 @@ int main( int argc, char **argv )
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
switch (output_format)
{
diff --git a/tools/src/hfst-unary-tool.cc b/tools/src/hfst-unary-tool.cc
index 6f31a52..54244bd 100644
--- a/tools/src/hfst-unary-tool.cc
+++ b/tools/src/hfst-unary-tool.cc
@@ -77,12 +77,12 @@ parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
- // add tool-specific options here
+ // add tool-specific options here
{0,0,0,0}
};
int option_index = 0;
- // add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ // add tool-specific options here
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT,
long_options, &option_index);
if (-1 == c)
@@ -115,11 +115,11 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
transducer_n++;
if (transducer_n==1)
{
- verbose_printf("Doing things %s...\n", inputfilename);
+ verbose_printf("Doing things %s...\n", inputfilename);
}
else
{
- verbose_printf("Doing things %s..." SIZE_T_SPECIFIER "\n", inputfilename, transducer_n);
+ verbose_printf("Doing things %s..." SIZE_T_SPECIFIER "\n", inputfilename, transducer_n);
}
HfstTransducer trans(instream);
@@ -147,7 +147,7 @@ int main( int argc, char **argv ) {
{
fclose(outfile);
}
- verbose_printf("Reading from %s, writing to %s\n",
+ verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/inc/check-params-unary.h b/tools/src/inc/check-params-unary.h
index dd626f4..5563eaf 100644
--- a/tools/src/inc/check-params-unary.h
+++ b/tools/src/inc/check-params-unary.h
@@ -15,8 +15,8 @@
if ((argc - optind) == 1)
{
inputfilename = hfst_strdup(argv[optind]);
- inputfile = hfst_fopen(inputfilename, "r");
- if (inputfile == stdin)
+ inputfile = ::hfst_fopen(inputfilename, "r");
+ if (inputfile == stdin)
{
free(inputfilename);
inputfilename = hfst_strdup("<stdin>");
@@ -24,7 +24,7 @@
}
else if ((argc - optind) > 1)
{
- error(EXIT_FAILURE, 0,
+ error(EXIT_FAILURE, 0,
"no more than one transducer file may be given");
}
else
@@ -33,11 +33,11 @@
inputfilename = hfst_strdup("<stdin>");
}
}
- else
+ else
{
if ((argc - optind) > 0)
{
- error(EXIT_FAILURE, 0,
+ error(EXIT_FAILURE, 0,
"no more than one transducer filename may be given");
}
}
diff --git a/tools/src/inc/getopt-cases-common.h b/tools/src/inc/getopt-cases-common.h
index 6db29ae..fa7c83a 100644
--- a/tools/src/inc/getopt-cases-common.h
+++ b/tools/src/inc/getopt-cases-common.h
@@ -32,8 +32,8 @@ case 's':
break;
case 'o':
outfilename = hfst_strdup(optarg);
- outfile = hfst_fopen(outfilename, "w");
- if (outfile == stdout)
+ outfile = ::hfst_fopen(outfilename, "w");
+ if (outfile == stdout)
{
free(outfilename);
outfilename = hfst_strdup("<stdout>");
diff --git a/tools/src/inc/getopt-cases-unary.h b/tools/src/inc/getopt-cases-unary.h
index 828bcd8..de00a5b 100644
--- a/tools/src/inc/getopt-cases-unary.h
+++ b/tools/src/inc/getopt-cases-unary.h
@@ -13,8 +13,8 @@
// common-unary-cases.h
case 'i':
inputfilename = hfst_strdup(optarg);
- inputfile = hfst_fopen(inputfilename, "r");
- if (inputfile == stdin)
+ inputfile = ::hfst_fopen(inputfilename, "r");
+ if (inputfile == stdin)
{
free(inputfilename);
inputfilename = hfst_strdup("<stdin>");
diff --git a/tools/src/inc/globals-common.h b/tools/src/inc/globals-common.h
index 541a42a..9b9a277 100644
--- a/tools/src/inc/globals-common.h
+++ b/tools/src/inc/globals-common.h
@@ -24,14 +24,14 @@ char* outfilename = 0;
FILE* outfile = 0;
bool outputNamed = false;
-// Windows-specific globals
+// Windows-specific globals
#ifdef WINDOWS
#ifndef _WINDOWS_STREAM_BINARY_MODE
#define _WINDOWS_STREAM_BINARY_MODE
#include <fcntl.h>
// Make sure that all standard streams are in binary mode.
// Line feed and carriage return characters are not handled correctly
-// in text mode.
+// in text mode.
int _CRT_fmode = _O_BINARY;
#endif // _WINDOWS_STREAM_BINARY_MODE
#endif // WINDOWS
diff --git a/tools/src/lexc-readline-ui.cc b/tools/src/lexc-readline-ui.cc
index 3162fad..5bbe60a 100644
--- a/tools/src/lexc-readline-ui.cc
+++ b/tools/src/lexc-readline-ui.cc
@@ -54,7 +54,7 @@ static HfstTransducerVector rules_;
void
lexc_banner()
{
- fprintf(message_out,
+ fprintf(message_out,
" *******************************************************\n"
" * Helsinki FS Lexicon Compiler %s (" PACKAGE_VERSION ") *\n"
" * cloned from Xerox original by *\n"
diff --git a/tools/src/parsers/Makefile.am b/tools/src/parsers/Makefile.am
index 220a7fc..ab13466 100644
--- a/tools/src/parsers/Makefile.am
+++ b/tools/src/parsers/Makefile.am
@@ -17,37 +17,15 @@
SUBDIRS=test
-#hfst_xfst_SOURCES = xfst-lexer.ll xfst-parser.yy xfst-utils.cc XfstCompiler.cc hfst-xfst.cc $(HFST_COMMON_SRC)
hfst_xfst_SOURCES = hfst-xfst.cc $(HFST_COMMON_SRC)
-#BUILT_SOURCES=xfst-parser.cc xfst-lexer.cc
-#CLEANFILES=xfst-parser.cc xfst-lexer.cc xfst-parser.hh
-
-#CLEANFILES+=xfst-parser.hh
-
-#AM_YFLAGS=-d
-#AM_LFLAGS=--outfile=$(LEX_OUTPUT_ROOT).c
-
AM_CPPFLAGS = -I${top_srcdir}/libhfst/src -I${top_srcdir}/libhfst/src/parsers -I${top_srcdir}/tools/src $(GLIB_CPPFLAGS) -Wno-deprecated
-AM_CPPFLAGS += -I${top_srcdir}/back-ends/foma \
- -I${top_srcdir}/back-ends
-
-if WANT_MINGW
- AM_CPPFLAGS += -I${top_srcdir}/back-ends/openfstwin/src/include \
- -I${top_srcdir}/back-ends/dlfcn -DWINDOWS
-else
- AM_CPPFLAGS += -I${top_srcdir}/back-ends/openfst/src/include
-endif
-
-
if WANT_XFST
MAYBE_XFST=hfst-xfst
endif
-#noinst_HEADERS = XfstCompiler.h xfst-utils.h xfst-parser.hh
-
-HFST_COMMON_SRC=../hfst-program-options.cc ../hfst-commandline.cc ../hfst-tool-metadata.cc ../hfst-file-to-mem.cc # ../hfst-string-conversions.cc xfst_help_message.cc
+HFST_COMMON_SRC=../hfst-program-options.cc ../hfst-commandline.cc ../hfst-tool-metadata.cc ../hfst-file-to-mem.cc
bin_PROGRAMS=$(MAYBE_XFST)
@@ -57,11 +35,5 @@ else
hfst_xfst_LDADD = $(top_builddir)/libhfst/src/libhfst.la $(GLIB_LIBS)
endif
-#xfst-parser.h: xfst-parser.cc
-
-#xfst-parser.cc: xfst-parser.yy
-
-#xfst-lexer.cc: xfst-lexer.ll
-
-EXTRA_DIST=init_help.cc cmd.h abbrcmd.h # xfst_help_message.h xfst-parser.cc xfst-lexer.cc xfst-parser.hh
+EXTRA_DIST=init_help.cc cmd.h abbrcmd.h
diff --git a/tools/src/parsers/abbrcmd.h b/tools/src/parsers/abbrcmd.h
index cc0df1c..1022cca 100644
--- a/tools/src/parsers/abbrcmd.h
+++ b/tools/src/parsers/abbrcmd.h
@@ -1,4 +1,6 @@
-char * abbrcmd [] =
+#ifndef GUARD_abbrcmd_h
+#define GUARD_abbrcmd_h
+const char * abbrcmd [] =
{
"add",
"aliases",
@@ -133,3 +135,4 @@ char * abbrcmd [] =
"zero-plus",
NULL
};
+#endif // GUARD_abbrcmd_h
diff --git a/tools/src/parsers/cmd.h b/tools/src/parsers/cmd.h
index edc7b3a..cb7ca19 100644
--- a/tools/src/parsers/cmd.h
+++ b/tools/src/parsers/cmd.h
@@ -1,4 +1,6 @@
-char * cmd [] =
+#ifndef GUARD_cmd_h
+#define GUARD_cmd_h
+const char * cmd [] =
{
"add properties",
"alias",
@@ -129,3 +131,4 @@ char * cmd [] =
"zero-plus net",
NULL,
};
+#endif // GUARD_cmd_h
diff --git a/tools/src/parsers/foma_commands b/tools/src/parsers/foma_commands
index 5bf10c8..6e542dd 100644
--- a/tools/src/parsers/foma_commands
+++ b/tools/src/parsers/foma_commands
@@ -55,15 +55,15 @@ quit
quit-on-fail
read att
read cmatrix
-read lexc
+read lexc
read prolog
-read spaced-text
-read text
+read spaced-text
+read text
regex
reverse net
rotate stack
-save defined
-save stack
+save defined
+save stack
set <ON|OFF>
show variables
shuffle net
@@ -71,10 +71,10 @@ sigma net
sort in
sort net
sort out
-source
+source
substitute defined X for Y
substitute symbol X for Y
-system
+system
test equivalent
test functional
test identity
@@ -87,7 +87,7 @@ test unambiguous
test upper-universal
turn stack
twosided flag-diacritics
-undefine
+undefine
union net
upper-side net
variable att-epsilon
diff --git a/tools/src/parsers/foma_commands.help b/tools/src/parsers/foma_commands.help
index fd3731e..a889b82 100644
--- a/tools/src/parsers/foma_commands.help
+++ b/tools/src/parsers/foma_commands.help
@@ -71,7 +71,7 @@ source <file> read and compile script file
sort net sorts arcs topologically on top FSM
sort in sorts input arcs by sigma numbers on top FSM
sort out sorts output arcs by sigma number on top FSM
-substitute defined X for Y substitutes defined network X at all arcs containing Y
+substitute defined X for Y substitutes defined network X at all arcs containing Y
substitute symbol X for Y substitutes all occurrences of Y in an arc with X
system <cmd> execute a system command
test unambiguous test if top FST is unambiguous
diff --git a/tools/src/parsers/hfst-xfst.cc b/tools/src/parsers/hfst-xfst.cc
index 0ae48bc..ddb127a 100644
--- a/tools/src/parsers/hfst-xfst.cc
+++ b/tools/src/parsers/hfst-xfst.cc
@@ -61,8 +61,8 @@ static bool print_weight = false;
void
print_usage()
{
- // c.f. http://www.gnu.org/prep/standards/standards.html#g_t_002d_002dhelp
- // Usage line
+ // c.f. http://www.gnu.org/prep/standards/standards.html#g_t_002d_002dhelp
+ // Usage line
fprintf(message_out, "Usage: %s [OPTIONS...]\n"
"Compile XFST scripts or execute XFST commands interactively\n"
"\n", "hfst-xfst" /*program_name*/);
@@ -70,8 +70,8 @@ print_usage()
print_common_program_options(message_out);
fprintf(message_out, "\n");
fprintf(message_out, "Xfst-specific options:\n");
- fprintf(message_out,
- " -e, --execute=CMD Execute command CMD on startup\n"
+ fprintf(message_out,
+ " -e, --execute=CMD Execute command CMD on startup\n"
" -f, --format=FMT Write result using FMT as backend format\n"
" -F, --scriptfile=FILE Read commands from FILE, and quit\n"
" -l, --startupfile=FILE Read commands from FILE on startup\n"
@@ -107,7 +107,7 @@ int
parse_options(int argc, char** argv)
{
extend_options_getenv(&argc, &argv);
- // use of this function requires options are settable on global scope
+ // use of this function requires options are settable on global scope
while (true)
{
static const struct option long_options[] =
@@ -126,7 +126,7 @@ parse_options(int argc, char** argv)
};
int option_index = 0;
// add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT "f:F:e:l:p::rwk",
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT "f:F:e:l:p::rwk",
long_options, &option_index);
if (-1 == c)
{
@@ -218,7 +218,7 @@ parse_options(int argc, char** argv)
int parse_file(const char* filename, hfst::xfst::XfstCompiler &comp)
{
char* line = hfst_file_to_mem(filename);
- if (NULL == line)
+ if (NULL == line)
{
error(EXIT_FAILURE, 0, "error when reading file %s\n", filename);
return EXIT_FAILURE;
@@ -237,7 +237,7 @@ void insert_zeroes(char * array, unsigned int number)
{
for (unsigned int i=0; i<number; i++)
{
- array[i] = '\0';
+ array[i] = '\0';
}
}
@@ -275,7 +275,7 @@ static int smatch;
// By Mans Hulden
char *my_generator(const char *text, int state) {
static int list_index, list_index2, len, nummatches;
- char *name;
+ const char *name;
text = rl_line_buffer;
if (!state) {
list_index = 0;
@@ -288,7 +288,7 @@ char *my_generator(const char *text, int state) {
list_index++;
if (strncmp (name, text, len) == 0) {
- nummatches++;
+ nummatches++;
return(strdup(name+smatch));
}
}
@@ -407,7 +407,7 @@ int main(int argc, char** argv)
}
}
- if (pipe_input)
+ if (pipe_input)
{
verbose_printf("Reading from standard input...\n");
comp.setReadInteractiveTextFromStdin(false);
@@ -458,7 +458,7 @@ int main(int argc, char** argv)
#else
char line [MAX_LINE_LENGTH];
insert_zeroes(line, MAX_LINE_LENGTH);
- while (cin.getline(line, MAX_LINE_LENGTH))
+ while (std::cin.getline(line, MAX_LINE_LENGTH))
{
std::string linestr(line);
expression += linestr;
@@ -525,6 +525,8 @@ int main(int argc, char** argv)
if (buf[0] != 0) {
add_history(expression.c_str()); }
+ free(buf);
+
if (0 != comp.parse_line((expression + "\n").c_str()))
{
#ifdef WINDOWS
@@ -547,14 +549,13 @@ int main(int argc, char** argv)
free(promptline);
promptline = (!silent) ? comp.get_prompt() : strdup("");
}
- free(buf);
+
free(promptline);
#else
fprintf(stderr, "ERROR: missing readline library\n");
return EXIT_FAILURE;
#endif
}
-
return EXIT_SUCCESS;
}
diff --git a/tools/src/parsers/name2cmd.h b/tools/src/parsers/name2cmd.h
index ab19452..fec41de 100644
--- a/tools/src/parsers/name2cmd.h
+++ b/tools/src/parsers/name2cmd.h
@@ -182,7 +182,7 @@ namespace hfst {
xfst_mapping(PRINT_SHORTEST_STRING_SIZE_CMD, "print shortest-string-size, print shortest-string-length, "
"shortest-string-size, shortest-string-length, psz");
xfst_mapping(PRINT_SHORTEST_STRING_CMD, "print shortest-string, shortest-string, pss");
- xfst_mapping(PRINT_SIGMA_CMD, "print sigma, sigma");
+ xfst_mapping(PRINT_SIGMA_CMD, "print sigma, sigma");
xfst_mapping(PRINT_SIGMA_COUNT_CMD, "print sigma-tally, sigma-tally, sitally, print sigma-word-tally");
xfst_mapping(PRINT_SIZE_CMD, "print size, size");
xfst_mapping(PRINT_STACK_CMD, "print stack, stack");
diff --git a/tools/src/parsers/test/Makefile.am b/tools/src/parsers/test/Makefile.am
index 908dd76..31d3a86 100644
--- a/tools/src/parsers/test/Makefile.am
+++ b/tools/src/parsers/test/Makefile.am
@@ -157,6 +157,7 @@ EXTRA_DIST=test.sh \
replace_test_51.xfst replace_test_51.output \
replace_test_52.xfst replace_test_52.output \
replace_test_53.xfst replace_test_53.output \
+ replace_test_flags_1.xfst replace_test_flags_1.att \
weighted_parallel_rules_1.xfst weighted_parallel_rules_1.output \
weighted_parallel_rules_2.xfst weighted_parallel_rules_2.output \
weighted_parallel_rules_3.xfst weighted_parallel_rules_3.output \
@@ -179,8 +180,8 @@ EXTRA_DIST=test.sh \
replace_markup_5.att replace_markup_5.xfst \
replace_markup_6.att replace_markup_6.xfst \
replace_markup_7.att replace_markup_7.xfst \
- priority_union_1.att priority_union_1.att \
- priority_union_2.att priority_union_2.att \
- priority_union_3.att priority_union_3.att \
- priority_union_4.att priority_union_4.att
+ priority_union_1.att priority_union_1.xfst \
+ priority_union_2.att priority_union_2.xfst \
+ priority_union_3.att priority_union_3.xfst \
+ priority_union_4.att priority_union_4.xfst
check_DATA=test.sh
diff --git a/tools/src/parsers/test/eliminate_flag.output b/tools/src/parsers/test/eliminate_flag.output
index 03e1f70..3972a89 100644
--- a/tools/src/parsers/test/eliminate_flag.output
+++ b/tools/src/parsers/test/eliminate_flag.output
@@ -6,3 +6,8 @@
1, (1 = TRUE, 0 = FALSE)
1, (1 = TRUE, 0 = FALSE)
1, (1 = TRUE, 0 = FALSE)
+1, (1 = TRUE, 0 = FALSE)
+1, (1 = TRUE, 0 = FALSE)
+1, (1 = TRUE, 0 = FALSE)
+1, (1 = TRUE, 0 = FALSE)
+1, (1 = TRUE, 0 = FALSE)
diff --git a/tools/src/parsers/test/eliminate_flag.xfst b/tools/src/parsers/test/eliminate_flag.xfst
index f2a1e83..e3d01ce 100644
--- a/tools/src/parsers/test/eliminate_flag.xfst
+++ b/tools/src/parsers/test/eliminate_flag.xfst
@@ -31,3 +31,48 @@ eliminate flags
equivalent
regex [ [x|X] a [x|X] ] | [ [y|Y|z|Z] a [y|Y|z|Z] ];
equivalent
+#
+#
+clear stack
+#
+#
+regex a "@P.FOO.ON@" "@P.BAR.ON@" "@D.FOO.ON@";
+eliminate flags
+regex 0 - 0;
+equivalent
+#
+#
+clear stack
+#
+#
+regex "@U.FOO.ON@" a:b "@U.BAR.ON@";
+eliminate flag FOO
+regex a:b "@U.BAR.ON@";
+equivalent
+#
+#
+clear stack
+#
+#
+regex "@U.FOO.ON@" a:b "@U.BAR.ON@";
+eliminate flag BAR
+regex "@U.FOO.ON@" a:b;
+equivalent
+#
+#
+clear stack
+#
+#
+regex "@U.FOO.ON@" a:b "@U.BAR.ON@" c:d "@U.FOO.OFF@" e:f "@U.BAR.ON@";
+eliminate flag FOO
+regex 0 - 0;
+equivalent
+#
+#
+clear stack
+#
+#
+regex "@U.FOO.ON@" a:b "@U.BAR.ON@" c:d "@U.FOO.OFF@" e:f "@U.BAR.ON@";
+eliminate flag BAR
+regex "@U.FOO.ON@" a:b c:d "@U.FOO.OFF@" e:f;
+equivalent
diff --git a/tools/src/parsers/test/test.sh b/tools/src/parsers/test/test.sh
index 09f0c4f..1ee3566 100755
--- a/tools/src/parsers/test/test.sh
+++ b/tools/src/parsers/test/test.sh
@@ -17,7 +17,7 @@ TEST="test"
TR="tr"
# ECHO="echo" # present on all platforms?
-EXTRA_FILES="tmp startup" # files generated by tests
+EXTRA_FILES="tmp startup tmpdiff test" # files generated by tests
# edit this is hfst-format is not available
for format in sfst openfst-tropical foma;
@@ -27,6 +27,10 @@ do
continue;
fi
+ echo ""
+ echo "Performing hfst-xfst tests for "$format" format..."
+ echo ""
+
## Create a transducer [Foo Bar Baz] where Foo is [foo], Bar [bar] and Baz [Baz].
## Definition of Foo is given in startup file, and definitions of Bar and Baz
## on command line. Baz is later undefined in input.
@@ -89,11 +93,9 @@ do
exit 1;
fi
if (! ${GREP} "3 states" tmp > /dev/null); then
- echo "here 1"
exit 1;
fi
if (! ${GREP} "9 arcs" tmp > /dev/null); then
- echo "here 2"
exit 1;
fi
@@ -101,7 +103,7 @@ do
## is the same as testfile.att using att-to-fst conversion.
for testfile in compose_net concatenate_net union_net ignore_net invert_net minus_net intersect_net \
determinize_net epsilon_remove_net invert_net minimize_net negate_net \
- one_plus_net prune_net reverse_net sort_net upper_side_net zero_plus_net lower_side_net \
+ one_plus_net reverse_net upper_side_net zero_plus_net lower_side_net \
define define_function prolog \
substitute_symbol_1 substitute_symbol_2 substitute_symbol_3 \
substitute_symbol_4 substitute_symbol_5 \
@@ -112,17 +114,16 @@ do
at_re_1 at_re_2 at_re_3 at_txt at_stxt at_txt_and_stxt at_pl \
quoted_literals replace_identity one_transition_regex merge \
replace_test_flags_1 \
- replace_markup_1 replace_markup_2 replace_markup_3 replace_markup_4 replace_markup_5\
+ replace_markup_1 replace_markup_2 replace_markup_3 replace_markup_4 replace_markup_5 \
replace_markup_6 replace_markup_7 \
priority_union_1 priority_union_2 priority_union_3 priority_union_4 \
-
-
- #replace_test_flags_2
+
+ # replace_test_flags_2
# substitute_symbol_6 fails on sfst, substitute_symbol_7 substitute_symbol_8 should be added
# angle_brackets omitted, since xfst and foma handle them differently
do
${REMOVE} result result1 result2
- if ! (${LS} $testfile.xfst 2> /dev/null); then
+ if ! (${LS} $testfile.xfst > /dev/null 2> /dev/null); then
echo "skipping missing test for "$testfile"..."
continue
fi
@@ -138,6 +139,7 @@ do
echo "ERROR: "$testfile" test failed"
exit 1;
fi
+ echo "* hfst-xfst subtest "$testfile" passed"
done
## The same as above but only for openfst format
@@ -145,7 +147,7 @@ do
for testfile in merge_weighted
do
${REMOVE} result result1 result2
- if ! (${LS} $testfile.xfst 2> /dev/null); then
+ if ! (${LS} $testfile.xfst > /dev/null 2> /dev/null); then
echo "skipping missing test for "$testfile"..."
continue
fi
@@ -161,6 +163,7 @@ do
echo "ERROR: "$testfile" test failed"
exit 1;
fi
+ echo "* hfst-xfst subtest "$testfile" passed"
done
fi
@@ -185,15 +188,15 @@ do
## Test that the result of testfile.xfst (written to standard output)
## is the same as testfile.output
for testfile in print_stack print_labels print_label_tally \
- shortest_string set_variable info print_net eliminate_flag empty_context xerox_composition \
+ shortest_string set_variable print_net eliminate_flag empty_context xerox_composition \
compile_replace_1 compile_replace_2 compile_replace_3 flag_with_unknown
do
- if ! (${LS} $testfile.xfst 2> /dev/null); then
+ if ! (${LS} $testfile.xfst > /dev/null 2> /dev/null); then
echo "skipping missing test for "$testfile"..."
continue
fi
# apply up/down leak to stdout with readline..
- if ! (${CAT} $testfile.xfst | ../hfst-xfst --pipe-mode=input -f $format -s | ${TR} -d '\r' > tmp); then
+ if ! (${CAT} $testfile.xfst | ../hfst-xfst --pipe-mode=input -f $format -s 2> /dev/null | ${TR} -d '\r' > tmp); then
echo "ERROR: in compiling "$testfile.xfst
exit 1;
fi
@@ -212,12 +215,13 @@ do
fi
${REMOVE} tmpdiff
fi
+ echo "* hfst-xfst subtest "$testfile" passed"
done
## Interactive commands
for testfile in apply_up apply_down inspect_net
do
- if ! (${LS} $testfile.xfst 2> /dev/null); then
+ if ! (${LS} $testfile.xfst > /dev/null 2> /dev/null); then
echo "skipping missing test for "$testfile"..."
continue
fi
@@ -248,6 +252,7 @@ do
${REMOVE} tmpdiff
fi
done
+ echo "* hfst-xfst subtest "$testfile" passed"
done
@@ -257,7 +262,7 @@ do
do
for testfile in test_overlap test_sublanguage # the function to be tested
do
- if ! (${LS} $testfile$testcase.xfst 2> /dev/null); then
+ if ! (${LS} $testfile$testcase.xfst > /dev/null 2> /dev/null); then
echo "skipping missing test for "$testfile$testcase"..."
continue
fi
@@ -286,6 +291,7 @@ do
echo "ERROR: in testing "$file
exit 1;
fi
+ echo "* hfst-xfst subtest "$file" passed"
done
if [ "$format" = "openfst-tropical" ];
@@ -302,7 +308,7 @@ do
weighted_ltr_longest_match_1 weighted_ltr_longest_match_2 weighted_ltr_longest_match_3 \
weighted_ltr_shortest_match_1 weighted_ltr_shortest_match_2 weighted_ltr_shortest_match_3
do
- if ! (${LS} $file.xfst 2> /dev/null); then
+ if ! (${LS} $file.xfst > /dev/null 2> /dev/null); then
echo "skipping missing test for "$file"..."
continue
fi
@@ -322,6 +328,7 @@ do
echo "ERROR: "$file" test failed"
exit 1;
fi
+ echo "* hfst-xfst subtest "$file" passed"
done
for file in contains contains_with_weight contains_once contains_once_optional \
replace_test_1 replace_test_2 replace_test_3 replace_test_4 replace_test_5 \
@@ -341,7 +348,7 @@ do
weighted_parallel_rules_10 weighted_parallel_rules_11 weighted_parallel_rules_12 \
weighted_parallel_rules_13
do
- if ! (${LS} $file.xfst 2> /dev/null); then
+ if ! (${LS} $file.xfst > /dev/null 2> /dev/null); then
echo "skipping missing test for "$file"..."
continue
fi
@@ -353,9 +360,11 @@ do
echo "ERROR: "$file" test failed"
exit 1;
fi
+ echo "* hfst-xfst subtest "$file" passed"
done
fi
+ echo ""
${REMOVE} result tmp1 tmp2 foo
## add properties
diff --git a/tools/src/parsers/xfst_commands.h b/tools/src/parsers/xfst_commands.h
index 0779447..e248808 100644
--- a/tools/src/parsers/xfst_commands.h
+++ b/tools/src/parsers/xfst_commands.h
@@ -1,6 +1,6 @@
namespace hfst {
namespace xfst {
- enum XfstCommand {
+ enum XfstCommand {
ADD_PROPS_CMD,
APPLY_UP_CMD,
APPLY_DOWN_CMD,
diff --git a/tools/src/sfst-main.cc b/tools/src/sfst-main.cc
new file mode 100644
index 0000000..cc611f5
--- /dev/null
+++ b/tools/src/sfst-main.cc
@@ -0,0 +1,248 @@
+#ifdef WINDOWS
+#include <io.h>
+#endif
+
+#include <stdio.h>
+#include "SfstCompiler.h"
+#include "HfstTransducer.h"
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <getopt.h>
+
+#include "hfst-commandline.h"
+#include "hfst-program-options.h"
+
+#include "inc/globals-common.h"
+#include "inc/globals-unary.h"
+
+bool verbose_ = false;
+std::string filename = "";
+std::string foldername = "";
+
+using std::cerr;
+using namespace hfst;
+
+int switch_=0;
+hfst::ImplementationType output_format = hfst::ImplementationType::ERROR_TYPE;
+
+int Compact=0;
+int LowMem=0;
+
+void
+print_usage()
+{
+ // c.f. http://www.gnu.org/prep/standards/standards.html#g_t_002d_002dhelp
+ fprintf(message_out, "Usage: %s %s [OPTIONS...] [INFILE]\n"
+ "Compile a file written with SFST programming language into a transducer.\n"
+ "\n", program_name, "(alias hfst-calculate)");
+ print_common_program_options(message_out);
+ fprintf(message_out, "Input/Output options:\n"
+ " -i, --input=INFILE Read input from INFILE\n"
+ " -o, --output=OUTFILE Write output transducer to OUTFILE\n");
+ fprintf(message_out, "String and format options:\n"
+ " -f, --format=FMT Write result in FMT format\n");
+ fprintf(message_out, "\n");
+
+ fprintf(message_out,
+ "If OUTFILE or INFILE is missing or -, standard streams will be used.\n"
+ "FMT must be one of the following: "
+ "{foma, sfst, openfst-tropical, openfst-log}\n"
+ );
+ fprintf(message_out, "\n");
+ fprintf(message_out, "Examples:\n"
+ " echo \"[a-z]*\" | %s -f foma > az.foma\n"
+ " create a foma transducer accepting any number of consecutive\n"
+ " characters between a and z, inclusive\n"
+ "\n"
+ " echo \"a:b (a:<>)+\" > ab.sfst-pl ; %s -f openfst-tropical \\\n"
+ " -i ab.sfst-pl -o ab.tropical\n"
+ " create a tropical OpenFst transducer that accepts two or more\n"
+ " consecutive 'a's and maps them into string \"b\"\n"
+ "\n", program_name, program_name);
+ print_report_bugs();
+ print_more_info();
+ fprintf(message_out, "\n");
+}
+
+
+int
+parse_options(int argc, char** argv)
+{
+ // use of this function requires options are settable on global scope
+ while (true)
+ {
+ static const struct option long_options[] =
+ {
+ HFST_GETOPT_COMMON_LONG,
+ HFST_GETOPT_UNARY_LONG,
+ {"format", required_argument, 0, 'f'},
+ {"harmonize-smaller", no_argument, 0, 'H'},
+ {"weighted", no_argument, 0, 'w'},
+ {0,0,0,0}
+ };
+ int option_index = 0;
+ int c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
+ HFST_GETOPT_UNARY_SHORT "f:Hw",
+ long_options, &option_index);
+ if (-1 == c)
+ {
+ break;
+ }
+
+ switch (c)
+ {
+#include "inc/getopt-cases-common.h"
+#include "inc/getopt-cases-unary.h"
+ case 'f':
+ output_format = hfst_parse_format_name(optarg);
+ break;
+ case 'H':
+ hfst::set_harmonize_smaller(true);
+ break;
+ case'w':
+ verbose_printf(
+ "Interpreting deprecated -w switch as openfst tropical format\n");
+ output_format = hfst::TROPICAL_OPENFST_TYPE;
+ break;
+
+#include "inc/getopt-cases-error.h"
+ }
+ }
+
+#include "inc/check-params-common.h"
+#include "inc/check-params-unary.h"
+ verbose_ = verbose;
+ if (output_format == hfst::ERROR_TYPE)
+ {
+ verbose_printf("Output format not specified, "
+ "defaulting to openfst tropical\n");
+ output_format = hfst::TROPICAL_OPENFST_TYPE;
+ }
+ filename = std::string(inputfilename);
+
+ if (NULL != inputfilename && strcmp(inputfilename, "<stdin>") != 0)
+ {
+ std::string str(inputfilename);
+ size_t found = str.find_last_of("/\\");
+ if (found != std::string::npos)
+ foldername = std::string(str.substr(0,found).c_str());
+ }
+
+ return EXIT_CONTINUE;
+}
+
+void get_flags( int *argc, char **argv )
+
+{
+ for( int i=1; i<*argc; i++ ) {
+ if (strcmp(argv[i],"-c") == 0) {
+ Compact = 1;
+ argv[i] = NULL;
+ }
+ else if (strcmp(argv[i],"-l") == 0) {
+ LowMem = 1;
+ argv[i] = NULL;
+ }
+ else if (strcmp(argv[i],"-q") == 0) {
+ verbose_ = 0;
+ argv[i] = NULL;
+ }
+ else if (strcmp(argv[i],"-h") == 0 || strcmp(argv[i],"--help") == 0) {
+ print_usage();
+ exit(0);
+ }
+ else if (strcmp(argv[i],"-v") == 0 || strcmp(argv[i],"--version") == 0) {
+ fprintf(stdout, "hfst-sfstpl2fst 0.1 (hfst 3.0)\n");
+ exit(0);
+ }
+ else if (strcmp(argv[i],"-s") == 0) {
+ switch_ = 1;
+ argv[i] = NULL;
+ }
+ // hfst addition
+ else if (strcmp(argv[i],"-sfst") == 0) {
+ output_format = SFST_TYPE;
+ argv[i] = NULL;
+ }
+ else if (strcmp(argv[i],"-tropical") == 0) {
+ output_format = TROPICAL_OPENFST_TYPE;
+ argv[i] = NULL;
+ }
+ else if (strcmp(argv[i],"-log") == 0) {
+ output_format = LOG_OPENFST_TYPE;
+ argv[i] = NULL;
+ }
+ else if (strcmp(argv[i],"-foma") == 0) {
+ output_format = FOMA_TYPE;
+ argv[i] = NULL;
+ }
+ }
+ // remove flags from the argument list
+ int k;
+ for( int i=k=1; i<*argc; i++)
+ if (argv[i] != NULL)
+ argv[k++] = argv[i];
+ *argc = k;
+}
+
+int main( int argc, char *argv[] )
+
+{
+#ifdef WINDOWS
+ _setmode(1, _O_BINARY);
+#endif
+
+ hfst_set_program_name(argv[0], "0.1", "HfstSfstPl2Fst");
+ int retval = parse_options(argc, argv);
+ if (retval != EXIT_CONTINUE)
+ return retval;
+
+ if (strcmp(outfilename,"<stdout>") != 0)
+ fclose(outfile); // stream is used when writing the result
+
+ // Unknown symbols cannot be used in SFST-PL syntax.
+ // If the HFST library is aware of this, some optimization can be done.
+ hfst::set_unknown_symbols_in_use(false);
+ SfstCompiler * compiler = new SfstCompiler(output_format, verbose_);
+ compiler->set_input(inputfile);
+ compiler->set_filename(filename);
+ compiler->set_foldername(foldername);
+ compiler->set_switch(switch_);
+
+ char * strarg = NULL;
+ try {
+ compiler->parse();
+ fclose(inputfile);
+ try {
+ if (strcmp(outfilename,"<stdout>") == 0)
+ {
+ strarg = strdup("");
+ compiler->write_to_file(compiler->get_result(), NULL, strarg);
+ }
+ else
+ {
+ strarg = strdup(outfilename);
+ compiler->write_to_file(compiler->get_result(), NULL, strarg);
+ }
+ } catch (HfstException e) {
+ printf("\nAn error happened when writing to file \"%s\"\n", outfilename); }
+
+ delete compiler->get_result();
+
+ }
+ catch(const char* p) {
+ cerr << "\n" << p << "\n\n";
+ if (strarg != NULL) { free(strarg); }
+ exit(1);
+ }
+ catch (HfstException e) {
+ cerr << "\n" << e.what() << "\n\n";
+ if (strarg != NULL) { free(strarg); }
+ exit(1);
+ }
+
+ if (strarg != NULL) { free(strarg); }
+ exit(0);
+}
diff --git a/tools/src/test-memory-leaks.sh b/tools/src/test-memory-leaks.sh
index 6036c81..002b071 100755
--- a/tools/src/test-memory-leaks.sh
+++ b/tools/src/test-memory-leaks.sh
@@ -18,7 +18,7 @@ LONGER_TEST_STRING="(a:bc:de:fg:hi:jk:lm:no:pq:rs:tu:vw:xy:z)| \
(3:BC:DE:FG:HI:JK:LM:NO:PQ:RS:TU:VW:XY:4)"
for impl in sfst openfst-tropical foma;
-do
+do
# The smaller test transducer
echo $SHORTER_TEST_STRING | ./hfst-calculate -f $impl > tr.$impl;
# The bigger test transducer
@@ -62,7 +62,7 @@ hfst-txt2fst"
# Store each line here before aligning and printing it
rm -f TMP
-# For all unary tools,
+# For all unary tools,
for tool in $UNARY_TOOLS;
do
# for all implementation formats,
diff --git a/tools/src/test.cc b/tools/src/test.cc
index b9328aa..ef5f9cc 100644
--- a/tools/src/test.cc
+++ b/tools/src/test.cc
@@ -152,7 +152,7 @@ int main(int argc, char **argv) {
// ----- repeat_star -----
{
- HfstTransducer test0_plus = HfstTransducer(test0).repeat_plus();
+ HfstTransducer test0_plus = HfstTransducer(test0).repeat_plus();
HfstTransducer test1_plus = HfstTransducer(test1).repeat_plus();
HfstTransducer test2_plus = HfstTransducer(test2).repeat_plus();
HfstTransducer test3_plus = HfstTransducer(test3).repeat_plus();
@@ -171,7 +171,7 @@ int main(int argc, char **argv) {
// ----- repeat_plus -----
{
- HfstTransducer test0_plus = HfstTransducer(test0).repeat_plus();
+ HfstTransducer test0_plus = HfstTransducer(test0).repeat_plus();
HfstTransducer test1_plus = HfstTransducer(test1).repeat_plus();
HfstTransducer test2_plus = HfstTransducer(test2).repeat_plus();
HfstTransducer test3_plus = HfstTransducer(test3).repeat_plus();
@@ -190,10 +190,10 @@ int main(int argc, char **argv) {
// ----- repeat_n -----
{
- const int MAX_REPEAT_N = 5;
+ const int MAX_REPEAT_N = 5;
for (int n=0; n <= MAX_REPEAT_N; n++) {
- HfstTransducer test0_n = HfstTransducer(test0).repeat_n(n);
+ HfstTransducer test0_n = HfstTransducer(test0).repeat_n(n);
HfstTransducer test1_n = HfstTransducer(test1).repeat_n(n);
HfstTransducer test2_n = HfstTransducer(test2).repeat_n(n);
HfstTransducer test3_n = HfstTransducer(test3).repeat_n(n);
@@ -213,9 +213,9 @@ int main(int argc, char **argv) {
// ----- repeat_n_minus -----
{
- const int MAX_REPEAT_N = 5;
+ const int MAX_REPEAT_N = 5;
for (int n=0; n <= MAX_REPEAT_N; n++) {
- HfstTransducer test0_n = HfstTransducer(test0).repeat_n_minus(n);
+ HfstTransducer test0_n = HfstTransducer(test0).repeat_n_minus(n);
HfstTransducer test1_n = HfstTransducer(test1).repeat_n_minus(n);
HfstTransducer test2_n = HfstTransducer(test2).repeat_n_minus(n);
HfstTransducer test3_n = HfstTransducer(test3).repeat_n_minus(n);
@@ -235,9 +235,9 @@ int main(int argc, char **argv) {
// ----- repeat_n_plus -----
{
- const int MAX_REPEAT_N = 1;
+ const int MAX_REPEAT_N = 1;
for (int n=0; n <= MAX_REPEAT_N; n++) {
- HfstTransducer test0_n = HfstTransducer(test0).repeat_n_plus(n);
+ HfstTransducer test0_n = HfstTransducer(test0).repeat_n_plus(n);
HfstTransducer test1_n = HfstTransducer(test1).repeat_n_plus(n);
HfstTransducer test2_n = HfstTransducer(test2).repeat_n_plus(n);
HfstTransducer test3_n = HfstTransducer(test3).repeat_n_plus(n);
@@ -249,7 +249,7 @@ int main(int argc, char **argv) {
assert (HfstTransducer::test_equivalence( test0_n, test1_n ) );
assert (HfstTransducer::test_equivalence( test0_n, test2_n ) );
- assert (HfstTransducer::test_equivalence( test0_n, test3_n ) );
+ assert (HfstTransducer::test_equivalence( test0_n, test3_n ) );
}
}
printf("repeat_n_plus tested\n");
@@ -257,11 +257,11 @@ int main(int argc, char **argv) {
// ----- repeat_n_to_k -----
{
- const int REPEAT_N_MAX = 2;
+ const int REPEAT_N_MAX = 2;
const int REPEAT_K_MAX = 3;
for (int n=0; n <= REPEAT_N_MAX; n++) {
for (int k=n; k <= REPEAT_K_MAX; k++) {
- HfstTransducer test0_n = HfstTransducer(test0).repeat_n_to_k(n,k);
+ HfstTransducer test0_n = HfstTransducer(test0).repeat_n_to_k(n,k);
HfstTransducer test1_n = HfstTransducer(test1).repeat_n_to_k(n,k);
HfstTransducer test2_n = HfstTransducer(test2).repeat_n_to_k(n,k);
HfstTransducer test3_n = HfstTransducer(test3).repeat_n_to_k(n,k);
@@ -282,7 +282,7 @@ int main(int argc, char **argv) {
// ----- optionalize -----
{
- HfstTransducer test0_opt = HfstTransducer(test0).optionalize();
+ HfstTransducer test0_opt = HfstTransducer(test0).optionalize();
HfstTransducer test1_opt = HfstTransducer(test1).optionalize();
HfstTransducer test2_opt = HfstTransducer(test2).optionalize();
HfstTransducer test3_opt = HfstTransducer(test3).optionalize();
@@ -339,8 +339,8 @@ int main(int argc, char **argv) {
ImplementationType types[] = {TROPICAL_OPENFST_TYPE, LOG_OPENFST_TYPE, SFST_TYPE, FOMA_TYPE};
- for (int i=0; i<4; i++)
- {
+ for (int i=0; i<4; i++)
+ {
fprintf(stderr, "testing transducers of type %i\n", types[i]);
// open two output streams to file
@@ -461,7 +461,7 @@ int main(int argc, char **argv) {
HfstTransducer &substitute(Key old_key, Key new_key);
HfstTransducer &substitute(const std::string &old_symbol,
const std::string &new_symbol);
- HfstTransducer &substitute(const KeyPair &old_key_pair,
+ HfstTransducer &substitute(const KeyPair &old_key_pair,
const KeyPair &new_key_pair);
HfstTransducer &substitute(const StringSymbolPair &old_symbol_pair,
const StringSymbolPair &new_symbol_pair);
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/hfst.git
More information about the debian-science-commits
mailing list