[hfst] 01/02: Imported Upstream version 3.12.2~r3289
Tino Didriksen
tinodidriksen-guest at moszumanska.debian.org
Mon Aug 7 09:27:03 UTC 2017
This is an automated email from the git hooks/post-receive script.
tinodidriksen-guest pushed a commit to branch master
in repository hfst.
commit e8eea355e3be1ede9e26cc88aaa5734be8d43d4c
Author: Tino Didriksen <tino at didriksen.cc>
Date: Mon Aug 7 09:25:54 2017 +0000
Imported Upstream version 3.12.2~r3289
---
ChangeLog | 153 ++++++-
ChangeLog.old | 27 +-
NEWS | 33 +-
README.rst | 20 -
back-ends/Makefile.am | 4 +
configure.ac | 62 ++-
libhfst/src/HfstInputStream.cc | 58 +--
libhfst/src/HfstInputStream.h | 8 +-
libhfst/src/HfstOutputStream.cc | 36 +-
libhfst/src/HfstOutputStream.h | 8 +-
libhfst/src/HfstTransducer.cc | 111 +++--
libhfst/src/HfstTransducer.h | 22 +-
libhfst/src/Makefile.am | 6 +-
.../src/implementations/ConvertFomaTransducer.cc | 19 +-
.../implementations/ConvertLogWeightTransducer.cc | 17 +-
.../src/implementations/ConvertSfstTransducer.cc | 18 +-
.../src/implementations/ConvertTransducerFormat.cc | 24 +-
.../src/implementations/ConvertTransducerFormat.h | 83 +---
libhfst/src/implementations/FomaTransducer.cc | 43 +-
libhfst/src/implementations/FomaTransducer.h | 8 +-
libhfst/src/implementations/LogWeightTransducer.cc | 503 ++++++++-------------
libhfst/src/implementations/LogWeightTransducer.h | 73 +--
libhfst/src/implementations/Makefile.am | 10 +-
libhfst/src/implementations/SfstTransducer.cc | 402 ++++++++--------
libhfst/src/implementations/SfstTransducer.h | 64 +--
.../implementations/TropicalWeightTransducer.cc | 61 ++-
.../src/implementations/TropicalWeightTransducer.h | 59 +--
libhfst/src/parsers/LexcCompiler.cc | 2 -
libhfst/src/parsers/Makefile.am | 10 +-
libhfst/src/parsers/SfstAlphabet.h | 30 +-
libhfst/src/parsers/SfstCompiler.h | 5 +-
libhfst/src/parsers/htwolcpre1-lexer.ll | 4 -
libhfst/src/parsers/htwolcpre2-lexer.ll | 9 +-
libhfst/src/parsers/htwolcpre3-lexer.ll | 6 +-
libhfst/src/parsers/lexc-lexer.ll | 7 +-
libhfst/src/parsers/pmatch_lex.ll | 7 +-
libhfst/src/parsers/sfst-scanner.ll | 8 +-
libhfst/src/parsers/xfst-lexer.ll | 7 +-
libhfst/src/parsers/xre_lex.ll | 7 +-
python/doc/Doxyfile | 2 +-
python/doc/README | 2 +-
python/pypi/MANIFEST.in | 1 +
python/pypi/setup.py | 2 +-
python/setup.py | 3 +-
test/tools/hfst-check-help.sh | 12 +-
test/tools/hfst-check-version.sh | 12 +-
tools/src/Makefile.am | 45 +-
tools/src/hfst-info.cc | 18 +-
tools/src/hfst-twolc/src/hfst-twolc.cc | 21 +-
49 files changed, 1033 insertions(+), 1119 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 0247c9f..289434e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,155 @@
-77c845d (HEAD -> master, origin/master, origin/HEAD) Fix flag elimination bug (reported in issue #342).
+56534b3 (HEAD -> master, origin/master, origin/HEAD) Add missing file.
+cc2dea5 Merge branch 'master' of https://github.com/hfst/hfst
+22383de Check more carefully which backends are actually enabled.
+4cebb75 Actually remove commented code instead of making it visible.
+c811b35 Remove commented code.
+357ab0f Remove commented code.
+f9796b3 Update README.
+60742c4 Add extra compile option -std=c++0x.
+25ce84e Reapply changes in commit de59747. Require at least automake 1.12, unless building from pre-yacc-generated sources.
+750a3ca Revert changes in commit de59747. Allow automake older than 1.12.
+0ae7b1e Use std::unordered_map instead of hash_map.
+0b3a248 Use unordered_map instead of hash_map.
+8b4b1d4 Skip lexc wrapper test.
+d11b00e Add a simple exception handling mechanism.
+02e16cb Fix a typo.
+de59747 Stop supporting automake < 1.12. Warn that build will fail unless building with pre-flex-generated sources.
+e776ecc Remove commented code. Remove unnecessary variables.
+8b30936 Skip hfst-train-tagger and hfst-twolc-loc tools in version and help message testing.
+ba28c39 Add missing file to dist.
+b29e140 Remove lexc wrapper tests.
+4033c44 Disable lexc and foma wrappers unless explicitly requested. Fix a typo in --enable-expand-equivalences.
+5035132 Actually enable hfst-calculate and hfst-xfst by default as promised in commit 244b9b5...
+31fc7c7 Add missing condition.
+5191c88 Add option --with-openfst-log=lean to configure which supports only reading, writing and converting log openfst transducers.
+30d3792 Actually check if --with-sfst=lean is configured when compiling sfst functions.
+94eda73 Add configure option --with-sfst=lean which supports a limited set of sfst functions (reading, writing and converting between formats).
+5df548e Add missing const to definition, remove second argument of HFST_THROW.
+214ab27 Add function 'has_weights'. Make function 'get_profile_seconds' public.
+a27f845 Add function 'get_profile_seconds' for profiling foma back-end.
+1b64f33 Add function 'has_weights' and make function 'get_profile_seconds' static.
+244b9b5 Remove duplicate of AC_ARG_ENABLE([xfst]). Enable hfst-calculate and hfst-xfst by default.
+e47f010 Remove commented code. Make sure that code compiles when openfst backend is disabled. Add comments to some functions.
+d0a1e91 Use iosfwd header instead of iostream when possible. Try to include only header files that are actually needed.
+389679d Clean temporary files generated by tests. Add some files to be ignored.
+610572f Merge branch 'master' of https://github.com/hfst/hfst
+f66f396 Check for nested context conditions, ignoring the inner ones
+d8db552 Update man pages. Remove man pages of tools that are no longer supported. Update copyright year.
+62da656 Disable hfst-twolc script and hfst-train-tagger tool unless explicitly enabled with --enable-twolc-script and --enable-train-tagger.
+e369608 Add files to be ignored.
+142bfca By default, implement hfst-twolc as a single program and disable the script (can be controlled via --enable-twolc-script).
+98e08f6 Get rid of hfst-twolc-system tool. The script hfst-twolc will soon be replaced by a single program.
+e30202c Update windows scripts according to recent changes in twolc processing.
+d676043 Update files ignored by git.
+ae820a5 Revert to earlier version of htwolc-main.cc which calls twolc parsers directly instead of TwolcCompiler class. The latter approach sometimes causes a segmentation fault.
+d96b441 Define warning and error streams in TwolcCompiler constructor.
+9c0cca4 Throw an error instead of calling exit(1) when parsing input.
+b820f11 Handle twolc error and warning streams.
+e116a74 Add TwolcCompiler class and use it in htwolc program. Move CommandLine under libhfst/src/parsers.
+9a19088 Move most twolc code from tools/src to libhfst/src/parsers.
+0705bec if no --superblanks, newlines won't be in [], so need to print on unblanked newlines
+1351257 --giella-cg: only treat superblanks if given --superblanks
+efa0964 Update list of files ignored by git.
+19fd1d1 Add an option to use htwolc in twolc tests. Disable it by default.
+59cfdce Rename twolc files.
+a6029ef Rename twolc parser and lexer files. Move functions under namespace hfst::twolcpre[1|2|3].
+5844621 Actually exclude htwolc from installation.
+7661b38 Add an alternative implementation of hfst-twolc which does all processing in a single program. Exclude it from installation until it is properly tested.
+52d5257 only warn for backtrack-on-substrings-without-analyses if --verbose
+3936fdb Control output of htwolcpre1.
+0cc7582 Change names of extern twolc variables and add functions to access some of them.
+f9a69b2 Add missing new file hfst_sfst_extensions.cc.
+d5baf5a Separate twolc parsing and command line tools to their own files.
+035b6ab Use separate prefixes in twolc lexers and parsers.
+d3c4de9 Handle '^<' and '^>' operators in lexc regular expressions correctly. Should fix issue #346.
+6165347 Space-separated mode
+144a3f3 Ignore generated sfst source files.
+dc90602 Merge branch 'master' of https://github.com/hfst/hfst
+9f7208b Improve pmatch compilation error handling, still leaks memory at exit
+3767b51 Add new function 'compile_sfst_file'.
+512245b Merge branch 'master' of https://github.com/hfst/hfst
+4649b83 Reorder and add norm caching
+460cded clearer error message for when there is backtracking into substrings that have no analyses
+0705c3c Apply recent changes in filenames and locations.
+3560d0b Move sfst compilation from tools/src to libhfst/src/parsers.
+bc49e55 Fix namespace and class names.
+9f356e7 Change file and namespace names.
+3f8d457 Small fixes to sfst compilation.
+c28d2c5 Further separate sfst compilation and command line program.
+0ccd6fe Reorganize sfst compilation into separate compilation units.
+b9b8585 Fix xfst and twolc tests. Use rm -f when removing files that are conditionally created.
+52d3cbf Remove all exe extensions from hfst-twolc and hfst-twolc-loc because they are scripts.
+99d37e6 Remove exe extensions from hfst-twolc and hfst-twolc-loc because they are scripts.
+b9b678e Start separating sfst parser and command line program to their own files. Add a prefix 'sfst' to flex/bison functions and variables.
+9733812 Rename HfstCompiler, hfst-compiler and hfst-scanner to SfstCompiler, sfst-compiler and sfst-scanner.
+ea9fb6b Add a test for hfst.regex using 'definitions'.
+2a7ad84 Add a keyword argument 'definitions' to regex that takes a dictionary mapping transducer names into transducers.
+6c76128 Add regexp operator documentation also to web pages.
+2322ecc Modify regex documentation.
+3155955 Tentatively add documentation about regexp operators.
+77b6444 use ccache if possible, slightly faster travises
+1ab384d Update fst_to_fsa and fsa_to_fst. Support both HfstTransducer and HfstBasicTransducer formats.
+663224d Catch exceptions when compiling pmatch expressions. Add new tests for pmatch functions.
+43e0644 Do not exit on pmatch compiling errors, throw 1 and catch it in command line program.
+0556161 Merge remote-tracking branch 'refs/remotes/origin/master'
+51b811e Small math fix + revert to float, which wasn't causing problems after all
+e1c9af0 Modify fst_to_fsa and fsa_to_fst.
+aaf2f8b Support special symbols and single-character symbols without separator in fsa_to_fst. Update also documentation.
+ebea326 Add simple tests for fst_to_fsa and fsa_to_fst. Also test modifying transitions of an HfstBasicTransducer.
+ecb78d0 Rewrite HfstBasicTransducer's transitions() and state_and_transitions(). Tentatively add an implementation for functions fst_to_fsa and fsa_to_fst.
+a65ca94 Remove HfstBasicTransitions and HfstStates from HfstBasicTransducer and use hfst::implementations::HfstBasicTransitions and hfst::implementations::HfstBasicStates instead.
+4579f7d Give the pythonpath as command line argument for tests that use 'from __future__ import' instead of catenating it to the beginning of file. The import must happen in the beginning of file.
+3dda726 Rename HfstTransitions to HfstBasicTransitions.
+68155b3 Add docstrings and dummy implementations for functions fst_to_fsa and fsa_to_fst.
+4a2814d Document the changes to api.
+3d1a1ce Use the version of HfstBasicTransducer.transitions() returning a reference. Add HfstBasicTransition.set_weight to python api. Add tests for modifying transition weights.
+17e0027 Fix a typo.
+04accec Add help message to copy-for-windows.sh and comments to foma wrappers.
+3688695 Typedef and templatize vector math, reformulate as per article description
+95c022c Add help message.
+1cd8cb8 Add help message.
+dd2f230 Fix vector reading for case where lines don't end with a separator
+5bcb93d Support hfst.compile_xfst_file only for python3.
+4995953 Fix a typo.
+20f39cd Fix copying files on mac as well as pip packaging issues.
+7f60005 Update file instead of deleting it...
+49b0ef9 Tentatively also support python version 2.
+52016b2 Actually copy all files.
+01744fa Omit sdk directory on windows if compiling for python3.5 or higher. Do not redefine _MSC_VER. Do not call subprocess on mac until it works correctly.
+b44f91d Update README.
+f677972 Check if we are compiling with visual studio 14.
+b17eb3d Update PyPI documentation.
+a6ca66b Add missing file.
+410d6ef Get rid of unnecessary libraries.
+f1572e1 Merge branch 'master' of https://github.com/hfst/hfst
+75fcdd9 Fix chaining of functions in example.
+d655229 Minor version bump to combat repo inconsistencies
+91539f0 oops, reset variable
+5566426 add transliterate output mode
+416373d Update documentation.
+b5736dc Add missing file.
+43db07a Move ReplaceType under hfst.xerox_rules. Reorganize some internal functions.
+bc8983e Move python code from swig interface file under module hfst.
+3aff142 Check if readline package is available.
+391f73a Do not keep input to interactive commands in readline history.
+68e1b87 Update description.
+0ff9821 Support apply up and apply down in hfst.start_xfst().
+c687c09 Document return values of compile_xfst_file and compile_lexc_file.
+12da4ae Warn user about missing readline support if --with-readline is not requested. Issue an error if --with-unicode-handler is called as such or with a value of 'yes' or 'no'. Make it possible to disable loading entries from shared object (openfst back-end feature).
+9731e75 Update version number to 3.12.1. Handle links to future and earlier releases so that they are easier to update. Add a README.
+764399e Remove doc/libhfst.py as documentation has been moved under directory doc/hfst/.
+f3df0b9 Add more documentation for XreCompiler. Fix links to installation instructions.
+d2666d6 pre-increment (thanks, cppcheck)
+5712054 Add the script for creating python package. Convert README to dos format.
+4abf93e Update package scripts.
+4888fc6 Add a script for creating package which contains 64-bit python bindings for python3 for mac osx.
+18df468 Add a script fpr creating 64-bit python bindings for python 3.3 and 3.4 for windows.
+4790986 Add README files for python packages for mac and windows.
+3519e4c Add a script for creating the lgpl package.
+0cb9fb5 Add alternative files for making LGPL release of HFST.
+fea6723 (tag: v3.12.1) Add missing files to dist. Do not print termination message when performing tokenize tests.
+65962bf Ready for release 3.12.1.
+77c845d Fix flag elimination bug (reported in issue #342).
c827bb6 Do not call fsm_destroy until issue with 'double free or corruption' has been fixed.
3b0e58a Provide a get_current_dir_name() for platforms without one
b0fb7a6 Merge branch 'master' of https://github.com/hfst/hfst
diff --git a/ChangeLog.old b/ChangeLog.old
index 0ac39cf..0247c9f 100644
--- a/ChangeLog.old
+++ b/ChangeLog.old
@@ -1,4 +1,28 @@
-42df833 (HEAD -> master, origin/master, origin/HEAD) Small fixes to memory leaks and efficiency.
+77c845d (HEAD -> master, origin/master, origin/HEAD) Fix flag elimination bug (reported in issue #342).
+c827bb6 Do not call fsm_destroy until issue with 'double free or corruption' has been fixed.
+3b0e58a Provide a get_current_dir_name() for platforms without one
+b0fb7a6 Merge branch 'master' of https://github.com/hfst/hfst
+65a4d79 Search for included files under scriptdir, not working dir. If an included filename is non-absolute, prefix it with the directory the script was in when trying to read it. This only happens in non-Windows environments for now.
+f5e94d7 Free allocated memory.
+6144e75 tokenise -g: check for eof
+cf46e69 Apply more changes from latest release of foma.
+a9f8397 Apply some changes from newest release of foma.
+cc8c96b Make a separate directory python/pypi for creating pypi packages.
+479789e Rename the pypi package to 'hfst' and use a four-number versioning scheme. Also define '__version__'. Add an empty config file as hfst c++ headers include it.
+eb304f0 tests: trying sed instead of awk for exit on <STREAMCMD:FLUSH>
+798383d tokenize: handle Apertium-style superblanks in --giella-cg format
+7922399 Provide a way to scale the vector similarity calculations via variable
+166d418 Merge branch 'master' of https://github.com/hfst/hfst
+9bc5315 parse word2vec -style text format and assume first line is info
+5d3f6b8 Do not allow unescaped dots in regular expressions.
+ce5e2a5 Merge branch 'master' of https://github.com/hfst/hfst
+a8f21f0 Observe HAVE_SFST and HAVE_FOMA when including back-end header files.
+8a9c0de nwords catenate-style argument for Like() function
+1433e45 Merge branch 'master' of https://github.com/hfst/hfst
+1ec8e03 Add experimental two-vector model for word sense
+72d4f72 (tag: v3.12.0) Ready for release 3.12.0.
+da86647 Read vector models and do simple ranking
+42df833 Small fixes to memory leaks and efficiency.
e2abd89 Fix some memory leaks.
da54acc Fix leaks reported by valgrind.
92cf4dc Fix more memory leaks.
@@ -13,6 +37,7 @@ e4c1304 Rename hfst.rules into hfst.sfst_rules. Fix hfst.types into hfst.Impleme
52a9d75 Add test for hfst.xerox_rules.
00b020c Rename xerox_replace_left and xerox_restriction back to replace_left and restriction in package hfst.xerox_rules.
d1556dd Fix documentation of parameter 'output' of function 'extract_paths'.
+3df4a48 Sentence separation for finnpos-mode
d3058c7 Issue a warning if automake is too old, which can make build fail if pre-generated yacc files are modified.
9166cdc Move ImplementationTypes and ReplaceTypes to their own classes and get rid of package hfst.types.
6d6c631 Fix datatypes used in for loops.
diff --git a/NEWS b/NEWS
index f00cdf4..6f5ab3d 100644
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,38 @@
This file contains all noteworthy changes in HFST development between releases.
For full listing of changes see ChangeLog.
+Noteworthy changes in 3.12.2
+----------------------------
+
+* Changes to configure:
+
+ * Disable lexc and foma wrappers as well as hfst-train-tagger tool unless explicitly requested
+
+ * Enable hfst-calculate and hfst-xfst by default
+
+ * Add experimental options --with-openfst-log=lean and --with-sfst=lean which support a limited nuber of operations for these types (reading, writing, converting between types and deleting)
+
+ * Require at least automake version 1.12 unless compiling from pre-yacc-generated sources
+
+* Implement hfst-twolc as a single program instead of a script. Get rid of hfst-twolc-system and hfst-twolc-loc tools.
+
+* Improve pmatch compilation and error handling mechanisms
+
+* Improve hfst-tokenize tool
+
+* Add transliterate output mode (--transliterate) to hfst-proc
+
+* Changes to python interface:
+
+ * Support twolc and sfst compilation
+
+ * Improve HfstBasicTransducer iteration mechanism
+
+ * Add experimental support for python version 2
+
+ * Support apply up and apply down commands in function start_xfst
+
+
Noteworthy changes in 3.12.1
----------------------------
@@ -733,4 +765,3 @@ Noteworthy changes in release 2.1
---------------------------------
* hfst-twolc and hfst-compose-intersect understand # like xerox originals
-
diff --git a/README.rst b/README.rst
index 1de4599..92d89cf 100644
--- a/README.rst
+++ b/README.rst
@@ -46,26 +46,6 @@ and Cygwin. Python bindings for Windows are currently available only for 64-bit
Python versions 3.3 and 3.4 on our `download page
<https://hfst.github.io/downloads/index.html>`_.
-Installation for MacPorts
--------------------------
-
-The port tree for HFST and related software can be downloaded from `HFST
-download directory <https://sourceforge.net/projects/hfst/files/hfst/>`_.
-To add this port tree to your macports installation, unpack it to wherever
-and insert that directory to your ``sources.conf``, e.g. in the default
-installation to ``/opt/local/etc/macports/sources.conf``::
-
- [...]
- # If an rsync URL points to a .tar file, a signed .rmd160 must exist next to
- # it on the server and will be used to verify its integrity.
-
- file:///Users/flammie/Sources/hfst-macport/
- rsync://rsync.macports.org/release/tarballs/ports.tar [default]
-
-Now you can install hfst and related software using port commands::
-
- sudo port install hfst
-
Installation for Gentoo Linux
-----------------------------
diff --git a/back-ends/Makefile.am b/back-ends/Makefile.am
index 93f23c0..9de9364 100644
--- a/back-ends/Makefile.am
+++ b/back-ends/Makefile.am
@@ -23,6 +23,10 @@ endif
if WANT_SFST
SUBDIRS += sfst
+else
+if WANT_LEAN_SFST
+ SUBDIRS += sfst
+endif
endif
if WANT_FOMA
diff --git a/configure.ac b/configure.ac
index 2036c7a..5cedc9b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -28,7 +28,7 @@ HFST_VERSION=$HFST_MAJOR.$HFST_MINOR.$HFST_EXTENSION
LIBHFST_NAME=hfst
LIBHFST_MAJOR=3
LIBHFST_MINOR=12
-LIBHFST_EXTENSION=1
+LIBHFST_EXTENSION=2
LIBHFST_VERSION=$LIBHFST_MAJOR.$LIBHFST_MINOR.$LIBHFST_EXTENSION
AC_INIT([hfst], [3.12.2], [hfst-bugs at helsinki.fi], [hfst])
@@ -44,8 +44,8 @@ AC_CONFIG_HEADERS([config.h libhfst/src/hfst.hpp])
AC_SUBST([LIBHFST_MAJOR], [3])
AC_SUBST([LIBHFST_MINOR], [12])
-AC_SUBST([LIBHFST_EXTENSION], [1])
-AC_SUBST([LIBHFST_VERSION], [3.12.1])
+AC_SUBST([LIBHFST_EXTENSION], [2])
+AC_SUBST([LIBHFST_VERSION], [3.12.2])
AC_SUBST([LIBHFST_NAME], [hfst])
# long version = version vector cast in base 10000, for automatic comparisons
@@ -56,7 +56,7 @@ AC_SUBST([LIBHFST_NAME], [hfst])
# $LIBHFST_MINOR * 10000 + $LIBHFST_EXTENSION + "L"
# NB! It turned out to be not portable, and can't be used!
-AC_DEFINE([HFST_LONGVERSION], [300120001L],
+AC_DEFINE([HFST_LONGVERSION], [300120002L],
[Define to hfst version vector as long in base 10000])
AC_DEFINE([HFST_REVISION], ["$Revision$"],
[Automatically substitute to configure.ac revision])
@@ -71,7 +71,12 @@ AC_ARG_WITH([sfst],
[with_sfst=yes])
AS_IF([test "x$with_sfst" != xno], [AC_DEFINE([HAVE_SFST], [1],
[Define to compile SFST support in HFST])])
-AM_CONDITIONAL([WANT_SFST], [test x$with_sfst != xno])
+AS_IF([test "x$with_sfst" == xlean], [AC_DEFINE([HAVE_SFST], [0], [Define not to compile full SFST support in HFST])
+ AC_DEFINE([HAVE_LEAN_SFST], [1], [Define to compile lean SFST support in HFST])])
+
+AM_CONDITIONAL([WANT_SFST], [test x$with_sfst != xno -a x$with_sfst != xlean])
+AM_CONDITIONAL([WANT_LEAN_SFST], [test x$with_sfst == xlean])
+
AC_ARG_WITH([openfst],
[AS_HELP_STRING([--with-openfst],
[process weighted fsts with OpenFst @<:@default=yes@:>@])],
@@ -80,6 +85,7 @@ AC_ARG_WITH([openfst],
AS_IF([test "x$with_openfst" != xno], [AC_DEFINE([HAVE_OPENFST], [1],
[Define to compile OpenFst support in HFST])])
AM_CONDITIONAL([WANT_OPENFST], [test x$with_openfst != xno])
+
AC_ARG_WITH([openfst-log],
[AS_HELP_STRING([--with-openfst-log],
[enable logarithmic weights @<:@default=yes@:>@])],
@@ -87,7 +93,12 @@ AC_ARG_WITH([openfst-log],
[with_openfst_log=yes])
AS_IF([test "x$with_openfst_log" != xno], [AC_DEFINE([HAVE_OPENFST_LOG], [1],
[Define to compile OpenFst with log weight support in HFST])])
-AM_CONDITIONAL([WANT_OPENFST_LOG], [test x$with_openfst_log != xno])
+AS_IF([test "x$with_openfst_log" == xlean], [AC_DEFINE([HAVE_OPENFST_LOG], [0], [Define not to compile OpenFst with full log weight support in HFST])
+ AC_DEFINE([HAVE_LEAN_OPENFST_LOG], [1], [Define to compile OpenFst with lean log weight support in HFST])])
+
+AM_CONDITIONAL([WANT_OPENFST_LOG], [test x$with_openfst_log != xno -a x$with_openfst_log != xlean])
+AM_CONDITIONAL([WANT_LEAN_OPENFST_LOG], [test x$with_openfst_log == xlean])
+
AC_ARG_WITH([foma],
[AS_HELP_STRING([--with-foma],
[process unweighted fsts with foma @<:@default=yes@:>@])],
@@ -158,7 +169,7 @@ AS_IF([test "x$enable_load_so_entries" != xno], [AC_DEFINE([ENABLE_LOAD_SO_ENTRI
AC_ARG_ENABLE([all_tools],
[AS_HELP_STRING([--enable-all-tools],
- [build all tools (except hfst-twolc-script and hfst-train-tagger which must be explicitly enabled with --enable-twolc-script and --enable-train-tagger) @<:@default=no@:>@])],
+ [build all tools (excluding legacy programs which must be explicitly enabled with --enable-lexc-wrapper --enable-foma-wrapper --enable-twolc-script and --enable-train-tagger) @<:@default=no@:>@])],
[enable_all_tools=$enableval],
[enable_all_tools=no])
@@ -180,7 +191,7 @@ AC_ARG_ENABLE([calculate],
[AS_HELP_STRING([--enable-calculate],
[build calculate tool @<:@default=yes@:>@])],
[enable_calculate=$enableval],
- [enable_calculate=$enable_all_tools])
+ [if test x$enable_no_tools != xno; then enable_calculate=no; else enable_calculate=yes; fi])
AM_CONDITIONAL([WANT_CALCULATE], [test x$enable_calculate != xno])
AC_ARG_ENABLE([compare],
[AS_HELP_STRING([--enable-compare],
@@ -231,7 +242,7 @@ AC_ARG_ENABLE([disjunct],
[if test x$enable_no_tools != xno; then enable_disjunct=no; else enable_disjunct=yes; fi])
AM_CONDITIONAL([WANT_DISJUNCT], [test x$enable_disjunct != xno])
AC_ARG_ENABLE([expand_equivalences],
- [AS_HELP_STRING([--enable-edit-metadata],
+ [AS_HELP_STRING([--enable-expand-equivalences],
[build acx tool @<:@default=no@:>@])],
[enable_expand_equivalences=$enableval],
[enable_expand_equivalences=$enable_all_tools])
@@ -304,9 +315,9 @@ AC_ARG_ENABLE([invert],
AM_CONDITIONAL([WANT_INVERT], [test x$enable_invert != xno])
AC_ARG_ENABLE([lexc-wrapper],
[AS_HELP_STRING([--enable-lexc-wrapper],
- [build lexc-wrapper cli and readline @<:@default=no@:>@])],
+ [build legacy lexc tool @<:@default=no@:>@])],
[enable_lexc_wrapper=$enableval],
- [enable_lexc_wrapper=$enable_all_tools])
+ [enable_lexc_wrapper=no])
AM_CONDITIONAL([WANT_LEXC_WRAPPER], [test "x$enable_lexc_wrapper" != xno])
AC_ARG_ENABLE([twolc],
[AS_HELP_STRING([--enable-twolc],
@@ -334,7 +345,7 @@ AC_ARG_ENABLE([train-tagger],
AM_CONDITIONAL([WANT_TRAIN_TAGGER], [test "x$enable_train_tagger" != xno])
AC_ARG_ENABLE([lexc],
[AS_HELP_STRING([--enable-lexc],
- [build lexc cli @<:@default=yes@:>@])],
+ [build lexc tool @<:@default=yes@:>@])],
[enable_lexc=$enableval],
[if test x$enable_no_tools != xno; then enable_lexc=no; else enable_lexc=yes; fi])
AM_CONDITIONAL([WANT_LEXC], [test "x$enable_lexc" != xno])
@@ -464,12 +475,6 @@ AC_ARG_ENABLE([strings2fst],
[enable_strings2fst=$enableval],
[if test x$enable_no_tools != xno; then enable_strings2fst=no; else enable_strings2fst=yes; fi])
AM_CONDITIONAL([WANT_STRINGS2FST], [test x$enable_strings2fst != xno])
-#AC_ARG_ENABLE([strip_header],
-# [AS_HELP_STRING([--enable-strip-header],
-# [build strip-header tool @<:@default=yes@:>@])],
-# [enable_strip_header=$enableval],
-# [enable_strip_header=yes])
-#AM_CONDITIONAL([WANT_STRIP_HEADER], [test x$enable_strip_header != xno])
AC_ARG_ENABLE([substitute],
[AS_HELP_STRING([--enable-substitute],
[build substitution tool @<:@default=yes@:>@])],
@@ -514,15 +519,15 @@ AC_ARG_ENABLE([txt2fst],
AM_CONDITIONAL([WANT_TXT2FST], [test x$enable_txt2fst != xno])
AC_ARG_ENABLE([foma_wrapper],
[AS_HELP_STRING([--enable-foma-wrapper],
- [build foma wrapper tool @<:@default=no@:>@])],
+ [build legacy foma wrapper tool @<:@default=no@:>@])],
[enable_foma_wrapper=$enableval],
- [enable_foma_wrapper=$enable_all_tools])
+ [enable_foma_wrapper=no])
AM_CONDITIONAL([WANT_FOMA_WRAPPER], [test x$enable_foma_wrapper != xno])
AC_ARG_ENABLE([xfst],
[AS_HELP_STRING([--enable-xfst],
[build xfst tool @<:@default=yes@:>@])],
[enable_xfst=$enableval],
- [enable_xfst=$enable_all_tools])
+ [if test x$enable_no_tools != xno; then enable_xfst=no; else enable_xfst=yes; fi])
AM_CONDITIONAL([WANT_XFST], [test x$enable_xfst != xno])
# Check if zlib is available
@@ -583,17 +588,6 @@ yacc_use_parser_h_extension=false
automake_version=`automake --version | head -n 1 | cut '-d ' -f 4`
AC_DEFINE_UNQUOTED([AUTOMAKE_VERSION], [$automake_version], [Defines automake version])
-if test "$automake_version" \< "1.12" ; then
- #Use *.h extension for parser header file
- yacc_use_parser_h_extension=true
- # issue a warning at the end of configure
- AC_DEFINE([YACC_USE_PARSER_H_EXTENSION], [1], [Use *.h extension for parser header file])
-else
- echo "automake version >= 1.12, using .hh extension for yacc/bison generated header files"
-fi
-
-AM_CONDITIONAL([YACC_USE_PARSER_H_EXTENSION], [test x$yacc_use_parser_h_extension = xtrue])
-
AC_PROG_LEX
AC_PATH_PROG([GETOPT], [getopts], [false])
@@ -855,5 +849,5 @@ AS_IF([test "x$enable_calculate" == "xno"],
AS_IF([test "x$enable_xfst" == "xno"],
[AC_MSG_WARN([hfst-xfst is not enabled; you will not be able to compile XFST scripts; enable using --enable-xfst])])
AC_MSG_WARN([Python bindings for HFST are not under autotools; see python/README for instructions about how to build and install them])
-AS_IF([test "x$yacc_use_parser_h_extension" == "xtrue"],
- [AC_MSG_WARN([automake version < 1.12; using .h extension for yacc/bison generated header files; if you are building with pre-generated files, modifying them will make building fail, because they use .hh extension])])
+AS_IF([test "$automake_version" \< "1.12"],
+ [AC_MSG_WARN([automake version < 1.12; building parsers will fail unless you are building with pre-yacc-generated *.cc files and do not modify the *.yy source files])])
diff --git a/libhfst/src/HfstInputStream.cc b/libhfst/src/HfstInputStream.cc
index 3aff8d5..b853642 100644
--- a/libhfst/src/HfstInputStream.cc
+++ b/libhfst/src/HfstInputStream.cc
@@ -37,7 +37,7 @@ namespace hfst
{
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
this->implementation.sfst->ignore(n);
break;
@@ -46,7 +46,7 @@ namespace hfst
case TROPICAL_OPENFST_TYPE:
this->implementation.tropical_ofst->ignore(n);
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
this->implementation.log_ofst->ignore(n);
break;
@@ -79,7 +79,7 @@ namespace hfst
return c = input_stream->get();
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
return c = this->implementation.sfst->stream_get();
break;
@@ -88,7 +88,7 @@ namespace hfst
case TROPICAL_OPENFST_TYPE:
return c = this->implementation.tropical_ofst->stream_get();
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
return c = this->implementation.log_ofst->stream_get();
break;
@@ -125,7 +125,7 @@ namespace hfst
}
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
return i = this->implementation.sfst->stream_get_short();
break;
@@ -134,7 +134,7 @@ namespace hfst
case TROPICAL_OPENFST_TYPE:
return i = this->implementation.tropical_ofst->stream_get_short();
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
return i = this->implementation.log_ofst->stream_get_short();
break;
@@ -177,7 +177,7 @@ namespace hfst
return (char) input_stream->get();
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
return this->implementation.sfst->stream_get();
break;
@@ -186,7 +186,7 @@ namespace hfst
case TROPICAL_OPENFST_TYPE:
return this->implementation.tropical_ofst->stream_get();
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
return this->implementation.log_ofst->stream_get();
break;
@@ -222,7 +222,7 @@ namespace hfst
}
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
this->implementation.sfst->stream_unget(c);
break;
@@ -231,7 +231,7 @@ namespace hfst
case TROPICAL_OPENFST_TYPE:
this->implementation.tropical_ofst->stream_unget(c);
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
this->implementation.log_ofst->stream_unget(c);
break;
@@ -289,7 +289,7 @@ namespace hfst
bool HfstInputStream::set_implementation_specific_header_data
(StringPairVector& /* data */, unsigned int /* index*/)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
switch (type)
{
case SFST_TYPE:
@@ -330,7 +330,7 @@ namespace hfst
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
{
t.implementation.sfst =
@@ -472,7 +472,7 @@ namespace hfst
}
break;
}
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
{
t.implementation.log_ofst =
@@ -948,13 +948,13 @@ namespace hfst
HFST_THROW(EndOfStreamException);
type = stream_fst_type();
- if ( ! HfstTransducer::is_implementation_type_available(type)) {
+ if ( ! HfstTransducer::is_lean_implementation_type_available(type)) {
throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
}
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
implementation.sfst = new hfst::implementations::SfstInputStream;
break;
@@ -964,7 +964,7 @@ namespace hfst
implementation.tropical_ofst =
new hfst::implementations::TropicalWeightInputStream;
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
implementation.log_ofst =
new hfst::implementations::LogWeightInputStream;
@@ -1026,13 +1026,13 @@ namespace hfst
type = stream_fst_type();
}
- if ( ! HfstTransducer::is_implementation_type_available(type)) {
+ if ( ! HfstTransducer::is_lean_implementation_type_available(type)) {
throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
}
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
implementation.sfst
= new hfst::implementations::SfstInputStream(filename);
@@ -1049,7 +1049,7 @@ namespace hfst
implementation.tropical_ofst =
new hfst::implementations::TropicalWeightInputStream(filename);
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
implementation.log_ofst =
new hfst::implementations::LogWeightInputStream(filename);
@@ -1094,7 +1094,7 @@ namespace hfst
{
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
delete implementation.sfst;
break;
@@ -1103,7 +1103,7 @@ namespace hfst
case TROPICAL_OPENFST_TYPE:
delete implementation.tropical_ofst;
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
delete implementation.log_ofst;
break;
@@ -1139,7 +1139,7 @@ namespace hfst
{
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
implementation.sfst->close();
break;
@@ -1148,7 +1148,7 @@ namespace hfst
case TROPICAL_OPENFST_TYPE:
implementation.tropical_ofst->close();
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
implementation.log_ofst->close();
break;
@@ -1182,7 +1182,7 @@ namespace hfst
{
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
return implementation.sfst->is_eof();
break;
@@ -1191,7 +1191,7 @@ namespace hfst
case TROPICAL_OPENFST_TYPE:
return implementation.tropical_ofst->is_eof();
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
return implementation.log_ofst->is_eof();
break;
@@ -1225,7 +1225,7 @@ namespace hfst
{
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
return implementation.sfst->is_bad();
break;
@@ -1234,7 +1234,7 @@ namespace hfst
case TROPICAL_OPENFST_TYPE:
return implementation.tropical_ofst->is_bad();
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
return implementation.log_ofst->is_bad();
break;
@@ -1269,7 +1269,7 @@ namespace hfst
{
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
return implementation.sfst->is_good();
break;
@@ -1278,7 +1278,7 @@ namespace hfst
case TROPICAL_OPENFST_TYPE:
return implementation.tropical_ofst->is_good();
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
return implementation.log_ofst->is_good();
break;
diff --git a/libhfst/src/HfstInputStream.h b/libhfst/src/HfstInputStream.h
index 96412f3..01c8dab 100644
--- a/libhfst/src/HfstInputStream.h
+++ b/libhfst/src/HfstInputStream.h
@@ -28,11 +28,11 @@ namespace hfst
namespace implementations {
#if HAVE_OPENFST
class LogWeightInputStream;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
class TropicalWeightInputStream;
#endif
#endif
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
class SfstInputStream;
#endif
#if HAVE_FOMA
@@ -100,12 +100,12 @@ For documentation on the HFST binary transducer format, see
union StreamImplementation
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
hfst::implementations::SfstInputStream * sfst;
#endif
#if HAVE_OPENFST
hfst::implementations::TropicalWeightInputStream * tropical_ofst;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
hfst::implementations::LogWeightInputStream * log_ofst;
#endif
#endif
diff --git a/libhfst/src/HfstOutputStream.cc b/libhfst/src/HfstOutputStream.cc
index d260eae..0f1c3da 100644
--- a/libhfst/src/HfstOutputStream.cc
+++ b/libhfst/src/HfstOutputStream.cc
@@ -21,13 +21,13 @@ namespace hfst
HfstOutputStream::HfstOutputStream(ImplementationType type, bool hfst_format):
type(type), hfst_format(hfst_format), is_open(false)
{
- if (! HfstTransducer::is_implementation_type_available(type)) {
+ if (! HfstTransducer::is_lean_implementation_type_available(type)) {
throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
}
switch(type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
implementation.sfst =
new hfst::implementations::SfstOutputStream();
@@ -38,7 +38,7 @@ namespace hfst
implementation.tropical_ofst =
new hfst::implementations::TropicalWeightOutputStream(hfst_format);
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
implementation.log_ofst =
new hfst::implementations::LogWeightOutputStream();
@@ -84,13 +84,13 @@ namespace hfst
(const std::string &filename,ImplementationType type, bool hfst_format_):
type(type), hfst_format(hfst_format_), is_open(false)
{
- if (! HfstTransducer::is_implementation_type_available(type)) {
+ if (! HfstTransducer::is_lean_implementation_type_available(type)) {
throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
}
switch(type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
implementation.sfst =
new hfst::implementations::SfstOutputStream(filename);
@@ -107,7 +107,7 @@ namespace hfst
new hfst::implementations::TropicalWeightOutputStream
(filename, hfst_format);
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
implementation.log_ofst =
new hfst::implementations::LogWeightOutputStream(filename);
@@ -154,7 +154,7 @@ namespace hfst
{
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
delete implementation.sfst;
break;
@@ -163,7 +163,7 @@ namespace hfst
case TROPICAL_OPENFST_TYPE:
delete implementation.tropical_ofst;
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
delete implementation.log_ofst;
break;
@@ -216,7 +216,7 @@ namespace hfst
{
switch(type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
implementation.sfst->write(c);
break;
@@ -225,7 +225,7 @@ namespace hfst
case TROPICAL_OPENFST_TYPE:
implementation.tropical_ofst->write(c);
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
implementation.log_ofst->write(c);
break;
@@ -267,7 +267,7 @@ namespace hfst
switch(type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
type_value=std::string("SFST");
break;
@@ -276,7 +276,7 @@ namespace hfst
case TROPICAL_OPENFST_TYPE:
type_value=std::string("TROPICAL_OPENFST");
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
type_value=std::string("LOG_OPENFST");
break;
@@ -310,7 +310,7 @@ namespace hfst
append(header, type_value);
}
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
void
HfstOutputStream::append_implementation_specific_header_data(std::vector<char>&
header,
@@ -322,7 +322,7 @@ HfstOutputStream::append_implementation_specific_header_data(std::vector<char>&,
HfstTransducer&)
#endif
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
switch(type)
{
case SFST_TYPE:
@@ -447,7 +447,7 @@ HfstOutputStream::append_implementation_specific_header_data(std::vector<char>&,
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
implementation.sfst->write_transducer
(transducer.implementation.sfst);
@@ -458,7 +458,7 @@ HfstOutputStream::append_implementation_specific_header_data(std::vector<char>&,
implementation.tropical_ofst->write_transducer
(transducer.implementation.tropical_ofst);
return *this;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
implementation.log_ofst->write_transducer
(transducer.implementation.log_ofst);
@@ -497,7 +497,7 @@ HfstOutputStream::append_implementation_specific_header_data(std::vector<char>&,
void HfstOutputStream::close(void) {
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
implementation.sfst->close();
break;
@@ -506,7 +506,7 @@ HfstOutputStream::append_implementation_specific_header_data(std::vector<char>&,
case TROPICAL_OPENFST_TYPE:
implementation.tropical_ofst->close();
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
implementation.log_ofst->close();
break;
diff --git a/libhfst/src/HfstOutputStream.h b/libhfst/src/HfstOutputStream.h
index a112b28..28578ea 100644
--- a/libhfst/src/HfstOutputStream.h
+++ b/libhfst/src/HfstOutputStream.h
@@ -26,12 +26,12 @@ namespace hfst
namespace implementations {
#if HAVE_OPENFST
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
class LogWeightOutputStream;
#endif
class TropicalWeightOutputStream;
#endif
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
class SfstOutputStream;
#endif
#if HAVE_FOMA
@@ -73,12 +73,12 @@ For more information on HFST transducer structure, see
union StreamImplementation
{
#if HAVE_OPENFST
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
hfst::implementations::LogWeightOutputStream * log_ofst;
#endif
hfst::implementations::TropicalWeightOutputStream * tropical_ofst;
#endif
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
hfst::implementations::SfstOutputStream * sfst;
#endif
#if HAVE_FOMA
diff --git a/libhfst/src/HfstTransducer.cc b/libhfst/src/HfstTransducer.cc
index b86a146..6eb145e 100644
--- a/libhfst/src/HfstTransducer.cc
+++ b/libhfst/src/HfstTransducer.cc
@@ -44,7 +44,7 @@ namespace hfst
#if HAVE_OPENFST
hfst::implementations::TropicalWeightTransducer
HfstTransducer::tropical_ofst_interface;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
hfst::implementations::LogWeightTransducer
HfstTransducer::log_ofst_interface;
#endif
@@ -1149,7 +1149,7 @@ HfstTransducer::HfstTransducer(const std::string& upper_utf8_str,
HfstTransducer::HfstTransducer(HfstInputStream &in):
type(in.type), anonymous(false),is_trie(false), name("")
{
- if (! is_implementation_type_available(type)) {
+ if (! is_lean_implementation_type_available(type)) {
throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
}
@@ -1223,12 +1223,12 @@ HfstTransducer::HfstTransducer
ImplementationType type):
type(type),anonymous(false),is_trie(false), name("")
{
- if (! is_implementation_type_available(type))
+ if (! is_lean_implementation_type_available(type))
throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
implementation.sfst =
ConversionFunctions::hfst_basic_transducer_to_sfst(&net);
@@ -1239,7 +1239,7 @@ HfstTransducer::HfstTransducer
implementation.tropical_ofst =
ConversionFunctions::hfst_basic_transducer_to_tropical_ofst(&net);
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
implementation.log_ofst =
ConversionFunctions::hfst_basic_transducer_to_log_ofst(&net);
@@ -1277,12 +1277,12 @@ HfstTransducer::HfstTransducer
HfstTransducer::~HfstTransducer(void)
{
- if (! is_implementation_type_available(type))
+ if (! is_lean_implementation_type_available(type))
throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
sfst_interface.delete_transducer(implementation.sfst);
break;
@@ -1291,7 +1291,7 @@ HfstTransducer::~HfstTransducer(void)
case TROPICAL_OPENFST_TYPE:
tropical_ofst_interface.delete_transducer(implementation.tropical_ofst);
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
log_ofst_interface.delete_transducer(implementation.log_ofst);
break;
@@ -3782,7 +3782,22 @@ HfstTransducer &HfstTransducer::transform_weights(float (*func)(float))
return *this;
}
-
+bool HfstTransducer::has_weights() const
+{
+#if HAVE_OPENFST
+ if (this->type == TROPICAL_OPENFST_TYPE)
+ {
+ return this->tropical_ofst_interface.has_weights(implementation.tropical_ofst);
+ }
+#if HAVE_OPENFST_LOG
+ if (this->type == LOG_OPENFST_TYPE)
+ {
+ HFST_THROW(FunctionNotImplementedException);
+ }
+#endif
+#endif
+ return false;
+}
// -----------------------------------------------------------------------
@@ -4859,7 +4874,7 @@ HfstTransducer &HfstTransducer::subtract
implementations::HfstBasicTransducer * HfstTransducer::
get_basic_transducer() const
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
if (this->type == SFST_TYPE)
{
hfst::implementations::HfstBasicTransducer * net =
@@ -4876,7 +4891,7 @@ get_basic_transducer() const
(implementation.tropical_ofst);
return net;
}
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
if (this->type == LOG_OPENFST_TYPE)
{
hfst::implementations::HfstBasicTransducer * net =
@@ -4905,7 +4920,7 @@ get_basic_transducer() const
implementations::HfstBasicTransducer * HfstTransducer::
convert_to_basic_transducer()
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
if (this->type == SFST_TYPE)
{
hfst::implementations::HfstBasicTransducer * net =
@@ -4924,7 +4939,7 @@ convert_to_basic_transducer()
tropical_ofst_interface.delete_transducer(implementation.tropical_ofst);
return net;
}
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
if (this->type == LOG_OPENFST_TYPE)
{
hfst::implementations::HfstBasicTransducer * net =
@@ -4955,7 +4970,7 @@ convert_to_basic_transducer()
HfstTransducer &HfstTransducer::
convert_to_hfst_transducer(implementations::HfstBasicTransducer *t)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
if (this->type == SFST_TYPE)
{
implementation.sfst =
@@ -4974,7 +4989,7 @@ convert_to_hfst_transducer(implementations::HfstBasicTransducer *t)
delete t;
return *this;
}
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
if (this->type == LOG_OPENFST_TYPE)
{
implementation.log_ofst =
@@ -5011,7 +5026,7 @@ HfstTransducer &HfstTransducer::convert(const HfstTransducer &t,
}
if (type == t.type)
{ return *(new HfstTransducer(t)); }
- if (! is_implementation_type_available(type)) {
+ if (! is_lean_implementation_type_available(type)) {
throw ImplementationTypeNotAvailableException("HfstTransducer::convert", __FILE__, __LINE__, type);
}
@@ -5029,15 +5044,19 @@ bool HfstTransducer::is_implementation_type_available
(ImplementationType type) {
#if !HAVE_FOMA
if (type == FOMA_TYPE)
- return false;
+ return false;
#endif
#if !HAVE_SFST
if (type == SFST_TYPE)
- return false;
+ return false;
#endif
#if !HAVE_OPENFST
if (type == TROPICAL_OPENFST_TYPE || type == LOG_OPENFST_TYPE)
- return false;
+ return false;
+#endif
+#if !HAVE_OPENFST_LOG
+ if (type == LOG_OPENFST_TYPE)
+ return false;
#endif
#if !HAVE_XFSM
if (type == XFSM_TYPE)
@@ -5052,10 +5071,40 @@ bool HfstTransducer::is_implementation_type_available
return true;
}
+bool HfstTransducer::is_lean_implementation_type_available
+(ImplementationType type) {
+#if !HAVE_FOMA
+ if (type == FOMA_TYPE)
+ return false;
+#endif
+#if !HAVE_SFST
+#if !HAVE_LEAN_SFST
+ if (type == SFST_TYPE)
+ return false;
+#endif
+#endif
+#if !HAVE_OPENFST
+ if (type == TROPICAL_OPENFST_TYPE || type == LOG_OPENFST_TYPE)
+ return false;
+#endif
+#if !HAVE_OPENFST_LOG
+#if !HAVE_LEAN_OPENFST_LOG
+ if (type == LOG_OPENFST_TYPE)
+ return false;
+#endif
+#endif
+#if !HAVE_XFSM
+ if (type == XFSM_TYPE)
+ return false;
+#endif
+ (void)type;
+ return true;
+}
+
HfstTransducer &HfstTransducer::convert(ImplementationType type,
std::string options)
{
- if (! is_implementation_type_available(this->type)) {
+ if (! is_lean_implementation_type_available(this->type)) {
HFST_THROW_MESSAGE(HfstFatalException,
"HfstTransducer::convert: the original type "
"of the transducer is not available!");
@@ -5067,7 +5116,7 @@ HfstTransducer &HfstTransducer::convert(ImplementationType type,
"HfstTransducer::convert"); }
if (type == this->type)
{ return *this; }
- if (! is_implementation_type_available(type)) {
+ if (! is_lean_implementation_type_available(type)) {
throw ImplementationTypeNotAvailableException("HfstTransducer::convert", __FILE__, __LINE__, type);
}
@@ -5100,7 +5149,7 @@ HfstTransducer &HfstTransducer::convert(ImplementationType type,
//delete(implementation.my_transducer_library);
//break;
//#endif
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
internal =
ConversionFunctions::sfst_to_hfst_basic_transducer
@@ -5116,7 +5165,7 @@ HfstTransducer &HfstTransducer::convert(ImplementationType type,
assert(internal != NULL);
tropical_ofst_interface.delete_transducer(implementation.tropical_ofst);
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
internal =
ConversionFunctions::log_ofst_to_hfst_basic_transducer
@@ -5141,7 +5190,7 @@ HfstTransducer &HfstTransducer::convert(ImplementationType type,
this->type = type;
switch (this->type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
implementation.sfst =
ConversionFunctions::hfst_basic_transducer_to_sfst(internal);
@@ -5165,7 +5214,7 @@ HfstTransducer &HfstTransducer::convert(ImplementationType type,
(internal);
delete internal;
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
implementation.log_ofst =
ConversionFunctions::hfst_basic_transducer_to_log_ofst(internal);
@@ -5306,7 +5355,7 @@ HfstTransducer::HfstTransducer(FILE * ifile,
#endif
unsigned int linecount=0;
- if (! is_implementation_type_available(type))
+ if (! is_lean_implementation_type_available(type))
throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
HfstTokenizer::check_utf8_correctness(epsilon_symbol);
@@ -5320,7 +5369,7 @@ HfstTransducer::HfstTransducer(FILE * ifile,
// Conversion is done here.
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
implementation.sfst =
ConversionFunctions::hfst_basic_transducer_to_sfst(&net);
@@ -5332,7 +5381,7 @@ HfstTransducer::HfstTransducer(FILE * ifile,
= ConversionFunctions::hfst_basic_transducer_to_tropical_ofst(&net);
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
implementation.log_ofst
= ConversionFunctions::hfst_basic_transducer_to_log_ofst(&net);
@@ -5382,7 +5431,7 @@ HfstTransducer::HfstTransducer(FILE * ifile,
HFST_THROW(FunctionNotImplementedException);
#endif
- if (! is_implementation_type_available(type))
+ if (! is_lean_implementation_type_available(type))
throw ImplementationTypeNotAvailableException("ImplementationTypeNotAvailableException", __FILE__, __LINE__, type);
HfstTokenizer::check_utf8_correctness(epsilon_symbol);
@@ -5396,7 +5445,7 @@ HfstTransducer::HfstTransducer(FILE * ifile,
// Conversion is done here.
switch (type)
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
case SFST_TYPE:
implementation.sfst =
ConversionFunctions::hfst_basic_transducer_to_sfst(&net);
@@ -5407,7 +5456,7 @@ HfstTransducer::HfstTransducer(FILE * ifile,
implementation.tropical_ofst
= ConversionFunctions::hfst_basic_transducer_to_tropical_ofst(&net);
break;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
case LOG_OPENFST_TYPE:
implementation.log_ofst
= ConversionFunctions::hfst_basic_transducer_to_log_ofst(&net);
diff --git a/libhfst/src/HfstTransducer.h b/libhfst/src/HfstTransducer.h
index fbf254d..9969194 100644
--- a/libhfst/src/HfstTransducer.h
+++ b/libhfst/src/HfstTransducer.h
@@ -20,12 +20,15 @@
#include "HfstSymbolDefs.h"
#include "parsers/LexcCompiler.h"
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
#include "implementations/SfstTransducer.h"
#endif
#if HAVE_OPENFST
#include "implementations/TropicalWeightTransducer.h"
+#endif
+
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
#include "implementations/LogWeightTransducer.h"
#endif
@@ -269,12 +272,12 @@ An example:
/* The union of possible backend implementations. */
union TransducerImplementation
{
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
hfst::implementations::Transducer * sfst;
#endif
#if HAVE_OPENFST
hfst::implementations::StdVectorFst * tropical_ofst;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
hfst::implementations::LogFst * log_ofst;
#endif
#endif
@@ -298,13 +301,13 @@ An example:
TransducerImplementation implementation;
/* Interfaces through which the backend implementations can be accessed */
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
static hfst::implementations::SfstTransducer sfst_interface;
#endif
#if HAVE_OPENFST
static hfst::implementations::TropicalWeightTransducer
tropical_ofst_interface;
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
static hfst::implementations::LogWeightTransducer log_ofst_interface;
#endif
#endif
@@ -367,6 +370,10 @@ An example:
needed by implementation type \a type. */
HFSTDLL static bool is_implementation_type_available(ImplementationType type);
+ /* whether HFST offers at least reading, writing, and conversion functions
+ for implementation type \a type. */
+ HFSTDLL static bool is_lean_implementation_type_available(ImplementationType type);
+
HFSTDLL unsigned int number_of_states() const;
HFSTDLL unsigned int number_of_arcs() const;
@@ -436,7 +443,6 @@ An example:
/* For debugging */
public:
HFSTDLL void print_alphabet();
- protected:
static float get_profile_seconds(ImplementationType type);
#include "hfst_apply_schemas.h"
@@ -1648,6 +1654,10 @@ transducer.transform_weights(&func);
*/
HFSTDLL HfstTransducer &push_weights(PushType type);
+ /**\brief Whether the transducer has non-zero weights in transitions or end states.
+
+ If the transducer is of unweighted type, false is always returned. */
+ HFSTDLL bool has_weights() const;
/** \brief Compile a lexc file in file \a filename into an HfstTransducer
of type \a type and return the transducer. */
diff --git a/libhfst/src/Makefile.am b/libhfst/src/Makefile.am
index c20bc6d..1046015 100644
--- a/libhfst/src/Makefile.am
+++ b/libhfst/src/Makefile.am
@@ -34,6 +34,10 @@ libhfst_la_LIBADD = implementations/libhfstimplementations.la \
if WANT_SFST
libhfst_la_LIBADD += $(top_builddir)/back-ends/sfst/libsfst.la
+else
+if WANT_LEAN_SFST
+ libhfst_la_LIBADD += $(top_builddir)/back-ends/sfst/libsfst.la
+endif
endif
if WANT_FOMA
@@ -100,7 +104,7 @@ HFST_HDRS = \
hfstinclude_HEADERS = $(HFST_HDRS)
-libhfst_la_LDFLAGS = -no-undefined -version-info 48:0:0
+libhfst_la_LDFLAGS = -no-undefined -version-info 49:0:0
LIBHFST_TSTS=HfstApply HfstInputStream HfstTransducer \
HfstOutputStream HfstXeroxRules HfstRules HfstSymbolDefs \
diff --git a/libhfst/src/implementations/ConvertFomaTransducer.cc b/libhfst/src/implementations/ConvertFomaTransducer.cc
index c5e74cc..6a56ffe 100644
--- a/libhfst/src/implementations/ConvertFomaTransducer.cc
+++ b/libhfst/src/implementations/ConvertFomaTransducer.cc
@@ -7,22 +7,23 @@
// See the file COPYING included with this distribution for more
// information.
+#ifndef MAIN_TEST
+
#if HAVE_CONFIG_H
# include <config.h>
#endif
#if HAVE_FOMA
- #ifndef _FOMALIB_H_
- #define _FOMALIB_H_
- #include "back-ends/foma/fomalib.h"
- #endif
+
+#ifndef _FOMALIB_H_
+ #define _FOMALIB_H_
+ #include "back-ends/foma/fomalib.h"
#endif
#include "ConvertTransducerFormat.h"
#include "HfstBasicTransducer.h"
#include "FomaTransducer.h"
-#ifndef MAIN_TEST
namespace hfst { namespace implementations
{
@@ -33,9 +34,6 @@ namespace hfst { namespace implementations
---------------------------------------------------------- */
-#if HAVE_FOMA
-
-
/* -----------------------------------------------------------------
Internal functions used by the actual conversion functions.
@@ -331,12 +329,9 @@ namespace hfst { namespace implementations
return net;
}
+ }}
#endif // HAVE_FOMA
-
-
-
- }}
#else // MAIN_TEST was defined
#include <iostream>
diff --git a/libhfst/src/implementations/ConvertLogWeightTransducer.cc b/libhfst/src/implementations/ConvertLogWeightTransducer.cc
index a0a4432..527a681 100644
--- a/libhfst/src/implementations/ConvertLogWeightTransducer.cc
+++ b/libhfst/src/implementations/ConvertLogWeightTransducer.cc
@@ -7,10 +7,14 @@
// See the file COPYING included with this distribution for more
// information.
+#ifndef MAIN_TEST
+
#if HAVE_CONFIG_H
# include <config.h>
#endif
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
+
#include "ConvertTransducerFormat.h"
#include "HfstBasicTransducer.h"
#include "LogWeightTransducer.h"
@@ -22,6 +26,7 @@
#endif // _MSC_VER
#ifndef _MSC_VER
+#if HAVE_OPENFST_LOG // if HAVE_LEAN_OPENFST_LOG is requested, these are not defined elsewhere (?)
namespace fst
{
extern template class LogWeightTpl<float>;
@@ -30,15 +35,12 @@ namespace fst
extern template class ArcIterator<LogFst>;
extern template class StateIterator<LogFst>;
}
-#endif // _MSC_VER
+#endif
+#endif // #ifndef _MSC_VER
-#ifndef MAIN_TEST
namespace hfst { namespace implementations
{
-#if HAVE_OPENFST
-#if HAVE_OPENFST_LOG
-
/* --- Conversion between log OpenFst and HfstBasicTransducer --- */
/* Create an HfstBasicTransducer equivalent to an OpenFst log weight
@@ -294,11 +296,10 @@ namespace hfst { namespace implementations
return t;
}
-#endif
-#endif // HAVE_OPENFST
+ }}
+#endif // HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
- }}
#else // MAIN_TEST was defined
#include <iostream>
diff --git a/libhfst/src/implementations/ConvertSfstTransducer.cc b/libhfst/src/implementations/ConvertSfstTransducer.cc
index 5f0b49e..9572240 100644
--- a/libhfst/src/implementations/ConvertSfstTransducer.cc
+++ b/libhfst/src/implementations/ConvertSfstTransducer.cc
@@ -7,14 +7,16 @@
// See the file COPYING included with this distribution for more
// information.
+#ifndef MAIN_TEST
+
#if HAVE_CONFIG_H
# include <config.h>
#endif
-#if HAVE_SFST
- #include "back-ends/sfst/interface.h"
- #include "back-ends/sfst/fst.h"
-#endif
+#if HAVE_SFST || HAVE_LEAN_SFST
+
+#include "back-ends/sfst/interface.h"
+#include "back-ends/sfst/fst.h"
#include "ConvertTransducerFormat.h"
#include "HfstBasicTransducer.h"
@@ -22,7 +24,7 @@
#include <stdexcept>
-#ifndef MAIN_TEST
+
namespace hfst { namespace implementations
{
@@ -33,8 +35,6 @@ namespace hfst { namespace implementations
----------------------------------------------------------- */
-#if HAVE_SFST
-
SFST::VType VMARK = 10000;
/* Recursively copy all transitions of \a node to \a net.
@@ -223,10 +223,10 @@ namespace hfst { namespace implementations
return t;
}
-
-#endif // HAVE_SFST
}}
+#endif // #if HAVE_SFST || HAVE_LEAN_SFST
+
#else // MAIN_TEST was defined
#include <iostream>
diff --git a/libhfst/src/implementations/ConvertTransducerFormat.cc b/libhfst/src/implementations/ConvertTransducerFormat.cc
index f4ca4c5..02c3041 100644
--- a/libhfst/src/implementations/ConvertTransducerFormat.cc
+++ b/libhfst/src/implementations/ConvertTransducerFormat.cc
@@ -11,16 +11,16 @@
# include <config.h>
#endif
-#if HAVE_SFST
-#include "back-ends/sfst/fst.h"
-#endif // HAVE_SFST
+//#if HAVE_SFST || HAVE_LEAN_SFST
+//#include "back-ends/sfst/fst.h"
+//#endif // HAVE_SFST || HAVE_LEAN_SFST
-#if HAVE_FOMA
-#ifndef _FOMALIB_H_
-#define _FOMALIB_H_
-#include "back-ends/foma/fomalib.h"
-#endif // _FOMALIB_H_
-#endif // HAVE_FOMA
+//#if HAVE_FOMA
+//#ifndef _FOMALIB_H_
+//#define _FOMALIB_H_
+//#include "back-ends/foma/fomalib.h"
+//#endif // _FOMALIB_H_
+//#endif // HAVE_FOMA
#include "ConvertTransducerFormat.h"
#include "optimized-lookup/convert.h"
@@ -80,13 +80,13 @@ namespace hfst { namespace implementations
HfstBasicTransducer * ConversionFunctions::
hfst_transducer_to_hfst_basic_transducer
(const hfst::HfstTransducer &t) {
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
if (t.type == SFST_TYPE) {
HfstBasicTransducer * retval = sfst_to_hfst_basic_transducer(t.implementation.sfst);
retval->name = t.get_name();
return retval;
}
-#endif // HAVE_SFST
+#endif // HAVE_SFST || HAVE_LEAN_SFST
#if HAVE_OPENFST
if (t.type == TROPICAL_OPENFST_TYPE) {
@@ -95,7 +95,7 @@ namespace hfst { namespace implementations
retval->name = t.get_name();
return retval;
}
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
if (t.type == LOG_OPENFST_TYPE) {
HfstBasicTransducer * retval = log_ofst_to_hfst_basic_transducer(t.implementation.log_ofst);
retval->name = t.get_name();
diff --git a/libhfst/src/implementations/ConvertTransducerFormat.h b/libhfst/src/implementations/ConvertTransducerFormat.h
index c910fee..2557f1c 100644
--- a/libhfst/src/implementations/ConvertTransducerFormat.h
+++ b/libhfst/src/implementations/ConvertTransducerFormat.h
@@ -42,17 +42,6 @@ namespace fst
#endif // OPENFST_VERSION_1_5_4
#endif // HAVE_OPENFST
-/*#if HAVE_SFST
-#include "back-ends/sfst/fst.h"
-#endif // HAVE_SFST
-
-#if HAVE_FOMA
-#ifndef _FOMALIB_H_
-#define _FOMALIB_H_
-#include "back-ends/foma/fomalib.h"
-#endif // _FOMALIB_H_
-#endif // HAVE_FOMA*/
-
#if HAVE_XFSM
#include "xfsm/xfsm_api.h"
#endif
@@ -67,7 +56,6 @@ namespace fst
#include "../HfstExceptionDefs.h"
#include "optimized-lookup/transducer.h"
-//#include "HfstConstantTransducer.h"
namespace SFST { class Node; class Transducer; }
struct fsm;
@@ -118,7 +106,7 @@ namespace implementations {
static HfstBasicTransducer * hfst_transducer_to_hfst_basic_transducer
(const hfst::HfstTransducer &t);
-#if HAVE_SFST
+#if HAVE_SFST || HAVE_LEAN_SFST
static void sfst_to_hfst_basic_transducer
( SFST::Node *node,
HfstBasicTransducer *net,
@@ -129,46 +117,13 @@ namespace implementations {
static SFST::Transducer * hfst_basic_transducer_to_sfst
(const HfstBasicTransducer * t);
-
- /* static void sfst_to_hfst_fast_transducer
- ( SFST::Node *node,
- HfstFastTransducer *net, NumberVector &harmonization_vector);
-
- static HfstFastTransducer * sfst_to_hfst_fast_transducer
- (SFST::Transducer * t);
-
- static SFST::Transducer * hfst_fast_transducer_to_sfst
- (const HfstFastTransducer * t);
-
- static void sfst_to_hfst_constant_transducer
- ( SFST::Node *node,
- HfstConstantTransducer *net);
-
- static HfstConstantTransducer * sfst_to_hfst_constant_transducer
- (SFST::Transducer * t);
-
- static SFST::Transducer * hfst_constant_transducer_to_sfst
- (const HfstConstantTransducer * t); */
-#endif // HAVE_SFST
+#endif // HAVE_SFST || HAVE_LEAN_SFST
#if HAVE_FOMA
static HfstBasicTransducer * foma_to_hfst_basic_transducer(fsm * t);
static fsm * hfst_basic_transducer_to_foma
(const HfstBasicTransducer * t);
-
-
- /* static HfstFastTransducer * foma_to_hfst_fast_transducer(struct fsm * t);
-
- static struct fsm * hfst_fast_transducer_to_foma
- (const HfstFastTransducer * t);
-
-
- static HfstConstantTransducer * foma_to_hfst_constant_transducer
- (struct fsm * t);
-
- static struct fsm * hfst_constant_transducer_to_foma
- (const HfstConstantTransducer * t); */
#endif // HAVE_FOMA
#if HAVE_XFSM
@@ -187,22 +142,7 @@ namespace implementations {
static fst::StdVectorFst * hfst_basic_transducer_to_tropical_ofst
(const HfstBasicTransducer * t);
-
- /* static HfstFastTransducer * tropical_ofst_to_hfst_fast_transducer
- (fst::StdVectorFst * t, bool has_hfst_header=true);
-
- static fst::StdVectorFst * hfst_fast_transducer_to_tropical_ofst
- (const HfstFastTransducer * t);
-
-
-
- static HfstConstantTransducer * tropical_ofst_to_hfst_constant_transducer
- (fst::StdVectorFst * t, bool has_hfst_header=true);
-
- static fst::StdVectorFst * hfst_constant_transducer_to_tropical_ofst
- (const HfstConstantTransducer * t); */
-
-#if HAVE_OPENFST_LOG
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
static HfstBasicTransducer * log_ofst_to_hfst_basic_transducer
(fst::LogFst * t, bool had_hfst_header=true);
@@ -212,24 +152,9 @@ namespace implementations {
static fst::LogFst * hfst_basic_transducer_to_log_ofst
(const HfstBasicTransducer * t);
-
-
- /* static HfstFastTransducer * log_ofst_to_hfst_fast_transducer
- (LogFst * t, bool has_hfst_header=true);
-
- static LogFst * hfst_fast_transducer_to_log_ofst
- (const HfstFastTransducer * t);
-
-
- static HfstConstantTransducer * log_ofst_to_hfst_constant_transducer
- (LogFst * t, bool had_hfst_header=true);
-
- static LogFst * hfst_constant_transducer_to_log_ofst
- (const HfstConstantTransducer * t); */
-
#endif
-#endif // HAVE_OPENFST
+#endif // HAVE_OPENFST || HAVE_LEAN_OPENFST_LOG
static HfstBasicTransducer * hfst_ol_to_hfst_basic_transducer
diff --git a/libhfst/src/implementations/FomaTransducer.cc b/libhfst/src/implementations/FomaTransducer.cc
index 3b034e9..72a1854 100644
--- a/libhfst/src/implementations/FomaTransducer.cc
+++ b/libhfst/src/implementations/FomaTransducer.cc
@@ -7,6 +7,14 @@
// See the file COPYING included with this distribution for more
// information.
+#ifndef MAIN_TEST
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#if HAVE_FOMA
+
#ifndef _FOMALIB_H_
#define _FOMALIB_H_
#include "back-ends/foma/fomalib.h"
@@ -14,9 +22,30 @@
#include "FomaTransducer.h"
-#ifndef MAIN_TEST
+#ifdef PROFILE_FOMA
+#include <ctime>
+#endif
+
namespace hfst { namespace implementations {
+ // add this to the beginning of code block to be profiled
+ //#ifdef PROFILE_FOMA
+ //clock_t startclock = clock();
+ //#endif
+
+ // and this to the end of the block
+ //#ifdef PROFILE_FOMA
+ //clock_t endclock = clock();
+ //foma_seconds = foma_seconds +
+ // ( (float)(endclock - startclock) / CLOCKS_PER_SEC);
+ //#endif
+
+ float foma_seconds=0;
+ float FomaTransducer::get_profile_seconds() {
+ return foma_seconds;
+ }
+
+
// ---------- FomaInputStream functions ----------
@@ -334,13 +363,13 @@ namespace hfst { namespace implementations {
// returns the argument, so a copy is taken here
return fsm_determinize(fsm_copy(t));
}
-
+
fsm * FomaTransducer::minimize(fsm * t)
{
// returns the argument, so a copy is taken here
return fsm_minimize(fsm_copy(t));
}
-
+
fsm * FomaTransducer::remove_epsilons(fsm * t)
{
// returns the argument, so a copy is taken here
@@ -1134,17 +1163,19 @@ static inline int explode_line (char *buf, int *values) {
}
} }
+#endif // HAVE_FOMA
#else // MAIN_TEST was defined
#include <cstdlib>
#include <cassert>
#include <iostream>
-using namespace hfst::implementations;
int main(int argc, char * argv[])
{
+#if HAVE_FOMA
std::cout << "Unit tests for " __FILE__ ":";
+ using namespace hfst::implementations;
fsm * epsilon
= FomaTransducer::define_transducer("@_EPSILON_SYMBOL_@");
fsm * epsilon_i = FomaTransducer::extract_input_language(epsilon);
@@ -1157,5 +1188,9 @@ int main(int argc, char * argv[])
std::cout << std::endl << "ok" << std::endl;
return EXIT_SUCCESS;
+#else // HAVE_FOMA
+ std::cout << "Skipping unit tests for " << __FILE__ << ", FomaTransducer has not been enabled" << std::endl;
+ return 77;
+#endif // HAVE_FOMA
}
#endif // MAIN_TEST
diff --git a/libhfst/src/implementations/FomaTransducer.h b/libhfst/src/implementations/FomaTransducer.h
index c881cc0..d712f6c 100644
--- a/libhfst/src/implementations/FomaTransducer.h
+++ b/libhfst/src/implementations/FomaTransducer.h
@@ -16,11 +16,6 @@
#include "HfstFlagDiacritics.h"
#include <stdlib.h>
-//#ifndef _FOMALIB_H_
-//#define _FOMALIB_H_
-//#include "back-ends/foma/fomalib.h"
-//#endif
-
struct fsm;
#include <cstdio>
@@ -72,7 +67,6 @@ namespace hfst {
private:
std::string filename;
FILE *ofile;
- //void write_3_0_library_header(FILE *file);
public:
FomaOutputStream(void);
FomaOutputStream(const std::string &filename);
@@ -157,6 +151,8 @@ namespace hfst {
static fsm * read_lexc(const std::string &filename, bool verbose);
#endif
+ static float get_profile_seconds();
+
static unsigned int number_of_states(fsm * net);
static unsigned int number_of_arcs(fsm * net);
diff --git a/libhfst/src/implementations/LogWeightTransducer.cc b/libhfst/src/implementations/LogWeightTransducer.cc
index d6cdda8..87113e3 100644
--- a/libhfst/src/implementations/LogWeightTransducer.cc
+++ b/libhfst/src/implementations/LogWeightTransducer.cc
@@ -7,6 +7,14 @@
// See the file COPYING included with this distribution for more
// information.
+#ifndef MAIN_TEST
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
+
#include "LogWeightTransducer.h"
#include "HfstSymbolDefs.h"
@@ -16,23 +24,8 @@
#include "back-ends/openfst/src/include/fst/fstlib.h"
#endif // _MSC_VER
-
-#ifndef MAIN_TEST
namespace hfst { namespace implementations
{
- float log_seconds_in_harmonize=0;
-
- float LogWeightTransducer::get_profile_seconds() {
- return log_seconds_in_harmonize;
- }
-
- bool openfst_log_use_hopcroft=false;
-
- void openfst_log_set_hopcroft(bool value) {
- openfst_log_use_hopcroft=value;
- }
-
- void initialize_symbol_tables(LogFst *t);
LogWeightInputStream::LogWeightInputStream(void):
i_stream(),input_stream(cin)
@@ -54,6 +47,173 @@ namespace hfst { namespace implementations
void LogWeightInputStream::stream_unget(char c) {
input_stream.putback(c); }
+
+ /* Skip the identifier string "LOG_OFST_TYPE" */
+ void LogWeightInputStream::skip_identifier_version_3_0(void)
+ { input_stream.ignore(14); }
+
+ void LogWeightInputStream::skip_hfst_header(void)
+ {
+ input_stream.ignore(6);
+ //char c;
+ //i_stream.get(c);
+ //switch (c)
+ //{
+ //case 0:
+ skip_identifier_version_3_0();
+ //break;
+ //default:
+ //assert(false);
+ //}
+ }
+
+ void LogWeightInputStream::close(void)
+ {
+ if (filename != string())
+ { i_stream.close(); }
+ }
+ bool LogWeightInputStream::is_eof(void) const
+ {
+ return input_stream.peek() == EOF;
+ }
+ bool LogWeightInputStream::is_bad(void) const
+ {
+ if (filename == string())
+ { return std::cin.bad(); }
+ else
+ { return input_stream.bad(); }
+ }
+ bool LogWeightInputStream::is_good(void) const
+ {
+ if(is_eof())
+ return false;
+ if (filename == string())
+ { return std::cin.good(); }
+ else
+ { return input_stream.good(); }
+ }
+
+ bool LogWeightInputStream::is_fst(void) const
+ {
+ return is_fst(input_stream);
+ }
+
+ bool LogWeightInputStream::is_fst(FILE * f)
+ {
+ if (f == NULL)
+ { return false; }
+ int c = getc(f);
+ ungetc(c, f);
+ return c == 0xd6;
+ }
+
+ bool LogWeightInputStream::is_fst(istream &s)
+ {
+ return s.good() && (s.peek() == 0xd6);
+ }
+
+ bool LogWeightInputStream::operator() (void) const
+ { return is_good(); }
+
+ void LogWeightInputStream::ignore(unsigned int n)
+ { input_stream.ignore(n); }
+
+ LogFst * LogWeightInputStream::read_transducer()
+ {
+ if (is_eof())
+ { //throw StreamIsClosedException();
+ HFST_THROW(StreamIsClosedException); }
+ LogFst * t;
+ FstHeader header;
+ try
+ {
+ if (filename == string())
+ {
+ header.Read(input_stream,"STDIN");
+ t = static_cast<LogFst*>
+ (LogFst::Read(input_stream,
+ FstReadOptions("STDIN",
+ &header)));
+ }
+ else
+ {
+ header.Read(input_stream,filename);
+ t = static_cast<LogFst*>
+ (LogFst::Read(input_stream,
+ FstReadOptions(filename,
+ &header)));
+ }
+ if (t == NULL)
+ { //throw TransducerHasWrongTypeException();
+ HFST_THROW(TransducerHasWrongTypeException); }
+ }
+ //catch (TransducerHasWrongTypeException e)
+ catch (const HfstException e)
+ { throw e; }
+
+ try
+ {
+ return t;
+ }
+ //catch (HfstInterfaceException e)
+ catch (const HfstException e)
+ { throw e; }
+ }
+
+ LogWeightOutputStream::LogWeightOutputStream(void):
+ filename(std::string()), output_stream(std::cout)
+ {
+ if (!output_stream)
+ fprintf(stderr, "LogWeightOutputStream: ERROR: failbit set (3).\n");
+ }
+
+ LogWeightOutputStream::LogWeightOutputStream(const std::string &str):
+ filename(std::string(str)),o_stream(str.c_str(),std::ios::out),
+ output_stream(o_stream)
+ {}
+
+ void LogWeightOutputStream::write(const char &c)
+ {
+ output_stream.put(char(c));
+ }
+
+ void LogWeightOutputStream::write_transducer(LogFst * transducer)
+ {
+ if (!output_stream)
+ fprintf(stderr, "LogWeightOutputStream: ERROR: failbit set (1).\n");
+ /* When writing a transducer, both input and output symbol tables are
+ included. */
+ fst::SymbolTable output_st(*(transducer->InputSymbols()));
+ transducer->SetOutputSymbols(&output_st);
+ transducer->Write(output_stream,FstWriteOptions()); }
+
+ void LogWeightOutputStream::close(void)
+ {
+ if (filename != string())
+ { o_stream.close(); }
+ }
+
+ void LogWeightTransducer::delete_transducer(LogFst * t)
+ {
+ delete t;
+ }
+
+#if HAVE_OPENFST_LOG
+
+ float log_seconds_in_harmonize=0;
+
+ float LogWeightTransducer::get_profile_seconds() {
+ return log_seconds_in_harmonize;
+ }
+
+ bool openfst_log_use_hopcroft=false;
+
+ void openfst_log_set_hopcroft(bool value) {
+ openfst_log_use_hopcroft=value;
+ }
+
+ void initialize_symbol_tables(LogFst *t);
+
void LogWeightTransducer::remove_symbol_table(LogFst *t)
{
t->SetInputSymbols(NULL);
@@ -819,199 +979,6 @@ namespace hfst { namespace implementations
}
-
-
- /* Skip the identifier string "LOG_OFST_TYPE" */
- void LogWeightInputStream::skip_identifier_version_3_0(void)
- { input_stream.ignore(14); }
-
- void LogWeightInputStream::skip_hfst_header(void)
- {
- input_stream.ignore(6);
- //char c;
- //i_stream.get(c);
- //switch (c)
- //{
- //case 0:
- skip_identifier_version_3_0();
- //break;
- //default:
- //assert(false);
- //}
- }
-
- void LogWeightInputStream::close(void)
- {
- if (filename != string())
- { i_stream.close(); }
- }
- bool LogWeightInputStream::is_eof(void) const
- {
- return input_stream.peek() == EOF;
- }
- bool LogWeightInputStream::is_bad(void) const
- {
- if (filename == string())
- { return std::cin.bad(); }
- else
- { return input_stream.bad(); }
- }
- bool LogWeightInputStream::is_good(void) const
- {
- if(is_eof())
- return false;
- if (filename == string())
- { return std::cin.good(); }
- else
- { return input_stream.good(); }
- }
-
- bool LogWeightInputStream::is_fst(void) const
- {
- return is_fst(input_stream);
- }
-
- bool LogWeightInputStream::is_fst(FILE * f)
- {
- if (f == NULL)
- { return false; }
- int c = getc(f);
- ungetc(c, f);
- return c == 0xd6;
- }
-
- bool LogWeightInputStream::is_fst(istream &s)
- {
- return s.good() && (s.peek() == 0xd6);
- }
-
- bool LogWeightInputStream::operator() (void) const
- { return is_good(); }
-
- void LogWeightInputStream::ignore(unsigned int n)
- { input_stream.ignore(n); }
-
- LogFst * LogWeightInputStream::read_transducer()
- {
- if (is_eof())
- { //throw StreamIsClosedException();
- HFST_THROW(StreamIsClosedException); }
- LogFst * t;
- FstHeader header;
- try
- {
- if (filename == string())
- {
- header.Read(input_stream,"STDIN");
- t = static_cast<LogFst*>
- (LogFst::Read(input_stream,
- FstReadOptions("STDIN",
- &header)));
- }
- else
- {
- header.Read(input_stream,filename);
- t = static_cast<LogFst*>
- (LogFst::Read(input_stream,
- FstReadOptions(filename,
- &header)));
- }
- if (t == NULL)
- { //throw TransducerHasWrongTypeException();
- HFST_THROW(TransducerHasWrongTypeException); }
- }
- //catch (TransducerHasWrongTypeException e)
- catch (const HfstException e)
- { throw e; }
-
- try
- {
- return t;
- }
- //catch (HfstInterfaceException e)
- catch (const HfstException e)
- { throw e; }
- }
-
- /*
- LogWeightStateIterator::LogWeightStateIterator(LogFst * t):
- iterator(new StateIterator<LogFst>(*t))
- {}
-
- LogWeightStateIterator::~LogWeightStateIterator(void)
- { delete iterator; }
-
- void LogWeightStateIterator::next(void)
- {
- iterator->Next();
- }
-
- bool LogWeightStateIterator::done(void)
- {
- return iterator->Done();
- }
-
- LogWeightState LogWeightStateIterator::value(void)
- {
- return iterator->Value();
- }
-
-
-
- LogWeightTransition::LogWeightTransition(const LogArc &arc, LogFst *t):
- arc(arc), t(t)
- {}
-
- LogWeightTransition::~LogWeightTransition(void)
- {}
-
- std::string LogWeightTransition::get_input_symbol(void) const
- {
- return t->InputSymbols()->Find(arc.ilabel);
- }
-
- std::string LogWeightTransition::get_output_symbol(void) const
- {
- return t->InputSymbols()->Find(arc.olabel);
- }
-
- LogWeightState LogWeightTransition::get_target_state(void) const
- {
- return arc.nextstate;
- }
-
- LogWeight LogWeightTransition::get_weight(void) const
- {
- return arc.weight;
- }
-
-
-
- LogWeightTransitionIterator::LogWeightTransitionIterator
- (LogFst *t, StateId state):
- arc_iterator(new ArcIterator<LogFst>(*t, state)),
- t(t)
- {}
-
- LogWeightTransitionIterator::~LogWeightTransitionIterator(void)
- {}
-
- void LogWeightTransitionIterator::next()
- {
- arc_iterator->Next();
- }
-
- bool LogWeightTransitionIterator::done()
- {
- return arc_iterator->Done();
- }
-
- LogWeightTransition LogWeightTransitionIterator::value()
- {
- return LogWeightTransition(arc_iterator->Value(), this->t);
- }*/
-
-
fst::SymbolTable LogWeightTransducer::create_symbol_table(std::string name) {
fst::SymbolTable st(name);
st.AddSymbol(internal_epsilon, 0);
@@ -1044,11 +1011,6 @@ namespace hfst { namespace implementations
return t;
}
- void LogWeightTransducer::delete_transducer(LogFst * t)
- {
- delete t;
- }
-
LogFst * LogWeightTransducer::define_transducer(unsigned int number)
{
LogFst * t = new LogFst;
@@ -1548,51 +1510,6 @@ namespace hfst { namespace implementations
return t;
}
- /*static LogFst * insert_freely(LogFst * t, const NumberPair &number_pair)
- {
- for (fst::StateIterator<LogFst> siter(*t); !siter.Done(); siter.Next()) {
- StateId state_id = siter.Value();
- t->AddArc(state_id, fst::LogArc(number_pair.first, number_pair.second,
- 0, state_id));
- }
- return t;
- }*/
-
- /*
- LogFst * LogWeightTransducer::substitute
- (LogFst *t, void (*func)(std::string &isymbol, std::string &osymbol) )
- {
- LogFst * tc = t->Copy();
- SymbolTable * st = tc->InputSymbols()->Copy();
- assert(st != NULL);
-
- for (fst::StateIterator<LogFst> siter(*tc);
- not siter.Done(); siter.Next())
- {
- StateId s = siter.Value();
- for (fst::MutableArcIterator<LogFst> aiter(tc,s);
- !aiter.Done(); aiter.Next())
- {
- const LogArc &arc = aiter.Value(); // current values
- LogArc new_arc; // new values
-
- std::string istring = st->Find(arc.ilabel);
- std::string ostring = st->Find(arc.olabel);
- func(istring,ostring);
- new_arc.ilabel = st->AddSymbol(istring);
- new_arc.olabel = st->AddSymbol(ostring);
- // copy weight and next state as such
- new_arc.weight = arc.weight.Value();
- new_arc.nextstate = arc.nextstate;
- aiter.SetValue(new_arc);
- }
- }
- tc->SetInputSymbols(st);
- delete st;
- return tc;
- }
- */
-
LogFst * LogWeightTransducer::substitute
(LogFst * t,unsigned int old_key,unsigned int new_key)
{
@@ -2247,32 +2164,6 @@ namespace hfst { namespace implementations
}
}
}
-
- /*
- int lp=lpos;
- int up=upos;
-
- if (arc.ilabel != 0 &&
- (!filter_fd || fd_state_stack->back().get_table().
- get_operation(arc.ilabel)==NULL))
- {
- std::string str = t->InputSymbols()->Find(arc.ilabel);
- if(lpos+str.length() >= lbuffer.size())
- lbuffer.resize(lbuffer.size()*2, 0);
- strcpy(&lbuffer[lpos], str.c_str());
- lp += str.length();
- }
- if (arc.olabel != 0 &&
- (!filter_fd || fd_state_stack->back().get_table().
- get_operation(arc.olabel)==NULL))
- {
- std::string str = t->InputSymbols()->Find(arc.olabel);
- if(upos+str.length() > ubuffer.size())
- ubuffer.resize(ubuffer.size()*2, 0);
- strcpy(&ubuffer[upos], str.c_str());
- up += str.length();
- }
- */
/* Handle spv here. Special symbols (flags, epsilons) are
always inserted. */
@@ -2361,53 +2252,21 @@ namespace hfst { namespace implementations
return;
}
-
-
- LogWeightOutputStream::LogWeightOutputStream(void):
- filename(std::string()), output_stream(std::cout)
- {
- if (!output_stream)
- fprintf(stderr, "LogWeightOutputStream: ERROR: failbit set (3).\n");
- }
-
- LogWeightOutputStream::LogWeightOutputStream(const std::string &str):
- filename(std::string(str)),o_stream(str.c_str(),std::ios::out),
- output_stream(o_stream)
- {}
-
- void LogWeightOutputStream::write(const char &c)
- {
- output_stream.put(char(c));
- }
-
- void LogWeightOutputStream::write_transducer(LogFst * transducer)
- {
- if (!output_stream)
- fprintf(stderr, "LogWeightOutputStream: ERROR: failbit set (1).\n");
- /* When writing a transducer, both input and output symbol tables are
- included. */
- fst::SymbolTable output_st(*(transducer->InputSymbols()));
- transducer->SetOutputSymbols(&output_st);
- transducer->Write(output_stream,FstWriteOptions()); }
-
- void LogWeightOutputStream::close(void)
- {
- if (filename != string())
- { o_stream.close(); }
- }
+#endif // #if HAVE_OPENFST_LOG
}
}
+#endif // HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
#else // MAIN_TEST was defined
#include <cassert>
#include <cstdlib>
#include <iostream>
-using namespace hfst::implementations;
-
int main(int argc, char * argv[])
{
+#if HAVE_OPENFST_LOG
std::cout << "Unit tests for " __FILE__ ":";
+ using namespace hfst::implementations;
LogWeightTransducer ofst;
LogFst * t = ofst.create_empty_transducer();
delete t;
@@ -2415,5 +2274,9 @@ int main(int argc, char * argv[])
delete t;
std::cout << std::endl << "ok" << std::endl;
return EXIT_SUCCESS;
+#else // HAVE_OPENFST_LOG
+ std::cout << "Skipping unit tests for " << __FILE__ << ", LogWeightTransducer has not been enabled" << std::endl;
+ return 77;
+#endif // HAVE_OPENFST_LOG
}
#endif // MAIN_TEST
diff --git a/libhfst/src/implementations/LogWeightTransducer.h b/libhfst/src/implementations/LogWeightTransducer.h
index c607a33..2d0653a 100644
--- a/libhfst/src/implementations/LogWeightTransducer.h
+++ b/libhfst/src/implementations/LogWeightTransducer.h
@@ -10,19 +10,15 @@
#ifndef _LOG_WEIGHT_TRANSDUCER_H_
#define _LOG_WEIGHT_TRANSDUCER_H_
-#include "HfstSymbolDefs.h"
-#include "HfstExceptionDefs.h"
-#include "HfstFlagDiacritics.h"
-
#if HAVE_CONFIG_H
#include "../../../config.h"
#endif
-//#ifdef _MSC_VER
-// #include "back-ends/openfstwin/src/include/fst/fstlib.h"
-//#else
-// #include "back-ends/openfst/src/include/fst/fstlib.h"
-//#endif // _MSC_VER
+#if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
+
+#include "HfstSymbolDefs.h"
+#include "HfstExceptionDefs.h"
+#include "HfstFlagDiacritics.h"
#include "HfstExtractStrings.h"
#include <cstdio>
@@ -30,7 +26,6 @@
#include <sstream>
#include <iosfwd>
#include <fstream>
-//#include "HfstAlphabet.h"
#ifdef OPENFST_VERSION_1_5_4
#include "back-ends/openfst/src/include/fst/fst-decl.h"
@@ -70,7 +65,6 @@ namespace implementations
{
using namespace fst;
;
- //typedef LogArc::StateId StateId;
typedef unsigned int StateId;
typedef VectorFst<LogArc> LogFst;
@@ -118,7 +112,6 @@ namespace implementations
std::string filename;
std::ofstream o_stream;
std::ostream &output_stream;
- //void write_3_0_library_header(std::ostream &out);
public:
LogWeightOutputStream(void);
LogWeightOutputStream(const std::string &filename);
@@ -127,58 +120,12 @@ namespace implementations
void write_transducer(LogFst * transducer);
};
- /* class LogWeightTransitionIterator;
-
- typedef StateId LogWeightState;
-
- class LogWeightStateIterator
- {
- protected:
- StateIterator<LogFst> * iterator;
- public:
- LogWeightStateIterator(LogFst * t);
- ~LogWeightStateIterator(void);
- void next(void);
- bool done(void);
- LogWeightState value(void);
- };
-
-
- class LogWeightTransition
- {
- protected:
- LogArc arc;
- LogFst * t;
- public:
- LogWeightTransition(const LogArc &arc, LogFst *t);
- ~LogWeightTransition(void);
- std::string get_input_symbol(void) const;
- std::string get_output_symbol(void) const;
- LogWeightState get_target_state(void) const;
- LogWeight get_weight(void) const;
- };
-
-
- class LogWeightTransitionIterator
- {
- protected:
- ArcIterator<LogFst> * arc_iterator;
- LogFst * t;
- public:
- LogWeightTransitionIterator(LogFst * t, StateId state);
- ~LogWeightTransitionIterator(void);
- void next(void);
- bool done(void);
- LogWeightTransition value(void);
- };
- */
-
class LogWeightTransducer
{
+#if HAVE_OPENFST_LOG
public:
static LogFst * create_empty_transducer(void);
static LogFst * create_epsilon_transducer(void);
- void delete_transducer(LogFst * t);
// string versions
static LogFst * define_transducer(const std::string &symbol);
@@ -336,7 +283,15 @@ namespace implementations
static StateId get_initial_state(LogFst *t);
static void represent_empty_transducer_as_having_one_state(LogFst *t);
+#endif // HAVE_OPENFST_LOG
+ // needed in lean implementation
+ public:
+ void delete_transducer(LogFst * t);
+
};
} }
+
+#endif // #if HAVE_OPENFST_LOG || HAVE_LEAN_OPENFST_LOG
+
#endif
diff --git a/libhfst/src/implementations/Makefile.am b/libhfst/src/implementations/Makefile.am
index 34ce58c..fd2e87c 100644
--- a/libhfst/src/implementations/Makefile.am
+++ b/libhfst/src/implementations/Makefile.am
@@ -44,11 +44,19 @@ endif
# the bridges
if WANT_SFST
MAYBE_SFST=SfstTransducer.cc
+else
+if WANT_LEAN_SFST
+MAYBE_SFST=SfstTransducer.cc
+endif
endif
if WANT_OPENFST
MAYBE_OPENFST=TropicalWeightTransducer.cc
if WANT_OPENFST_LOG
MAYBE_OPENFST += LogWeightTransducer.cc
+else
+if WANT_LEAN_OPENFST_LOG
+MAYBE_OPENFST += LogWeightTransducer.cc
+endif
endif
endif
if WANT_FOMA
@@ -218,4 +226,4 @@ compose_intersect_ComposeIntersectUtilities_LDADD=../libhfst.la
#HfstConstantTransducer_CXXFLAGS=-DMAIN_TEST
#HfstConstantTransducer_LDADD=../libhfst.la
-TESTS=$(FOMA_TSTS) $(HFSTOL_TSTS) $(OFST_TSTS) $(SFST_TSTS) $(LIBHFST_TSTS) $(XFSM_TSTS)
\ No newline at end of file
+TESTS=$(FOMA_TSTS) $(HFSTOL_TSTS) $(OFST_TSTS) $(SFST_TSTS) $(LIBHFST_TSTS) $(XFSM_TSTS)
diff --git a/libhfst/src/implementations/SfstTransducer.cc b/libhfst/src/implementations/SfstTransducer.cc
index 0964452..63fa7c8 100644
--- a/libhfst/src/implementations/SfstTransducer.cc
+++ b/libhfst/src/implementations/SfstTransducer.cc
@@ -7,6 +7,14 @@
// See the file COPYING included with this distribution for more
// information.
+#ifndef MAIN_TEST
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#if HAVE_SFST || HAVE_LEAN_SFST
+
#include "back-ends/sfst/interface.h"
#include "back-ends/sfst/fst.h"
#include "SfstTransducer.h"
@@ -15,45 +23,8 @@
using namespace SFST;
-#ifndef MAIN_TEST
namespace hfst { namespace implementations {
- class HfstNode2Int {
-
- struct hashf {
- size_t operator()(const SFST::Node *node) const {
- return (size_t)node;
- }
- };
- struct equalf {
- int operator()(const SFST::Node *n1, const SFST::Node *n2) const {
- return (n1 == n2);
- }
- };
- typedef SFST::hash_map<SFST::Node*, int, hashf, equalf> NL;
-
- private:
- NL number;
-
- public:
- int &operator[]( SFST::Node *node ) {
- NL::iterator it=number.find(node);
- if (it == number.end())
- return number.insert(NL::value_type(node, 0)).first->second;
- return it->second;
- };
- };
-
- float sfst_seconds_in_harmonize=0;
-
- float SfstTransducer::get_profile_seconds() {
- return sfst_seconds_in_harmonize;
- }
-
- void sfst_set_hopcroft(bool value) {
- SFST::Transducer::hopcroft_minimisation=value;
- }
-
/** Create an SfstInputStream that reads from stdin. */
SfstInputStream::SfstInputStream(void):
is_minimal(false)
@@ -151,73 +122,6 @@ namespace hfst { namespace implementations {
"SfstInputStream: symbol redefined"); }
}
- Transducer * SfstTransducer::expand_arcs(Transducer * t, StringSet &unknown)
- {
- Transducer &tc = t->copy();
- SfstTransducer::expand(&tc, unknown);
- return &tc;
- }
-
-
- std::pair<Transducer*, Transducer*> SfstTransducer::harmonize
- (Transducer *t1, Transducer *t2, bool unknown_symbols_in_use)
- {
-
- try {
-
- // 1. Calculate the set of unknown symbols for transducers t1 and t2.
-
- StringSet unknown_t1; // symbols known to another but not this
- StringSet unknown_t2; // and vice versa
-
- if (unknown_symbols_in_use) {
- StringSet t1_symbols = get_alphabet(t1);
- StringSet t2_symbols = get_alphabet(t2);
- hfst::symbols::collect_unknown_sets(t1_symbols, unknown_t1,
- t2_symbols, unknown_t2);
- }
-
-
- Transducer * new_t1 = &t1->copy(false, &t2->alphabet);
- new_t1->alphabet.insert_symbols(t2->alphabet);
- SFST::Alphabet::CharMap cm = t1->alphabet.get_char_map();
- for (SFST::Alphabet::CharMap::const_iterator it = cm.begin();
- it != cm.end(); it++) {
- new_t1->alphabet.add_symbol(it->second);
- }
-
- t2->alphabet.insert_symbols(new_t1->alphabet);
- delete t1;
- t1 = new_t1;
-
- // 3. Calculate the set of symbol pairs to which a non-identity "?:?"
- // transition is expanded for both transducers.
-
- Transducer *harmonized_t1;
- Transducer *harmonized_t2;
-
- if (unknown_symbols_in_use) {
- harmonized_t1 = expand_arcs(t1, unknown_t1);
- delete t1;
-
- harmonized_t2 = expand_arcs(t2, unknown_t2);
- delete t2;
- }
- else {
- harmonized_t1 = &t1->copy();
- harmonized_t2 = &t2->copy();
- }
-
-
- return std::pair<Transducer*, Transducer*>(harmonized_t1, harmonized_t2);
-
- }
- catch (const char *msg) {
- HFST_THROW_MESSAGE(HfstFatalException, std::string(msg));
- }
-
- }
-
void SfstInputStream::ignore(unsigned int n)
{
for (unsigned int i=0; i<n; i++)
@@ -242,25 +146,8 @@ namespace hfst { namespace implementations {
return true;
}
-
- unsigned int SfstTransducer::number_of_states(Transducer* t)
- {
- std::vector<SFST::Node*> indexing;
- std::pair<size_t, size_t> number_of_nodes_and_transitions =
- t->nodeindexing(&indexing);
- return number_of_nodes_and_transitions.first;
- }
-
- unsigned int SfstTransducer::number_of_arcs(Transducer* t)
- {
- std::vector<SFST::Node*> indexing;
- std::pair<size_t, size_t> number_of_nodes_and_transitions =
- t->nodeindexing(&indexing);
- return number_of_nodes_and_transitions.second;
- }
-
- Transducer * SfstInputStream::read_transducer()
+ Transducer * SfstInputStream::read_transducer()
{
if (is_eof())
{
@@ -288,8 +175,7 @@ namespace hfst { namespace implementations {
}
return NULL;
}
-
-
+
// ---------- SfstOutputStream functions ----------
SfstOutputStream::SfstOutputStream(void)
@@ -346,52 +232,136 @@ namespace hfst { namespace implementations {
}
}
- void SfstTransducer::print_test(Transducer *t)
+#if HAVE_SFST
+
+ class HfstNode2Int {
+
+ struct hashf {
+ size_t operator()(const SFST::Node *node) const {
+ return (size_t)node;
+ }
+ };
+ struct equalf {
+ int operator()(const SFST::Node *n1, const SFST::Node *n2) const {
+ return (n1 == n2);
+ }
+ };
+ typedef SFST::hash_map<SFST::Node*, int, hashf, equalf> NL;
+
+ private:
+ NL number;
+
+ public:
+ int &operator[]( SFST::Node *node ) {
+ NL::iterator it=number.find(node);
+ if (it == number.end())
+ return number.insert(NL::value_type(node, 0)).first->second;
+ return it->second;
+ };
+ };
+
+ float sfst_seconds_in_harmonize=0;
+
+ float SfstTransducer::get_profile_seconds() {
+ return sfst_seconds_in_harmonize;
+ }
+
+ void sfst_set_hopcroft(bool value) {
+ SFST::Transducer::hopcroft_minimisation=value;
+ }
+
+
+ Transducer * SfstTransducer::expand_arcs(Transducer * t, StringSet &unknown)
{
- std::cerr << *t;
+ Transducer &tc = t->copy();
+ SfstTransducer::expand(&tc, unknown);
+ return &tc;
}
- unsigned int SfstTransducer::get_biggest_symbol_number(Transducer * t)
- {
- unsigned int biggest_number=0;
- SFST::Alphabet::CharMap cm = t->alphabet.get_char_map();
+
+ std::pair<Transducer*, Transducer*> SfstTransducer::harmonize
+ (Transducer *t1, Transducer *t2, bool unknown_symbols_in_use)
+ {
+
+ try {
+
+ // 1. Calculate the set of unknown symbols for transducers t1 and t2.
+
+ StringSet unknown_t1; // symbols known to another but not this
+ StringSet unknown_t2; // and vice versa
+
+ if (unknown_symbols_in_use) {
+ StringSet t1_symbols = get_alphabet(t1);
+ StringSet t2_symbols = get_alphabet(t2);
+ hfst::symbols::collect_unknown_sets(t1_symbols, unknown_t1,
+ t2_symbols, unknown_t2);
+ }
+
+
+ Transducer * new_t1 = &t1->copy(false, &t2->alphabet);
+ new_t1->alphabet.insert_symbols(t2->alphabet);
+ SFST::Alphabet::CharMap cm = t1->alphabet.get_char_map();
for (SFST::Alphabet::CharMap::const_iterator it = cm.begin();
- it != cm.end(); it++) {
- if (it->first > biggest_number)
- biggest_number = it->first;
+ it != cm.end(); it++) {
+ new_t1->alphabet.add_symbol(it->second);
}
- return biggest_number;
- }
+
+ t2->alphabet.insert_symbols(new_t1->alphabet);
+ delete t1;
+ t1 = new_t1;
- StringVector SfstTransducer::get_symbol_vector
- (Transducer * t)
- {
- unsigned int biggest_symbol_number = get_biggest_symbol_number(t);
- StringVector symbol_vector;
- symbol_vector.reserve(biggest_symbol_number+1);
- symbol_vector.resize(biggest_symbol_number+1,"");
+ // 3. Calculate the set of symbol pairs to which a non-identity "?:?"
+ // transition is expanded for both transducers.
+
+ Transducer *harmonized_t1;
+ Transducer *harmonized_t2;
+
+ if (unknown_symbols_in_use) {
+ harmonized_t1 = expand_arcs(t1, unknown_t1);
+ delete t1;
- StringSet alphabet = get_alphabet(t);
- for (StringSet::const_iterator it = alphabet.begin(); it != alphabet.end(); it++)
- {
- unsigned int symbol_number = get_symbol_number(t, it->c_str());
- symbol_vector.at(symbol_number) = *it;
- }
- return symbol_vector;
+ harmonized_t2 = expand_arcs(t2, unknown_t2);
+ delete t2;
+ }
+ else {
+ harmonized_t1 = &t1->copy();
+ harmonized_t2 = &t2->copy();
}
+
+
+ return std::pair<Transducer*, Transducer*>(harmonized_t1, harmonized_t2);
+
+ }
+ catch (const char *msg) {
+ HFST_THROW_MESSAGE(HfstFatalException, std::string(msg));
+ }
+
+ }
- std::map<std::string, unsigned int> SfstTransducer::get_symbol_map
- (Transducer * t)
+ unsigned int SfstTransducer::number_of_states(Transducer* t)
{
- StringSet alphabet = get_alphabet(t);
- std::map<std::string, unsigned int> symbol_map;
- for (StringSet::const_iterator it = alphabet.begin(); it != alphabet.end(); it++)
- {
- symbol_map[*it] = get_symbol_number(t, it->c_str());
- }
- return symbol_map;
+ std::vector<SFST::Node*> indexing;
+ std::pair<size_t, size_t> number_of_nodes_and_transitions =
+ t->nodeindexing(&indexing);
+ return number_of_nodes_and_transitions.first;
}
+ unsigned int SfstTransducer::number_of_arcs(Transducer* t)
+ {
+ std::vector<SFST::Node*> indexing;
+ std::pair<size_t, size_t> number_of_nodes_and_transitions =
+ t->nodeindexing(&indexing);
+ return number_of_nodes_and_transitions.second;
+ }
+
+
+
+
+ void SfstTransducer::print_test(Transducer *t)
+ {
+ std::cerr << *t;
+ }
+
void SfstTransducer::print_alphabet(Transducer *t) {
fprintf(stderr, "alphabet..\n");
SFST::Alphabet::CharMap cm = t->alphabet.get_char_map();
@@ -566,9 +536,6 @@ namespace hfst { namespace implementations {
n->set_final(1);
return t; }
- void SfstTransducer::delete_transducer(Transducer * t)
- { delete t; }
-
Transducer * SfstTransducer::copy(Transducer * t)
{ return &t->copy(); }
@@ -1150,33 +1117,6 @@ namespace hfst { namespace implementations {
alpha.insert( label[i] );
}
- StringSet SfstTransducer::get_alphabet(Transducer * t)
- {
- StringSet s;
- SFST::Alphabet::CharMap cm = t->alphabet.get_char_map();
- for ( SFST::Alphabet::CharMap::const_iterator it = cm.begin();
- it != cm.end(); it++ ) {
- if (strcmp(it->second, "<>") == 0)
- s.insert(internal_epsilon);
- else
- s.insert( std::string(it->second) );
- }
- return s;
- }
-
- unsigned int SfstTransducer::get_symbol_number
- (Transducer *t,
- const std::string &symbol)
- {
- if (symbol == "@_EPSILON_SYMBOL_@")
- return 0;
- int i = t->alphabet.symbol2code(symbol.c_str());
- if (i == EOF) {
- HFST_THROW(SymbolNotFoundException);
- }
- return (unsigned int)i;
- }
-
StringPairSet SfstTransducer::get_symbol_pairs(Transducer *t)
{
StringPairSet s;
@@ -1336,13 +1276,90 @@ namespace hfst { namespace implementations {
expand2(t, t->root_node(), new_symbols, visited_nodes);
}
+#endif // HAVE_SFST
+
+ // These functions are needed in transducer type conversions
+
+ void SfstTransducer::delete_transducer(Transducer * t)
+ { delete t; }
+
+ StringSet SfstTransducer::get_alphabet(Transducer * t)
+ {
+ StringSet s;
+ SFST::Alphabet::CharMap cm = t->alphabet.get_char_map();
+ for ( SFST::Alphabet::CharMap::const_iterator it = cm.begin();
+ it != cm.end(); it++ ) {
+ if (strcmp(it->second, "<>") == 0)
+ s.insert(internal_epsilon);
+ else
+ s.insert( std::string(it->second) );
+ }
+ return s;
+ }
+
+ unsigned int SfstTransducer::get_symbol_number
+ (Transducer *t,
+ const std::string &symbol)
+ {
+ if (symbol == "@_EPSILON_SYMBOL_@")
+ return 0;
+ int i = t->alphabet.symbol2code(symbol.c_str());
+ if (i == EOF) {
+ HFST_THROW(SymbolNotFoundException);
+ }
+ return (unsigned int)i;
+ }
+
+ unsigned int SfstTransducer::get_biggest_symbol_number(Transducer * t)
+ {
+ unsigned int biggest_number=0;
+ SFST::Alphabet::CharMap cm = t->alphabet.get_char_map();
+ for (SFST::Alphabet::CharMap::const_iterator it = cm.begin();
+ it != cm.end(); it++) {
+ if (it->first > biggest_number)
+ biggest_number = it->first;
+ }
+ return biggest_number;
+ }
+
+ StringVector SfstTransducer::get_symbol_vector
+ (Transducer * t)
+ {
+ unsigned int biggest_symbol_number = get_biggest_symbol_number(t);
+ StringVector symbol_vector;
+ symbol_vector.reserve(biggest_symbol_number+1);
+ symbol_vector.resize(biggest_symbol_number+1,"");
+
+ StringSet alphabet = get_alphabet(t);
+ for (StringSet::const_iterator it = alphabet.begin(); it != alphabet.end(); it++)
+ {
+ unsigned int symbol_number = get_symbol_number(t, it->c_str());
+ symbol_vector.at(symbol_number) = *it;
+ }
+ return symbol_vector;
+ }
+
+ std::map<std::string, unsigned int> SfstTransducer::get_symbol_map
+ (Transducer * t)
+ {
+ StringSet alphabet = get_alphabet(t);
+ std::map<std::string, unsigned int> symbol_map;
+ for (StringSet::const_iterator it = alphabet.begin(); it != alphabet.end(); it++)
+ {
+ symbol_map[*it] = get_symbol_number(t, it->c_str());
+ }
+ return symbol_map;
+ }
} }
+#endif // HAVE_SFST || HAVE_LEAN_SFST
+
#else // MAIN_TEST was defined
-using namespace hfst::implementations;
+
#include <iostream>
+#if HAVE_SFST
bool does_sfst_alphabet_contain(SFST::Transducer *t, const char *str)
{
SFST::Alphabet::CharMap cm = t->alphabet.get_char_map();
@@ -1353,11 +1370,14 @@ bool does_sfst_alphabet_contain(SFST::Transducer *t, const char *str)
}
return false;
}
+#endif
int main(int argc, char * argv[])
{
+#if HAVE_SFST
std::cout << "Unit tests for " __FILE__ ":";
+ using namespace hfst::implementations;
// Test alphabet pruning
SFST::Transducer * t = SfstTransducer::define_transducer("a", "b");
@@ -1384,6 +1404,10 @@ int main(int argc, char * argv[])
std::cout << std::endl << "ok" << std::endl;
return EXIT_SUCCESS;
+#else // HAVE_SFST
+ std::cout << "Skipping unit tests for " << __FILE__ << ", SfstTransducer has not been enabled" << std::endl;
+ return 77;
+#endif // HAVE_SFST
}
#endif // MAIN_TEST
diff --git a/libhfst/src/implementations/SfstTransducer.h b/libhfst/src/implementations/SfstTransducer.h
index ae4f67d..eab89bc 100644
--- a/libhfst/src/implementations/SfstTransducer.h
+++ b/libhfst/src/implementations/SfstTransducer.h
@@ -14,8 +14,6 @@
#include "HfstFlagDiacritics.h"
#include "HfstSymbolDefs.h"
#include "HfstExtractStrings.h"
-//#include "back-ends/sfst/interface.h"
-//#include "back-ends/sfst/fst.h"
namespace SFST { class Transducer; typedef short unsigned int Character; class Alphabet; class Label; }
@@ -31,14 +29,10 @@ namespace SFST { class Transducer; typedef short unsigned int Character; class A
namespace hfst {
namespace implementations
{
- //using namespace SFST;
typedef SFST::Transducer Transducer;
- ;
using std::ostream;
using std::ostringstream;
- void sfst_set_hopcroft(bool);
-
class SfstInputStream
{
private:
@@ -77,7 +71,6 @@ namespace implementations
private:
std::string filename;
FILE *ofile;
- //void write_3_0_library_header(FILE *file, bool is_minimal);
public:
SfstOutputStream(void);
SfstOutputStream(const std::string &filename);
@@ -86,38 +79,14 @@ namespace implementations
void append_implementation_specific_header_data
(std::vector<char> &header, SFST::Transducer *t);
void write_transducer(SFST::Transducer * transducer);
- };
+ };
+
+ void sfst_set_hopcroft(bool);
+
- /*
- class HfstNode2Int {
-
- struct hashf {
- size_t operator()(const SFST::Node *node) const {
- return (size_t)node;
- }
- };
- struct equalf {
- int operator()(const SFST::Node *n1, const SFST::Node *n2) const {
- return (n1 == n2);
- }
- };
- typedef SFST::hash_map<SFST::Node*, int, hashf, equalf> NL;
-
- private:
- NL number;
-
- public:
- int &operator[]( SFST::Node *node ) {
- NL::iterator it=number.find(node);
- if (it == number.end())
- return number.insert(NL::value_type(node, 0)).first->second;
- return it->second;
- };
- };*/
-
-
class SfstTransducer
{
+#if HAVE_SFST
public:
static SFST::Transducer * create_empty_transducer(void);
static SFST::Transducer * create_epsilon_transducer(void);
@@ -126,8 +95,6 @@ namespace implementations
static SFST::Transducer * define_transducer
(unsigned int inumber, unsigned int onumber);
- static void delete_transducer(SFST::Transducer * t);
-
static SFST::Transducer * define_transducer(const std::string &symbol);
static SFST::Transducer * define_transducer
(const std::string &isymbol, const std::string &osymbol);
@@ -188,12 +155,6 @@ namespace implementations
static void print_test(SFST::Transducer *t);
static void print_alphabet(SFST::Transducer *t);
- static unsigned int get_biggest_symbol_number(SFST::Transducer * t);
-
- static StringVector get_symbol_vector(SFST::Transducer * t);
-
- static std::map<std::string, unsigned int> get_symbol_map(SFST::Transducer * t);
-
static SFST::Transducer * disjunct(SFST::Transducer * t, const StringPairVector &spv);
static StringPairSet get_symbol_pairs(SFST::Transducer *t);
@@ -202,12 +163,10 @@ namespace implementations
static unsigned int number_of_states(SFST::Transducer *t);
static unsigned int number_of_arcs(SFST::Transducer *t);
- static StringSet get_alphabet(SFST::Transducer *t);
+
static void insert_to_alphabet(SFST::Transducer *t, const std::string &symbol);
static void remove_from_alphabet
(SFST::Transducer *t, const std::string &symbol);
- static unsigned int get_symbol_number(SFST::Transducer *t,
- const std::string &symbol);
protected:
static void initialize_alphabet(SFST::Transducer *t);
@@ -220,6 +179,17 @@ namespace implementations
hfst::StringSet &new_symbols, std::set<SFST::Node*> &visited_nodes );
static void expand(SFST::Transducer *t, hfst::StringSet &new_symbols);
+#endif
+ // needed in transducer format conversions
+ public:
+ static void delete_transducer(SFST::Transducer * t);
+ static unsigned int get_symbol_number(SFST::Transducer *t,
+ const std::string &symbol);
+ static StringSet get_alphabet(SFST::Transducer *t);
+ static unsigned int get_biggest_symbol_number(SFST::Transducer * t);
+ static StringVector get_symbol_vector(SFST::Transducer * t);
+ static std::map<std::string, unsigned int> get_symbol_map(SFST::Transducer * t);
};
+
} }
#endif
diff --git a/libhfst/src/implementations/TropicalWeightTransducer.cc b/libhfst/src/implementations/TropicalWeightTransducer.cc
index 1314db1..6c9f759 100644
--- a/libhfst/src/implementations/TropicalWeightTransducer.cc
+++ b/libhfst/src/implementations/TropicalWeightTransducer.cc
@@ -19,6 +19,9 @@
#include "back-ends/openfst/src/include/fst/fstlib.h"
#endif // _MSC_VER
+#ifdef PROFILE_OPENFST
+#include <ctime>
+#endif
#ifndef MAIN_TEST
@@ -29,6 +32,23 @@ namespace hfst {
namespace implementations {
+ // add this to the beginning of code block to be profiled
+ //#ifdef PROFILE_OPENFST
+ //clock_t startclock = clock();
+ //#endif
+
+ // and this to the end of the block
+ //#ifdef PROFILE_OPENFST
+ //clock_t endclock = clock();
+ //tropical_seconds = tropical_seconds +
+ // ( (float)(endclock - startclock) / CLOCKS_PER_SEC);
+ //#endif
+
+ float tropical_seconds=0;
+ float TropicalWeightTransducer::get_profile_seconds() {
+ return tropical_seconds;
+ }
+
std::ostream * TropicalWeightTransducer::warning_stream = NULL;
std::ostream * TropicalWeightTransducer::get_warning_stream()
@@ -113,6 +133,32 @@ namespace hfst {
return retval;
}
+ bool TropicalWeightTransducer::has_weights(const StdVectorFst * t)
+ {
+ for (fst::StateIterator<StdVectorFst> siter(*t);
+ ! siter.Done(); siter.Next())
+ {
+ StateId s = siter.Value();
+ for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
+ !aiter.Done(); aiter.Next())
+ {
+ const StdArc &arc = aiter.Value();
+ if (arc.weight.Value() != 0)
+ {
+ return true;
+ }
+ }
+ if (t->Final(s) != TropicalWeight::Zero())
+ {
+ if (t->Final(s).Value() != 0)
+ {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
// This function can be moved to its own file if TropicalWeightTransducer.o
// yields a 'File too big' error.
StdVectorFst * TropicalWeightTransducer::minimize(StdVectorFst * t)
@@ -147,11 +193,6 @@ namespace hfst {
void print_att_number(StdVectorFst *t, FILE * ofile);
- float tropical_seconds_in_harmonize=0;
- float TropicalWeightTransducer::get_profile_seconds() {
- return tropical_seconds_in_harmonize;
- }
-
bool openfst_tropical_use_hopcroft=false;
void openfst_tropical_set_hopcroft(bool value) {
openfst_tropical_use_hopcroft=value;
@@ -1047,10 +1088,6 @@ namespace hfst {
{
bool DEBUG=false;
-#ifdef PROFILE_MINIMIZATION
- clock_t startclock = clock();
-#endif
-
// 1. Calculate the set of unknown symbols for transducers t1 and t2.
StringSet unknown_t1; // symbols known to another but not this
@@ -1120,12 +1157,6 @@ namespace hfst {
harmonized_t2->SetInputSymbols(t2->InputSymbols());
}
-#ifdef PROFILE_MINIMIZATION
- clock_t endclock = clock();
- tropical_seconds_in_harmonize = tropical_seconds_in_harmonize +
- ( (float)(endclock - startclock) / CLOCKS_PER_SEC);
-#endif
-
return std::pair<StdVectorFst*, StdVectorFst*>
(harmonized_t1, harmonized_t2);
diff --git a/libhfst/src/implementations/TropicalWeightTransducer.h b/libhfst/src/implementations/TropicalWeightTransducer.h
index 2b346af..48cdf20 100644
--- a/libhfst/src/implementations/TropicalWeightTransducer.h
+++ b/libhfst/src/implementations/TropicalWeightTransducer.h
@@ -18,12 +18,6 @@
#include "../../../config.h"
#endif
-//#ifdef _MSC_VER
-//#include "back-ends/openfstwin/src/include/fst/fstlib.h"
-//#else
-//#include "back-ends/openfst/src/include/fst/fstlib.h"
-//#endif // _MSC_VER
-
#include "HfstExtractStrings.h"
#include <cstdio>
#include <string>
@@ -31,8 +25,6 @@
#include <iosfwd>
#include <fstream>
-//#include "HfstAlphabet.h"
-
#ifdef OPENFST_VERSION_1_5_4
#include "back-ends/openfst/src/include/fst/fst-decl.h"
#else
@@ -75,7 +67,6 @@ namespace implementations
using namespace fst;
;
typedef unsigned int StateId;
- //typedef StdArc::StateId StateId;
typedef std::vector<StdArc> StdArcVector;
struct StdArcLessThan {
@@ -122,7 +113,6 @@ namespace implementations
std::ofstream o_stream;
std::ostream &output_stream;
bool hfst_format;
- //void write_3_0_library_header(std::ostream &out);
public:
TropicalWeightOutputStream(bool hfst_format=true);
TropicalWeightOutputStream
@@ -132,52 +122,6 @@ namespace implementations
void write_transducer(StdVectorFst * transducer);
};
- /*class TropicalWeightTransitionIterator;
-
- typedef StateId TropicalWeightState;
-
- class TropicalWeightStateIterator
- {
- protected:
- fst::StateIterator<fst::StdVectorFst> * iterator;
- public:
- TropicalWeightStateIterator(StdVectorFst * t);
- ~TropicalWeightStateIterator(void);
- void next(void);
- bool done(void);
- TropicalWeightState value(void);
- };*/
-
-
- /* class TropicalWeightTransition
- {
- protected:
- StdArc arc;
- StdVectorFst * t;
- public:
- TropicalWeightTransition(const StdArc &arc, StdVectorFst *t);
- ~TropicalWeightTransition(void);
- std::string get_input_symbol(void) const;
- std::string get_output_symbol(void) const;
- TropicalWeightState get_target_state(void) const;
- TropicalWeight get_weight(void) const;
- };
-
-
- class TropicalWeightTransitionIterator
- {
- protected:
- fst::ArcIterator<fst::StdVectorFst> * arc_iterator;
- fst::StdVectorFst * t;
- public:
- TropicalWeightTransitionIterator(StdVectorFst * t, StateId state);
- ~TropicalWeightTransitionIterator(void);
- void next(void);
- bool done(void);
- TropicalWeightTransition value(void);
- };*/
-
-
class TropicalWeightTransducer
{
public:
@@ -258,6 +202,7 @@ namespace implementations
(StdVectorFst * t,float (*func)(float f));
static StdVectorFst * push_weights
(StdVectorFst * t, bool to_initial_state);
+ static bool has_weights(const StdVectorFst * t);
static std::pair<StdVectorFst*, StdVectorFst*> harmonize
(StdVectorFst *t1, StdVectorFst *t2, bool unknown_symbols_in_use=true);
@@ -337,7 +282,7 @@ namespace implementations
Grammar * grammar);
#endif
- float get_profile_seconds();
+ static float get_profile_seconds();
static unsigned int number_of_states(const StdVectorFst * t);
static unsigned int number_of_arcs(const StdVectorFst * t);
diff --git a/libhfst/src/parsers/LexcCompiler.cc b/libhfst/src/parsers/LexcCompiler.cc
index 2d714b8..9fe3358 100644
--- a/libhfst/src/parsers/LexcCompiler.cc
+++ b/libhfst/src/parsers/LexcCompiler.cc
@@ -33,13 +33,11 @@ using std::set_difference;
#include "HfstTransducer.h"
#include "XreCompiler.h"
#include "lexc-utils.h"
-
#ifdef YACC_USE_PARSER_H_EXTENSION
#include "lexc-parser.h"
#else
#include "lexc-parser.hh"
#endif
-
#include "xre_utils.h"
#include "HfstSymbolDefs.h"
diff --git a/libhfst/src/parsers/Makefile.am b/libhfst/src/parsers/Makefile.am
index b6418d1..451d367 100644
--- a/libhfst/src/parsers/Makefile.am
+++ b/libhfst/src/parsers/Makefile.am
@@ -98,11 +98,9 @@ TWOLC_BUILT=htwolcpre1-parser.cc htwolcpre1-lexer.cc htwolcpre2-parser.cc htwolc
BUILT_SOURCES=$(XRE_BUILT) $(PMATCH_BUILT) $(LEXC_BUILT) $(XFST_BUILT) $(SFST_BUILT) $(TWOLC_BUILT)
-if YACC_USE_PARSER_H_EXTENSION
- HEADER=h
-else
- HEADER=hh
-endif
+# If automake is < 1.12, *.h extension is used instead and build will fail,
+# unless pre-yacc-generated *.cc files are used and *.yy source files are not modified.
+HEADER=hh
CLEANFILES=$(XRE_BUILT) $(PMATCH_BUILT) $(LEXC_BUILT) $(XFST_BUILT) $(SFST_BUILT) $(TWOLC_BUILT) \
xre_parse.$(HEADER) pmatch_parse.$(HEADER) lexc-parser.$(HEADER) xfst-parser.$(HEADER) sfst-compiler.$(HEADER) \
@@ -215,7 +213,7 @@ TwolCGrammar_LDADD=../libhfst.la
TESTS=$(LIBHFST_PARSER_TSTS) $(HFST_TWOLC_TESTS)
EXTRA_DIST=xre_parse.$(HEADER) pmatch_parse.$(HEADER) lexc-parser.$(HEADER) xfst-parser.$(HEADER) sfst-compiler.$(HEADER) \
- htwolcpre1-parser.$(HEADER) htwolcpre2-parser.$(HEADER) htwolcpre3-parser.$(HEADER) $(BUILT_SOURCES)
+ htwolcpre1-parser.$(HEADER) htwolcpre2-parser.$(HEADER) htwolcpre3-parser.$(HEADER) $(BUILT_SOURCES) common_globals.h
# some systems might require all these
diff --git a/libhfst/src/parsers/SfstAlphabet.h b/libhfst/src/parsers/SfstAlphabet.h
index 5b60a8e..0434070 100644
--- a/libhfst/src/parsers/SfstAlphabet.h
+++ b/libhfst/src/parsers/SfstAlphabet.h
@@ -5,17 +5,7 @@
# include <config.h>
#endif
-#if HAVE_BACKWARD_HASH_MAP
-# include <backward/hash_map>
-#elif HAVE_EXT_HASH_MAP
-# include <ext/hash_map>
-#elif HAVE_HASH_MAP
-# include <hash_map>
-#else
-# warning "unknown hash_map"
-# include <hash_map>
-#endif
-
+#include <unordered_map>
#include <set>
#include <vector>
#include <string.h>
@@ -32,27 +22,19 @@ namespace hfst {
public:
typedef std::pair<unsigned int,unsigned int> NumberPair;
- // hash table used to map the codes back to the symbols
-#ifdef __GNUC__
- typedef __gnu_cxx::hash_map<unsigned int, char*> CharMap;
-#else
- typedef std::hash_map<unsigned int, char*> CharMap;
-#endif
+ // used to map the codes back to the symbols
+ typedef std::unordered_map<unsigned int, char*> CharMap;
private:
- // string comparison operators needed to stored strings in a hash table
+ // string comparison operators needed ???
struct eqstr {
bool operator()(const char* s1, const char* s2) const {
return strcmp(s1, s2) == 0;
}
};
- // hash table used to map the symbols to their codes
-#ifdef __GNUC__
- typedef __gnu_cxx::hash_map<const char*, unsigned int, __gnu_cxx::hash<const char*>,eqstr> SymbolMap;
-#else
- typedef std::hash_map<const char*, unsigned int, std::hash<const char*>,eqstr> SymbolMap;
-#endif
+ // used to map the symbols to their codes
+ typedef std::unordered_map<const char*, unsigned int> SymbolMap;
// set of symbol pairs
typedef std::set<NumberPair> NumberPairSet;
diff --git a/libhfst/src/parsers/SfstCompiler.h b/libhfst/src/parsers/SfstCompiler.h
index c195b55..412e5f5 100644
--- a/libhfst/src/parsers/SfstCompiler.h
+++ b/libhfst/src/parsers/SfstCompiler.h
@@ -17,6 +17,7 @@
#include "SfstAlphabet.h"
#include <iosfwd>
#include <fstream>
+#include <unordered_map>
namespace hfst
{
@@ -60,8 +61,8 @@ namespace hfst
};
typedef std::set<char*, ltstr> RVarSet;
- typedef __gnu_cxx::hash_map<char*, HfstTransducer*, __gnu_cxx::hash<const char*>, eqstr> VarMap;
- typedef __gnu_cxx::hash_map<char*, Range*, __gnu_cxx::hash<const char*>, eqstr> SVarMap;
+ typedef std::unordered_map<char*,HfstTransducer*> VarMap;
+ typedef std::unordered_map<char*,Range*> SVarMap;
private:
VarMap VM;
diff --git a/libhfst/src/parsers/htwolcpre1-lexer.ll b/libhfst/src/parsers/htwolcpre1-lexer.ll
index 5444fd0..bc50162 100644
--- a/libhfst/src/parsers/htwolcpre1-lexer.ll
+++ b/libhfst/src/parsers/htwolcpre1-lexer.ll
@@ -19,11 +19,7 @@
#endif
// The parser. Supplies flex with its symbol tables.
-#ifdef YACC_USE_PARSER_H_EXTENSION
- #include "htwolcpre1-parser.h"
-#else
#include "htwolcpre1-parser.hh"
-#endif
// Functions for manipulating strings.
#include "string_src/string_manipulation.h"
diff --git a/libhfst/src/parsers/htwolcpre2-lexer.ll b/libhfst/src/parsers/htwolcpre2-lexer.ll
index d242be3..7a88af4 100644
--- a/libhfst/src/parsers/htwolcpre2-lexer.ll
+++ b/libhfst/src/parsers/htwolcpre2-lexer.ll
@@ -19,12 +19,9 @@
#endif
- // The parser. Supplies flex with its symbol tables.
-#ifdef YACC_USE_PARSER_H_EXTENSION
- #include "htwolcpre2-parser.h"
-#else
- #include "htwolcpre2-parser.hh"
-#endif
+// The parser. Supplies flex with its symbol tables.
+#include "htwolcpre2-parser.hh"
+
#include "../HfstExceptionDefs.h"
extern int htwolcpre2error(const char*);
diff --git a/libhfst/src/parsers/htwolcpre3-lexer.ll b/libhfst/src/parsers/htwolcpre3-lexer.ll
index fd8ad41..4269955 100644
--- a/libhfst/src/parsers/htwolcpre3-lexer.ll
+++ b/libhfst/src/parsers/htwolcpre3-lexer.ll
@@ -45,11 +45,7 @@ result = 1; \
#include "rule_src/TwolCGrammar.h"
-#ifdef YACC_USE_PARSER_H_EXTENSION
- #include "htwolcpre3-parser.h"
-#else
- #include "htwolcpre3-parser.hh"
-#endif
+#include "htwolcpre3-parser.hh"
/*
#include "io_src/InputReader.h"
diff --git a/libhfst/src/parsers/lexc-lexer.ll b/libhfst/src/parsers/lexc-lexer.ll
index f69f764..fc4de0c 100644
--- a/libhfst/src/parsers/lexc-lexer.ll
+++ b/libhfst/src/parsers/lexc-lexer.ll
@@ -20,12 +20,7 @@
# include <config.h>
#endif
-#ifdef YACC_USE_PARSER_H_EXTENSION
- #include "lexc-parser.h"
-#else
- #include "lexc-parser.hh"
-#endif
-
+#include "lexc-parser.hh"
#include "lexc-utils.h"
#include "HfstDataTypes.h"
diff --git a/libhfst/src/parsers/pmatch_lex.ll b/libhfst/src/parsers/pmatch_lex.ll
index dfd864f..7444a1d 100644
--- a/libhfst/src/parsers/pmatch_lex.ll
+++ b/libhfst/src/parsers/pmatch_lex.ll
@@ -16,12 +16,7 @@
#include "HfstInputStream.h"
#include "HfstXeroxRules.h"
#include "pmatch_utils.h"
-
-#ifdef YACC_USE_PARSER_H_EXTENSION
- #include "pmatch_parse.h"
-#else
- #include "pmatch_parse.hh"
-#endif
+#include "pmatch_parse.hh"
#undef YY_INPUT
#define YY_INPUT(buf, retval, maxlen) (retval = hfst::pmatch::getinput(buf, maxlen))
diff --git a/libhfst/src/parsers/sfst-scanner.ll b/libhfst/src/parsers/sfst-scanner.ll
index 2badc4b..abc1d84 100644
--- a/libhfst/src/parsers/sfst-scanner.ll
+++ b/libhfst/src/parsers/sfst-scanner.ll
@@ -16,13 +16,7 @@
#include <string.h>
#include "SfstCompiler.h"
-
-#ifdef YACC_USE_PARSER_H_EXTENSION
- #include "sfst-compiler.h"
-#else
- #include "sfst-compiler.hh"
-#endif
-
+#include "sfst-compiler.hh"
#include "SfstBasic.h"
#include "SfstUtf8.h"
#include "HfstTransducer.h"
diff --git a/libhfst/src/parsers/xfst-lexer.ll b/libhfst/src/parsers/xfst-lexer.ll
index 28e92de..8cc93e1 100644
--- a/libhfst/src/parsers/xfst-lexer.ll
+++ b/libhfst/src/parsers/xfst-lexer.ll
@@ -24,12 +24,7 @@ namespace hfst {
class HfstTransducer;
}
-#ifdef YACC_USE_PARSER_H_EXTENSION
- #include "xfst-parser.h"
-#else
- #include "xfst-parser.hh"
-#endif
-
+#include "xfst-parser.hh"
#include "xfst-utils.h"
#include "XfstCompiler.h"
diff --git a/libhfst/src/parsers/xre_lex.ll b/libhfst/src/parsers/xre_lex.ll
index e5be7f1..ad6d051 100644
--- a/libhfst/src/parsers/xre_lex.ll
+++ b/libhfst/src/parsers/xre_lex.ll
@@ -16,12 +16,7 @@
#include "HfstTransducer.h"
#include "HfstInputStream.h"
#include "HfstXeroxRules.h"
-
-#ifdef YACC_USE_PARSER_H_EXTENSION
- #include "xre_parse.h"
-#else
- #include "xre_parse.hh"
-#endif
+#include "xre_parse.hh"
#include "xre_utils.h"
#undef YY_INPUT
diff --git a/python/doc/Doxyfile b/python/doc/Doxyfile
index 6dd80cc..a1034fe 100644
--- a/python/doc/Doxyfile
+++ b/python/doc/Doxyfile
@@ -31,7 +31,7 @@ PROJECT_NAME = "HFST - Helsinki Finite-State Transducer Technology - P
# This could be handy for archiving the generated documentation or
# if some version control system is used.
-PROJECT_NUMBER = "version 3.12.1"
+PROJECT_NUMBER = "version 3.12.2"
# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute)
# base path where the generated documentation will be put.
diff --git a/python/doc/README b/python/doc/README
index 1c7aa6f..741ad45 100644
--- a/python/doc/README
+++ b/python/doc/README
@@ -3,7 +3,7 @@ This directory contains documentation for hfst package in Doxygen format.
Run 'doxygen Doxyfile' to generate the documentation (it will be written in
directory html which is created when doxygen is run) and copy everything under
directory 'html' to HFST github repository (hfst.github.io/python/VERSION/,
-where VERSION is the version number of HFST, e.g. '3.12.0', or 'next_release'
+where VERSION is the version number of HFST, e.g. '3.12.2', or 'next_release'
for upcoming releases). The documentation will be visible on page
<https://hfst.github.io/python/VERSION/index.html>. Also remember to update
the redirection link in hfst.github.io/python/index.html to point to the
diff --git a/python/pypi/MANIFEST.in b/python/pypi/MANIFEST.in
index bc4613a..1860c4e 100644
--- a/python/pypi/MANIFEST.in
+++ b/python/pypi/MANIFEST.in
@@ -7,6 +7,7 @@ include hfst_prolog_extensions.cc
include hfst_regex_extensions.cc
include hfst_rules_extensions.cc
include hfst_xfst_extensions.cc
+include hfst_sfst_extensions.cc
include docstrings.i
include libhfst/src/*.h
include libhfst/src/implementations/*.h
diff --git a/python/pypi/setup.py b/python/pypi/setup.py
index 16745f6..1abf673 100644
--- a/python/pypi/setup.py
+++ b/python/pypi/setup.py
@@ -220,7 +220,7 @@ libhfst_module = Extension('_libhfst',
)
setup(name = 'hfst',
- version = '3.12.1.0_beta',
+ version = '3.12.2.0_beta',
author = 'HFST team',
author_email = 'hfst-bugs at helsinki.fi',
url = 'http://hfst.github.io/',
diff --git a/python/setup.py b/python/setup.py
index 3c2b627..36a9d89 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -29,13 +29,14 @@ libhfst_module = Extension('_libhfst',
include_dirs = [absolute_libhfst_src_path],
library_dirs = [absolute_libhfst_src_path + "/.libs"],
libraries = ["hfst"],
+ extra_compile_args = ["-std=c++0x"],
extra_link_args = extra_link_arguments
)
# When making the windows package, replace data_files with
# ["libhfst-NN.dll", "libgcc_s_seh-1.dll"] or
# ["libhfst-NN.dll", "libgcc_s_dw2-1.dll"] or
setup(name = 'libhfst_swig',
- version = '3.12.1_beta',
+ version = '3.12.2_beta',
author = 'HFST team',
author_email = 'hfst-bugs at helsinki.fi',
url = 'http://hfst.github.io/',
diff --git a/test/tools/hfst-check-help.sh b/test/tools/hfst-check-help.sh
index 19ba7b8..d59cf09 100755
--- a/test/tools/hfst-check-help.sh
+++ b/test/tools/hfst-check-help.sh
@@ -5,7 +5,7 @@ HFST_TOOLS="hfst-affix-guessify hfst-calculate hfst-compare hfst-compose \
hfst-compose-intersect hfst-concatenate hfst-conjunct hfst-determinize \
hfst-disjunct hfst-edit-metadata hfst-format hfst-fst2fst \
hfst-fst2strings hfst-fst2txt hfst-grep hfst-guess hfst-guessify \
-hfst-head hfst-info hfst-invert hfst-lexc-wrapper hfst-lexc \
+hfst-head hfst-info hfst-invert hfst-lexc \
hfst-lookup hfst-minimize hfst-multiply hfst-name \
hfst-optimized-lookup hfst-pair-test hfst-pmatch hfst-pmatch2fst \
hfst-project hfst-prune-alphabet hfst-push-weights hfst-regexp2fst \
@@ -14,14 +14,8 @@ hfst-shuffle hfst-split hfst-strings2fst hfst-substitute \
hfst-subtract hfst-summarize hfst-tail hfst-traverse \
hfst-txt2fst"
-TWOLC_TOOLS=
-if [ -e "$TOOLDIR/hfst-twolc/src/hfst-twolc-loc" ] ; then
- TWOLC_TOOLS="hfst-twolc/src/hfst-twolc-loc"
-else
- TWOLC_TOOLS="$TOOLDIR/hfst-twolc/src/hfst-twolc"
-fi
-
-TAGGER_TOOLS="hfst-tagger/src/hfst-tag /hfst-tagger/src/hfst-train-tagger-loc"
+TWOLC_TOOLS="$TOOLDIR/hfst-twolc/src/hfst-twolc"
+TAGGER_TOOLS="hfst-tagger/src/hfst-tag"
PROC_TOOLS="hfst-proc/hfst-apertium-proc"
XFST_TOOLS="parsers/hfst-xfst"
diff --git a/test/tools/hfst-check-version.sh b/test/tools/hfst-check-version.sh
index 04261ae..efd8541 100755
--- a/test/tools/hfst-check-version.sh
+++ b/test/tools/hfst-check-version.sh
@@ -5,7 +5,7 @@ HFST_TOOLS="hfst-affix-guessify hfst-calculate hfst-compare hfst-compose \
hfst-compose-intersect hfst-concatenate hfst-conjunct hfst-determinize \
hfst-disjunct hfst-edit-metadata hfst-format hfst-fst2fst \
hfst-fst2strings hfst-fst2txt hfst-grep hfst-guess hfst-guessify \
-hfst-head hfst-info hfst-invert hfst-lexc-wrapper hfst-lexc \
+hfst-head hfst-info hfst-invert hfst-lexc \
hfst-lookup hfst-minimize hfst-multiply hfst-name \
hfst-optimized-lookup hfst-pair-test hfst-pmatch hfst-pmatch2fst \
hfst-project hfst-prune-alphabet hfst-push-weights hfst-regexp2fst \
@@ -14,14 +14,8 @@ hfst-shuffle hfst-split hfst-strings2fst hfst-substitute \
hfst-subtract hfst-summarize hfst-tail hfst-traverse \
hfst-txt2fst"
-TWOLC_TOOLS=
-if [ -e "$TOOLDIR/hfst-twolc/src/hfst-twolc-loc" ] ; then
- TWOLC_TOOLS="hfst-twolc/src/hfst-twolc-loc"
-else
- TWOLC_TOOLS="$TOOLDIR/hfst-twolc/src/hfst-twolc"
-fi
-
-TAGGER_TOOLS="hfst-tagger/src/hfst-tag /hfst-tagger/src/hfst-train-tagger-loc"
+TWOLC_TOOLS="$TOOLDIR/hfst-twolc/src/hfst-twolc"
+TAGGER_TOOLS="hfst-tagger/src/hfst-tag"
PROC_TOOLS="hfst-proc/hfst-apertium-proc"
XFST_TOOLS="parsers/hfst-xfst"
diff --git a/tools/src/Makefile.am b/tools/src/Makefile.am
index c159ea7..54e874a 100644
--- a/tools/src/Makefile.am
+++ b/tools/src/Makefile.am
@@ -151,9 +151,6 @@ endif
if WANT_STRINGS2FST
MAYBE_STRINGS2FST=hfst-strings2fst$(EXEEXT)
endif
-#if WANT_STRIP_HEADER
-#MAYBE_STRIP_HEADER=hfst-strip-header$(EXEEXT)
-#endif
if WANT_SUBTRACT
MAYBE_SUBTRACT=hfst-subtract$(EXEEXT)
endif
@@ -176,12 +173,6 @@ if WANT_SUBSTITUTE
MAYBE_SUBSTITUTE=hfst-substitute$(EXEEXT)
endif
-if YACC_USE_PARSER_H_EXTENSION
- HEADER=h
-else
- HEADER=hh
-endif
-
noinst_HEADERS= \
calculate_functions.h hfst-commandline.h hfst-program-options.h \
inc/check-params-binary.h inc/check-params-common.h \
@@ -192,12 +183,6 @@ noinst_HEADERS= \
hfst-file-to-mem.h \
hfst-tool-metadata.h hfst-optimized-lookup.h \
guessify_fst.h generate_model_forms.h
-# parsers/XreCompiler.h parsers/xre_utils.h
-
-# sort alphabetically
-# sort alphabetically
-# sort alphabetically
-# sort alphabetically
# sort in abc order
# finish with xyz
# see <http://en.wikipedia.org/wiki/Alphabets#Alphabetic_order> for details
@@ -225,10 +210,8 @@ bin_PROGRAMS= \
$(MAYBE_TRAVERSE) $(MAYBE_GUESSIFY) $(MAYBE_GUESS) \
$(MAYBE_INFO) $(MAYBE_EXPAND_EQUIVALENCES)
-#$(MAYBE_PREPROCESS_FOR_OPTIMIZED_LOOKUP)
+HFST_COMMON_SRC=hfst-program-options.cc hfst-commandline.cc hfst-tool-metadata.cc
-HFST_COMMON_SRC=hfst-program-options.cc hfst-commandline.cc \
- hfst-tool-metadata.cc
# sort alphabetically
hfst_affix_guessify_SOURCES=hfst-affix-guessify.cc $(HFST_COMMON_SRC)
hfst_calculate_SOURCES=sfst-main.cc $(HFST_COMMON_SRC)
@@ -247,21 +230,13 @@ hfst_fst2fst_SOURCES=hfst-fst2fst.cc $(HFST_COMMON_SRC)
hfst_fst2strings_SOURCES=hfst-fst2strings.cc $(HFST_COMMON_SRC)
hfst_fst2txt_SOURCES=hfst-fst2txt.cc $(HFST_COMMON_SRC)
hfst_grep_SOURCES=hfst-grep.cc $(HFST_COMMON_SRC)
-hfst_guess_SOURCES=hfst-guess.cc generate_model_forms.cc guessify_fst.cc \
-$(HFST_COMMON_SRC)
-hfst_guessify_SOURCES=hfst-guessify.cc guessify_fst.cc \
-$(HFST_COMMON_SRC)
+hfst_guess_SOURCES=hfst-guess.cc generate_model_forms.cc guessify_fst.cc $(HFST_COMMON_SRC)
+hfst_guessify_SOURCES=hfst-guessify.cc guessify_fst.cc $(HFST_COMMON_SRC)
hfst_head_SOURCES=hfst-head.cc $(HFST_COMMON_SRC)
hfst_info_SOURCES=hfst-info.cc $(HFST_COMMON_SRC)
hfst_invert_SOURCES=hfst-invert.cc $(HFST_COMMON_SRC)
hfst_lexc_wrapper_SOURCES=hfst-lexc-wrapper.cc lexc-readline-ui.cc $(HFST_COMMON_SRC)
hfst_lexc_SOURCES=hfst-lexc-compiler.cc $(HFST_COMMON_SRC)
-# parsers/LexcCompiler.cc parsers/lexc-parser.yy \
-# parsers/lexc-lexer.ll parsers/lexc-utils.cc \
-# parsers/XreCompiler.cc parsers/xre_parse.yy \
-# parsers/xre_lex.ll parsers/xre_utils.cc \
-# parsers/LexcCompiler.h parsers/lexc-utils.h
-#hfst_lexc2fst_BUILT=parsers/lexc-parser.h parsers/xre_parse.h
hfst_lookup_SOURCES=hfst-lookup.cc $(HFST_COMMON_SRC)
hfst_flookup_SOURCES=hfst-flookup.cc $(HFST_COMMON_SRC)
hfst_pair_test_SOURCES=hfst-pair-test.cc $(HFST_COMMON_SRC)
@@ -269,8 +244,6 @@ hfst_minimize_SOURCES=hfst-minimize.cc $(HFST_COMMON_SRC)
hfst_name_SOURCES=hfst-name.cc $(HFST_COMMON_SRC)
hfst_optimized_lookup_SOURCES=hfst-optimized-lookup.cc
hfst_pmatch_SOURCES=hfst-pmatch.cc $(HFST_COMMON_SRC)
-#hfst_preprocess_for_optimized_lookup_format_SOURCES=hfst-preprocess-for-optimized-lookup-format.cc
-# $(HFST_COMMON_SRC)
hfst_tokenize_SOURCES=hfst-tokenize.cc $(HFST_COMMON_SRC)
hfst_project_SOURCES=hfst-project.cc $(HFST_COMMON_SRC)
hfst_prune_alphabet_SOURCES=hfst-prune-alphabet.cc $(HFST_COMMON_SRC)
@@ -284,7 +257,6 @@ hfst_reweight_SOURCES=hfst-reweight.cc $(HFST_COMMON_SRC)
hfst_shuffle_SOURCES=hfst-shuffle.cc $(HFST_COMMON_SRC)
hfst_split_SOURCES=hfst-split.cc $(HFST_COMMON_SRC)
hfst_strings2fst_SOURCES=hfst-strings2fst.cc $(HFST_COMMON_SRC)
-#hfst_strip_header_SOURCES=hfst-strip-header.cc $(HFST_COMMON_SRC)
hfst_substitute_SOURCES=hfst-substitute.cc $(HFST_COMMON_SRC)
hfst_subtract_SOURCES=hfst-subtract.cc $(HFST_COMMON_SRC)
hfst_summarize_SOURCES=hfst-summarize.cc $(HFST_COMMON_SRC)
@@ -307,7 +279,6 @@ install-exec-hook:
$(LN_S) -f hfst-disjunct$(EXEEXT) hfst-union$(EXEEXT) && \
$(LN_S) -f hfst-tokenize$(EXEEXT) hfst-proc2$(EXEEXT) && \
$(LN_S) -f hfst-tokenize$(EXEEXT) hfst-tokenise$(EXEEXT)
-# $(LN_S) -f hfst-preprocess-for-optimized-lookup-format$(EXEEXT) hfst-preprocess-for-optimised-lookup-format$(EXEEXT) &&
uninstall-local:
cd $(DESTDIR)$(bindir) && \
@@ -323,14 +294,4 @@ uninstall-local:
rm hfst-tokenise && \
rm hfst-proc2
-BUILT_SOURCES=$(hfst_lexc_BUILT)
-
-CLEANFILES=$(hfst_lexc_BUILT)
-
-
hfst_calculate_CXXFLAGS= -Wall -Wno-error -Wno-sign-compare -Wno-deprecated
-
-## YACC and FLEX with autotools argh
-AM_YFLAGS=-d
-AM_LFLAGS=--outfile=$(LEX_OUTPUT_ROOT).c
-
diff --git a/tools/src/hfst-info.cc b/tools/src/hfst-info.cc
index bf1b088..42b3e6c 100644
--- a/tools/src/hfst-info.cc
+++ b/tools/src/hfst-info.cc
@@ -215,28 +215,32 @@ int main (int argc, char * argv[])
if ((*f == "sfst") || (*f == "SFST") || (*f == "HAVE_SFST"))
{
verbose_printf("Requiring SFST support from library");
-#ifndef HAVE_SFST
+#if !HAVE_SFST
+#if !HAVE_LEAN_SFST
error(EXIT_FAILURE, 0, "Required SFST support not present");
+#else
+ error(EXIT_FAILURE, 0, "Required SFST support present only in limited form");
+#endif
#endif
}
else if ((*f == "foma") || (*f == "FOMA") || (*f == "HAVE_FOMA"))
{
verbose_printf("Requiring foma support from library");
-#ifndef HAVE_FOMA
+#if HAVE_FOMA
error(EXIT_FAILURE, 0, "Required foma support not present");
#endif
}
else if ((*f == "xfsm") || (*f == "XFSM") || (*f == "HAVE_XFSM"))
{
verbose_printf("Requiring xfsm support from library");
-#ifndef HAVE_XFSM
+#if HAVE_XFSM
error(EXIT_FAILURE, 0, "Required xfsm support not present");
#endif
}
else if ((*f == "openfst") || (*f == "OPENFST") || (*f == "HAVE_OPENFST"))
{
verbose_printf("Requiring OpenFst support from library");
-#ifndef HAVE_OPENFST
+#if HAVE_OPENFST
error(EXIT_FAILURE, 0, "Required OpenFst support not present");
#endif
}
@@ -245,7 +249,7 @@ int main (int argc, char * argv[])
verbose_printf("Requiring Unicode parsed by Glib");
#ifndef USE_GLIB_UNICODE
error(EXIT_FAILURE, 0,
- "Required GLIB-based Unicode handling not presesnt");
+ "Required GLIB-based Unicode handling not present");
#endif
}
else
@@ -269,6 +273,10 @@ int main (int argc, char * argv[])
#endif
#if HAVE_SFST
verbose_printf("SFST supported\n");
+#else
+#if HAVE_LEAN_SFST
+ verbose_printf("SFST limitedly supported\n");
+#endif
#endif
#if HAVE_FOMA
verbose_printf("foma supported\n");
diff --git a/tools/src/hfst-twolc/src/hfst-twolc.cc b/tools/src/hfst-twolc/src/hfst-twolc.cc
index e3e7691..630ae62 100644
--- a/tools/src/hfst-twolc/src/hfst-twolc.cc
+++ b/tools/src/hfst-twolc/src/hfst-twolc.cc
@@ -93,17 +93,34 @@ int main(int argc, char * argv[])
(void)command_line.set_output_file();
std::ostringstream oss1;
hfst::twolcpre1::set_output(oss1);
- if (hfst::twolcpre1::parse() != 0)
+ try
+ {
+ if (hfst::twolcpre1::parse() != 0)
+ {
+ exit(1);
+ }
+ }
+ catch(const HfstException & e)
{
+ std::cerr << e.what() << std::endl;
exit(1);
}
std::istringstream iss1(oss1.str());
hfst::twolcpre2::set_input(iss1);
- if (hfst::twolcpre2::parse() != 0)
+ try
{
+ if (hfst::twolcpre2::parse() != 0)
+ {
+ exit(1);
+ }
+ }
+ catch(const HfstException & e)
+ {
+ std::cerr << e.what() << std::endl;
exit(1);
}
+
hfst::twolcpre2::complete_alphabet();
std::ostringstream oss2;
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/hfst.git
More information about the debian-science-commits
mailing list