[hfst-ospell] 01/02: Imported Upstream version 0.4.3~r338
Tino Didriksen
tinodidriksen-guest at moszumanska.debian.org
Tue Jan 10 20:50:03 UTC 2017
This is an automated email from the git hooks/post-receive script.
tinodidriksen-guest pushed a commit to branch master
in repository hfst-ospell.
commit 545c13e73f17743849b2106eb02a51956cd71828
Author: Tino Didriksen <tino at didriksen.cc>
Date: Tue Jan 10 20:42:44 2017 +0000
Imported Upstream version 0.4.3~r338
---
.gitignore | 22 +
.travis.yml | 15 +
ChangeLog | 1827 ++++----------------
Makefile.am | 103 +-
NEWS | 19 +-
README | 47 +-
ZHfstOspeller.cc | 235 ++-
ZHfstOspeller.h | 56 +-
ZHfstOspellerXmlMetadata.cc | 41 +-
ZHfstOspellerXmlMetadata.h | 2 +
autogen.sh | 2 +-
configure.ac | 24 +-
hfst-ol.cc | 362 +++-
hfst-ol.h | 77 +-
hfst-ospell-office.1 | 2 +-
hfst-ospell.1 | 2 +-
hfstol-stdafx.h | 44 +
main-cicling.cc | 1 -
main-fsmnlp-2012.cc | 1 -
main-norvig.cc | 1 -
main-survey.cc | 1 -
main.cc | 1 -
office.cpp => office.cc | 2 -
ol-exceptions.h | 3 +-
ospell.cc | 63 +-
ospell.h | 21 +-
acceptor.basic.txt => tests/acceptor.basic.txt | 0
analyse-spell.sh => tests/analyse-spell.sh | 2 +-
analyser.default.txt => tests/analyser.default.txt | 0
bad-errormodel.sh => tests/bad-errormodel.sh | 2 +-
tests/bad_errormodel.zhfst | Bin 0 -> 1408 bytes
basic-edit1.sh => tests/basic-edit1.sh | 2 +-
basic-zhfst.sh => tests/basic-zhfst.sh | 2 +-
basic_test.xml => tests/basic_test.xml | 0
.../empty-descriptions.sh | 2 +-
empty-locale.sh => tests/empty-locale.sh | 2 +-
empty-titles.sh => tests/empty-titles.sh | 2 +-
empty-zhfst.sh => tests/empty-zhfst.sh | 3 +-
.../empty_descriptions.xml | 0
tests/empty_descriptions.zhfst | Bin 0 -> 1265 bytes
empty_locale.xml => tests/empty_locale.xml | 0
tests/empty_locale.zhfst | Bin 0 -> 1400 bytes
empty_titles.xml => tests/empty_titles.xml | 0
tests/empty_titles.zhfst | Bin 0 -> 1370 bytes
errmodel.basic.txt => tests/errmodel.basic.txt | 0
errmodel.edit1.txt => tests/errmodel.edit1.txt | 0
.../errmodel.extrachars.txt | 0
no-errormodel.sh => tests/no-errormodel.sh | 2 +-
no_errmodel.xml => tests/no_errmodel.xml | 0
tests/no_errormodel.zhfst | Bin 0 -> 972 bytes
tests/speller_analyser.zhfst | Bin 0 -> 3298 bytes
tests/speller_basic.zhfst | Bin 0 -> 1405 bytes
tests/speller_edit1.zhfst | Bin 0 -> 3245 bytes
test.strings => tests/test.strings | 0
trailing-spaces.sh => tests/trailing-spaces.sh | 2 +-
trailing_spaces.xml => tests/trailing_spaces.xml | 0
tests/trailing_spaces.zhfst | Bin 0 -> 1410 bytes
windows-Makefile.am | 4 +-
58 files changed, 994 insertions(+), 2005 deletions(-)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..c367c75
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,22 @@
+.deps
+.libs
+autom4te.cache
+build-aux
+*.o
+*.lo
+Makefile
+Makefile.in
+config.*
+configure
+hfst-ospell
+hfst-ospell-office
+hfstospell.pc
+libhfstospell.la
+libtool
+stamp-h1
+aclocal.m4
+m4/libtool.m4
+m4/ltoptions.m4
+m4/ltsugar.m4
+m4/ltversion.m4
+m4/lt~obsolete.m4
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..22ea2ab
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,15 @@
+sudo: required
+dist: trusty
+language: cpp
+
+addons:
+ apt:
+ packages:
+ - libxml++2.6-dev
+ - libarchive-dev
+
+script:
+ - autoreconf -fi
+ - ./configure --disable-static --enable-zhfst
+ - make -j3
+ - make -j1 check
diff --git a/ChangeLog b/ChangeLog
index 27d681d..4f13d12 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,1507 +1,320 @@
-2016-01-13 16:39 hardwick
-
- * ospell.cc, ospell.h: Only call clock() every millionth time for
- speed
-
-2016-01-13 09:54 hardwick
-
- * ospell.cc, ospell.h: Fix time cutoff handling
-
-2016-01-11 14:01 jezral
-
- * Makefile.am, hfst-ospell-office.1, hfst-ospell.1: Update man
- pages
-
-2016-01-11 13:54 jezral
-
- * hfst-ospell.1: Update man page
-
-2015-12-03 10:09 jezral
-
- * office.cpp: If the input token had any upper case, also try a
- first-upper variant of it
-
-2015-12-03 09:38 jezral
-
- * office.cpp: Special case nuvviDspeller
-
-2015-11-30 11:13 moshagen
-
- * ospell.h: Bugfix: class -> struct (libvoikko complained under
- OSX, and err’ed out).
-
-2015-11-28 23:51 jezral
-
- * office.cpp: If we are looking for suggestions, don't use the
- cache
-
-2015-11-27 15:55 eaxelson
-
- * main.cc: Fix a couple of typos in argument list and verbose
- prints. Add a missing statement that actually sets the time
- cutoff for the speller.
-
-2015-11-27 15:52 eaxelson
-
- * main.cc: Fix a typo in wide string conversion function.
-
-2015-11-27 11:35 hardwick
-
- * ZHfstOspeller.cc, ZHfstOspeller.h, main.cc, ospell.cc, ospell.h:
- Time cutoff option during correction for library and demo app
-
-2015-11-24 09:31 hardwick
-
- * hfst-ol.cc: Apply Brendan Molloy's alignment patch for ARM CPUs &
- reindent to namespace=0
-
-2015-11-18 13:34 jezral
-
- * office.cpp: Propagate first-upper and all-upper to suggestions
-
-2015-11-18 11:29 eaxelson
-
- * Makefile.am, ZHfstOspeller.cc, configure.ac: Add option
- --enable-hfst-ospell-office to configure (defaults to yes). Add
- support for using libarchive2 if libarchive3 is not available.
-
-2015-11-04 09:37 jezral
-
- * office.cpp: Add --verbatim to disable the variations
-
-2015-10-23 11:19 jezral
-
- * office.cpp: Don't use UTF-8 length for UTF-16 buffer
-
-2015-10-22 10:21 jezral
-
- * office.cpp: I meant 20480
-
-2015-10-22 10:15 jezral
-
- * office.cpp: Use ICU to determine what's alphanumeric - should fix
- https://github.com/TinoDidriksen/spellers/issues/1
-
-2015-10-15 14:02 jezral
-
- * configure.ac: If pkg-config fails, fall back on icu-config -
- blame Ubuntu 12.04
-
-2015-10-15 13:28 jezral
-
- * office.cpp: Silly mistake
-
-2015-10-15 13:21 jezral
-
- * configure.ac, office.cpp: Use ICU instead
-
-2015-10-15 10:40 jezral
-
- * office.cpp: Use wide chars during case folding; Set locale
-
-2015-10-02 12:44 hardwick
-
- * ospell.cc: Fix typo that broke UNKNOWN handling the the error
- source
-
-2015-10-02 12:32 hardwick
-
- * hfst-ol.cc, hfst-ol.h, ospell.cc: Fix bug in novel symbol
- tokenization
-
-2015-10-01 12:16 hardwick
-
- * hfst-ol.cc, hfst-ol.h: Bounds check table accesses
- Now that new symbols from input can be added to the lexicon's
- alphabet,
- it's possible that during checking we try to access beyond the
- lexicon's table boundaries, so that needs to be prevented.
-
-2015-09-30 09:05 jezral
-
- * configure.ac, m4/ax_check_compile_flag.m4: CentOS 6 fixes: Move
- AC_INIT up to prevent _m4_divert_diversion error; Use older
- ax_check_compile_flag.m4
-
-2015-09-30 07:42 jezral
-
- * m4/ax_check_compile_flag.m4: autoconf-archive doesn't exist on
- RPM platforms, so bundle ax_check_compile_flag.m4
-
-2015-09-30 07:07 jezral
-
- * configure.ac: Determine and use highest supported C++ standard
-
-2015-09-29 07:26 hardwick
-
- * ospell.cc, ospell.h: Reinstate legacy function
- Transducer::lookup() due to Voikko's use of it
-
-2015-09-28 17:59 hardwick
-
- * hfst-ol.cc, hfst-ol.h, ospell.cc, ospell.h: Unknown and identity
- arcs can now appear both in the lexicon and error model
- This is a rather large overhaul of everything having to do with
- symbols,
- and causes an approximately 8% slowdown at this time. Some more
- efforts
- will be made to bring that down..
-
-2015-09-22 13:16 hardwick
-
- * ospell.cc, ospell.h: Make weight checking methods const
-
-2015-09-22 13:09 hardwick
-
- * ospell.cc, ospell.h: Correct nbest handling for cached results
- too
- + remove extranous comparison class
-
-2015-09-22 12:46 hardwick
-
- * ospell.cc, ospell.h: Special-case limit checking for nbest
-
-2015-09-22 12:15 hardwick
-
- * ospell.cc, ospell.h: Correct limit setting
-
-2015-09-22 12:11 hardwick
-
- * ospell.cc: Remove debugging print
-
-2015-09-22 12:09 hardwick
-
- * ospell.cc, ospell.h: Use a list to emulate a sorted double-ended
- weight queue
-
-2015-06-17 10:21 moshagen
-
- * office.cpp: Removed keyword 'typename', it caused GCC with C++03
- to bark.
-
-2015-06-05 14:01 jezral
-
- * Makefile.am, office.cpp: Drop down to boring ol' C++03
-
-2015-06-04 12:24 hardwick
-
- * test/editdist.py: Fix declaration order and unicode issues
-
-2015-06-03 11:51 hardwick
-
- * test/editdist.py: Add --default-weight, --outputfile, pair
- specification syntax for regex output
-
-2015-06-01 11:48 hardwick
-
- * test/editdist.py: Add a --regex mode to produce a large regular
- expression with weighted replace rules
-
-2015-05-20 09:55 hardwick
-
- * ospell.cc, ospell.h: Add a first-symbol cache and do some reorg
-
-2015-04-17 10:04 jezral
-
- * office.cpp: Fix -Wall -Wextra issues
-
-2015-04-17 09:52 jezral
-
- * office.cpp: New protocol: Let client choose max suggestions,
- input must now match "^(\d+) (.+)$"
-
-2015-04-17 09:16 jezral
-
- * office.cpp: Search up the normalization list until something
- gives suggestions
-
-2015-04-11 21:15 jezral
-
- * office.cpp: Disable log2 max-dist and instead set hard limit due
- to FST weights
-
-2015-04-11 19:27 jezral
-
- * office.cpp: Tokenization is done by the host, so don't second
- guess
-
-2015-04-11 18:39 jezral
-
- * Makefile.am, office.cpp: Add hfst-ospell-office helper so the
- algorithm is in one place, instead of in every frontend
-
-2015-04-09 09:23 hardwick
-
- * ospell.cc: Arguably clearer scheme for combining limiting options
-
-2015-02-23 18:27 hardwick
-
- * ZHfstOspeller.cc, ZHfstOspeller.h, main.cc: Reinstate legacy
- spelling, slightly adding to but not altering interface
-
-2015-01-30 14:04 hardwick
-
- * ZHfstOspeller.cc, ZHfstOspeller.h, main.cc, ospell.cc, ospell.h:
- Add option --beam=W for restricting the search to a margin above
- the optimum.
- Any combination of n-best, weight limit and beam is possible, all
- restrict each other.
-
-2015-01-29 11:38 hardwick
-
- * ospell.h: Use vector instead of deque now that breadth search
- isn't used for anything
-
-2015-01-29 11:08 hardwick
-
- * ospell.cc, ospell.h: Be a bit more systematic about
- search-limiting behaviour, also fail earlier,
- this gives a small speed improvement
-
-2015-01-20 06:42 hardwick
-
- * main.cc: Make argument parsing further agree with documentation,
- reinstating -X
-
-2015-01-20 06:38 hardwick
-
- * ospell.cc: nbest wasn't producing any runtime benefit due to
- incorrect comparison
-
-2015-01-20 06:35 hardwick
-
- * main.cc: Reinstate the : lost from the -n arg in getopt_long()
-
-2015-01-19 14:10 hardwick
-
- * main.cc: Change max_weight to max-weight in the help string and
- argument name
-
-2015-01-19 14:09 hardwick
-
- * ospell.cc, ospell.h: Check that maxweight is positively set
- before using it, default is negative
-
-2015-01-19 14:05 hardwick
-
- * ZHfstOspeller.cc, ZHfstOspeller.h, configure.ac, main.cc,
- ospell.cc, ospell.h: Version 0.3.1 - this introduces API/linking
- changes
- Support limiting the correction search to a maximum weight.
- The hfst-ospell command supports this via --max_weight / -w.
-
-2015-01-19 12:29 hardwick
-
- * ospell.cc: Make -n (max n corrections) work again
-
-2014-07-18 17:28 mie
-
- * configure.ac: Have separate switches for each XML mangling
- library
-
-2014-06-19 10:10 eaxelson
-
- * hfst-ospell.1, main.cc, windows-configure.ac: Updated hfst-ospell
- man page.
-
-2014-06-08 22:52 mie
-
- * ZHfstOspellerXmlMetadata.cc, configure.ac: Just error out when
- missing XML
-
-2014-06-08 19:53 mie
-
- * hfst-ospell.1, main.cc: Allow selecting real-word errors
-
-2014-06-04 12:18 mie
-
- * main.cc: Missing stdarg reqd in >gcc 4.8 or somesuch
-
-2014-05-21 09:23 eaxelson
-
- * windows-Makefile.am, windows-configure.ac: Added configure and
- Makefile files for Windows compilation.
-
-2014-05-09 12:08 hardwick
-
- * ospell.cc: A tiny performance improvement
- (about 5%, this is not what was discussed on irc, but every
- little helps)
-
-2014-05-09 12:04 hardwick
-
- * main.cc: Fix missing newline in verbose message
-
-2014-05-09 11:46 hardwick
-
- * ZHfstOspeller.cc: This is how I got the analysing speller to work
-
-2014-05-09 11:33 mie
-
- * main.cc: Simple example of analysis spelling logic
-
-2014-05-09 11:05 mie
-
- * ZHfstOspeller.cc: can analyse if speller or sugger
-
-2014-05-08 16:38 hardwick
-
- * ospell.cc, ospell.h: Fix bug #239
- When correcting, keep track of mutator's output / lexicon's
- input;
- when analysing, keep track of lexicon's output
-
-2014-05-06 10:49 hardwick
-
- * ospell.cc: Fix bug #238
- Now that alphabets are translated even if the error source
- contains symbols
- not found in the lexicon, the lexicon needs to be prepared to be
- asked whether
- there are transitions with NO_SYMBOL
-
-2014-04-23 14:11 mie
-
- * doc, doc/index.html: documentation front page
-
-2014-04-23 13:35 mie
-
- * ChangeLog, Doxyfile, Makefile.am, NEWS, configure.ac: Dist
- changes, 0.3.0 API stable though pending other fixes for .1
-
-2014-04-23 12:34 mie
-
- * Doxyfile, Makefile.am, ZHfstOspeller.cc, ZHfstOspeller.h,
- ZHfstOspellerXmlMetadata.cc, ZHfstOspellerXmlMetadata.h,
- configure.ac, edit2-small.png, hfst-ol.h, ol-exceptions.h,
- ospell.h: Some beginnings for a documented stable API.
-
-2014-04-23 09:50 mie
-
- * COPYING: This has been missing all along?
-
-2014-04-23 08:13 mie
-
- * basic-legacy.sh, basic-zhfst-fallback.sh, main.cc: Allow setting
- max suggestions from command line tool
-
-2014-04-23 08:00 mie
-
- * ZHfstOspeller.cc, ZHfstOspeller.h, main.cc: Remove legacy, allow
- setting suggestion limits
-
-2014-04-22 20:13 mie
-
- * Makefile.am, ZHfstOspeller.cc, ZHfstOspeller.h, ospell.cc,
- ospell.h: Tentative support for analyse-as-you-error-correct
-
-2014-04-22 17:28 mie
-
- * Makefile.am, basic-edit1.sh, main.cc: Errmodel with out of
- lexicon alphabet fails
-
-2014-04-22 17:10 mie
-
- * Makefile.am, acceptor.basic.txt, acceptor.default.txt,
- configure.ac, errmodel.basic.txt, errmodel.default.txt,
- errmodel.edit1.txt, test.strings: Better test case for analyser
-
-2014-04-22 16:43 mie
-
- * Makefile.am, analyse-spell.sh, analyser.default.txt,
- hfst-ospell.1, main-ispell.cc, no-errormodel.sh, no_errmodel.xml:
- Add tests for analysing speller and missing errmodels
-
-2014-04-22 16:39 mie
-
- * ZHfstOspeller.cc, ZHfstOspeller.h, main.cc, ospell.cc, ospell.h:
- Analysis interface and implementation for two-tape language
- models. The
- suggestion engine still erroneously gives the analysis side of
- suggestion
- LM though.
-
- Also safeguarded some null lms and ems.
-
-2014-04-10 08:18 eaxelson
-
- * main.cc: Now printing and reading to/from console works in
- hfst-ospell is it is compiled with -DWINDOWS.
-
-2014-03-28 08:15 eaxelson
-
- * Makefile.am, hfst-ospell.1: Added a rule for man page creation in
- Makefile.am.
-
-2014-03-14 07:28 moshagen
-
- * Makefile.am, empty-zhfst.sh: Added a test for an empty zhfst
- file. It should fail, but presently hfst-ospell behaves as if all
- is ok, just that it isn't able to correct anything.
-
-2014-03-14 07:24 moshagen
-
- * bad-errormodel.sh, basic-legacy.sh, basic-zhfst-fallback.sh,
- basic-zhfst.sh, empty-descriptions.sh, empty-locale.sh,
- empty-titles.sh, trailing-spaces.sh: Corrected return value to
- what it should be for a SKIP result, which I assume was intended.
-
-2014-03-11 17:15 moshagen
-
- * Makefile.am: Added files to the distro.
-
-2014-03-11 06:14 moshagen
-
- * Makefile.am, bad-errormodel.sh, errmodel.extrachars.txt: Added
- test to check whether error models with extra chars are correctly
- handled. The test passes with revision 3793 of ospell.cc, and
- fails with revision 3729.
-
-2014-03-09 14:24 hardwick
-
- * ospell.cc: Silently ignore non-lexicon symbols in the error
- source, translating them
- to NO_SYMBOL on the output side.
-
-2014-02-13 21:50 mie
-
- * ospell.cc, ospell.h: revert
-
-2014-02-13 19:46 mie
-
- * ospell.cc, ospell.h: What happens if I do this?
-
-2014-02-13 19:17 mie
-
- * Makefile.am, hfst-ol.cc, hfst-ol.h, ospell.cc, ospell.h: Moving
- implementations?
-
-2014-02-13 17:59 mie
-
- * Makefile.am: Install men
-
-2014-02-13 17:37 mie
-
- * hfst-ospell.1: A man page for debian's lint
-
-2014-02-05 05:55 mie
-
- * ZHfstOspeller.cc: Probably libarchive deprecation will go away
- with different function name
- here too
-
-2014-02-05 05:33 mie
-
- * ospell.cc, ospell.h: Move some of the code away from headers
-
-2014-01-27 12:26 mie
-
- * ZHfstOspeller.cc: some old changes on work desktop?
-
-2013-12-23 05:52 mie
-
- * ZHfstOspellerXmlMetadata.cc: Ensure other potentially empty tags
- for libxmlpp
-
-2013-12-23 05:22 mie
-
- * configure.ac: Bumped version for tinyxml2 check to get lengthful
- Parse by Børre Gaup
-
-2013-12-22 07:01 mie
-
- * ZHfstOspeller.cc, ZHfstOspellerXmlMetadata.cc, ospell.cc:
- unarchive to mem and tinyxml2 xml length fixes from Harri
- Pitkänen
-
-2013-12-20 08:57 moshagen
-
- * ZHfstOspeller.cc, ZHfstOspeller.h: Reverted the patch by Børre to
- clean the xml off of random byte noise after the end of the xml
- document. The issue was caused by zip compression
- incompatibilities between different plattforms, and the solution
- (for now) is to just skip compression altogether, using 'zip -Z
- store' instead when building the zhfst files. This creates a zip
- file readable everywhere (but without compression).
-
-2013-12-18 13:03 mie
-
- * Makefile.am, trailing-spaces.sh, trailing_spaces.xml: Test for
- trailing spaces in xml structure
-
-2013-12-16 15:20 moshagen
-
- * ZHfstOspeller.cc, ZHfstOspeller.h: Patch by Børre Gaup to clean
- corrupted xml data returned from Libarchive when reading xml data
- from a zhfst file stored in RAM:
-
- libarchive seems to add one char from the following file in the
- stream pr whitespace char it (erroneously) removes from the xml
- data around newlines, since the returned xml file is truncated
- compared to the original file length reported when stored in the
- archive. Although the xml info set is exactly the same, the extra
- chars after the final closing element causes TinyXML2 to crash.
-
- This change removes everything after the final '>', which should
- clean the string representation of the xml document properly in
- all cases but one: when the semi random noise contains an '>'.
- This situation is presently not handled.
-
-2013-12-14 13:11 moshagen
-
- * Makefile.am: Whitespace changes to make the file easier to my
- eyes, comments. Added tests for empty locale and empty title
- elements.
-
-2013-12-14 13:10 moshagen
-
- * empty-locale.sh, empty-titles.sh, empty_locale.xml,
- empty_titles.xml: Data and shell scripts to test empty titles and
- empty locale.
-
-2013-12-14 13:09 moshagen
-
- * ZHfstOspellerXmlMetadata.cc: Check for empty locale node.
-
-2013-12-13 23:50 moshagen
-
- * ., Makefile.am: Clean and ignore index.xml.
-
-2013-12-13 09:37 moshagen
-
- * ZHfstOspellerXmlMetadata.cc: One more test for empty data.
-
-2013-12-13 09:37 moshagen
-
- * main.cc: Added newline before error cause.
-
-2013-12-13 01:10 mie
-
- * Makefile.am, ZHfstOspellerXmlMetadata.cc, configure.ac, main.cc:
- Empty descriptions will throw (there might be others left)
-
-2013-12-13 00:48 mie
-
- * Makefile.am, basic-zhfst.sh, basic_test.xml,
- empty-descriptions.sh, empty_descriptions.xml, index.xml:
- index.xml with empty descriptions
-
-2013-12-13 00:36 mie
-
- * Makefile.am: Test.strings shall not be deleted when cleaning
-
-2013-12-11 14:12 moshagen
-
- * configure.ac: Added check for the availability of pkg-config -
- without it configuration will fail in subtle but bad ways.
-
-2013-11-20 01:36 mie
-
- * ChangeLog, NEWS: Document for next release candidate
-
-2013-11-19 22:58 mie
-
- * ZHfstOspellerXmlMetadata.cc: Missing model attribute parsing for
- errm
-
-2013-11-19 22:27 mie
-
- * ZHfstOspellerXmlMetadata.cc, ZHfstOspellerXmlMetadata.h: Use
- Elements instead of Nodes and other such fixes
-
-2013-11-19 22:26 mie
-
- * Makefile.am, configure.ac: Use automake conditionals to avoid
- pkg-config linking to libraries that
- are not in configure's code paths
-
-2013-11-19 21:07 mie
-
- * configure.ac: update configuration for that
-
-2013-11-19 20:52 mie
-
- * ZHfstOspellerXmlMetadata.cc: Add tinyxml2 versions of XML parsing
-
-2013-11-14 02:06 mie
-
- * Makefile.am, configure.ac: conference demos as configure option
-
-2013-10-08 11:21 moshagen
-
- * .: Ignore generated files.
-
-2013-10-08 07:55 spectre360
-
- * main-lrec2013.cc: stats -> status
-
-2013-10-02 16:03 mie
-
- * main-lrec2013.cc: Allow to end correct correction
-
-2013-09-27 16:25 hardwick
-
- * ospell.cc, ospell.h: Merge lookup() from devbranch
-
-2013-09-24 14:52 mie
-
- * Makefile.am, main-ispell.cc: This should be useful
-
-2013-09-24 11:25 mie
-
- * main-lrec2013.cc: Fix the output tsv a bit more
-
-2013-09-23 15:56 mie
-
- * Makefile.am, configure.ac, main-lrec2013.cc: Slight udpate for
- new measure ments
-
-2013-07-04 05:02 mie
-
- * ZHfstOspellerXmlMetadata.cc, ZHfstOspellerXmlMetadata.h: Few
- starts
-
-2013-07-04 02:29 mie
-
- * configure.ac: Allow selection of xml backend
-
-2013-05-28 14:34 mie
-
- * configure.ac: Switch <3 into >3 because it looks nicer.
-
-2013-05-24 15:32 moshagen
-
- * ospell.cc, ospell.h: strndup() fixed by Tommi.
-
-2013-05-24 11:44 hardwick
-
- * ZHfstOspeller.cc, ZHfstOspellerXmlMetadata.cc, ospell.h: Pending
- proper understanding of why strndup is getting defined several
- times over, let's always use hfst_strndup() instead.
-
-2013-05-24 11:29 hardwick
-
- * ospell.h: Add include guard to custom strndup so it doesn't get
- compiled more than once
-
-2013-05-24 08:58 hardwick
-
- * ZHfstOspeller.cc, ospell.h: Move our custom strndup to ospell.h
- so it will be seen by
- all compilation units that need it
-
-2013-05-21 12:33 moshagen
-
- * Makefile.am, NEWS, configure.ac, hfstospell.pc.in: Renamed the
- hfstospell package and variables to what they used to be, in case
- that can help solve a build issue with libvoikko+hfst.
-
-2013-04-27 11:36 mie
-
- * ChangeLog: Wrap and load
-
-2013-04-27 11:32 mie
-
- * Makefile.am: versioninfo 3:1:1
-
-2013-04-26 17:58 moshagen
-
- * NEWS: Preparing for 0.2.3 release.
-
-2013-04-26 14:34 moshagen
-
- * configure.ac, hfstospell.pc.in: Changed version number to 0.2.3,
- and renamed the package and tool name to 'hfst-ospell', since
- that is what the command-line tool is actually called, and it is
- consistent with the rest of the hfst tool names.
-
-2013-04-26 12:44 hardwick
-
- * ospell.cc: Should fix #176
- Flag state from the stack back was getting clobbered in some
- situations.
- Now we restore flag state after modifying it for new nodes.
-
-2013-03-22 07:49 moshagen
-
- * .: More ignore patterns to ignore files generated during 'make
- check'.
-
-2013-03-22 06:21 mie
-
- * Makefile.am: Fix dist stuff, wrap.
-
-2013-03-20 10:13 mie
-
- * Makefile.am, ZHfstOspeller.cc, configure.ac: Use pkg-config
- instead of autoconf to check libarchive
-
-2013-03-18 12:43 mie
-
- * ChangeLog, Makefile.am, NEWS, configure.ac: Prepare files for
- 0.2.2
-
-2013-03-18 12:37 mie
-
- * ZHfstOspellerXmlMetadata.cc, main.cc: Fix few leaks
-
-2013-03-18 10:26 mie
-
- * ZHfstOspeller.cc, hfst-ol.cc, hfst-ol.h, ospell.h: Fixes to
- [#165] by Harri Pitkänen
-
-2013-02-04 13:54 hardwick
-
- * ospell.cc, ospell.h: Made some changes to depth-first searching
- to prevent just-added
- nodes getting removed. This appears fo fix a very long-standing
- and serious bug dating from r2763.
-
-2013-01-16 12:46 hardwick
-
- * hfst-ol.cc, hfst-ol.h: When data is in memory, allocate & copy it
- in ospell rather than
- expect caller to keep track of it (which it usually doesn't)
-
-2013-01-08 06:21 mie
-
- * ZHfstOspeller.cc, ZHfstOspeller.h: Add access to metadata
-
-2012-12-08 10:32 hardwick
-
- * hfst-ol.h: When initialised from memory, don't assume
- responsibility for
- freeing it
-
-2012-12-04 08:48 hardwick
-
- * hfst-ol.cc, hfst-ol.h, ospell.cc, ospell.h: Somewhat experimental
- (no checking) way of saving memory by ignoring alignment
- issues (4-5 x memory savings)
-
-2012-10-05 07:48 hardwick
-
- * ospell.h: Remove leftover agenda variable
-
-2012-10-05 05:18 mie
-
- * Makefile.am, main-survey.cc: Add stuff for survey article
-
-2012-10-03 12:19 hardwick
-
- * main.cc, ospell.h: Revert to breadth-first searching pending
- bugfix
-
-2012-09-25 09:20 hardwick
-
- * ospell.cc: Search depth-first by preference for hope
-
-2012-09-20 16:30 hardwick
-
- * ospell.cc: Forgot some important checks that we actually want to
- limit the results
-
-2012-09-20 16:27 hardwick
-
- * ospell.cc, ospell.h: Enforce an n-best limit for continuing the
- search, just for breadth-first for
- now
-
-2012-09-04 19:11 mie
-
- * main.cc: Don’t just break on empty lines
-
-2012-08-21 17:59 mie
-
- * ZHfstOspeller.cc: Switch order of errmodels and acceptors in
- legacy read
-
-2012-08-21 17:50 mie
-
- * ZHfstOspeller.cc: set spelling caps on legacy read
-
-2012-08-21 17:50 mie
-
- * Makefile.am, basic-zhfst-fallback.sh: Test fallback
-
-2012-08-15 08:48 moshagen
-
- * basic-legacy.sh, basic-legacy.sh.in, basic-zhfst.sh,
- basic-zhfst.sh.in, configure.ac: The VPATH approach to the test
- shell scripts was wrong. Now everything is working as it should.
-
-2012-08-15 04:57 moshagen
-
- * basic-legacy.sh.in, basic-zhfst.sh.in: Proper linebreaks.
-
-2012-08-15 04:56 moshagen
-
- * ., basic-legacy.sh, basic-legacy.sh.in, basic-zhfst.sh,
- basic-zhfst.sh.in, configure.ac: Enabling VPATH building for
- 'make check'.
-
-2012-08-15 04:15 moshagen
-
- * ., configure.ac: Whitespace change only in configure.ac. More
- generated files to ignore.
-
-2012-08-15 03:57 moshagen
-
- * Makefile.am: Added some whitespace to ease readability. Replaced
- pattern rules with suffix rules to make automake happy.
-
-2012-08-14 20:19 mie
-
- * ZHfstOspellerXmlMetadata.cc: Fix compilation without xml,
- as suggested by Harri Pitkänen on libvoikko-devel
-
-2012-08-14 18:06 mie
-
- * ZHfstOspeller.cc, main.cc: do not close unopened files, handle
- legacy fallback reading errors
-
-2012-08-14 17:25 mie
-
- * main.cc: print details of xml parsing errors
-
-2012-08-14 16:13 mie
-
- * Makefile.am, acceptor.default.txt, basic-legacy.sh,
- basic-zhfst.sh, configure.ac, errmodel.default.txt, index.xml,
- test.strings: Add the very basic test suite
-
-2012-08-13 20:23 mie
-
- * ZHfstOspeller.cc, configure.ac: Throw exception if reading a zip
- without existing zip file.
-
-2012-08-13 20:02 mie
-
- * configure.ac: Optionalise the libraries
-
-2012-07-22 01:52 mie
-
- * configure.ac: Update for automake-1.12 and AM_PROG_AR
-
-2012-07-06 19:03 mie
-
- * main.cc: kill everyone if windows linebreaks
-
-2012-07-05 18:13 mie
-
- * AUTHORS, ChangeLog, NEWS, README, authors.xml, configure.ac,
- main.cc: Fix documentation and set for 0.2.1 release
-
-2012-07-04 15:38 mie
-
- * ZHfstOspeller.cc, ZHfstOspeller.h: free more things and stuff
-
-2012-07-04 15:37 mie
-
- * Makefile.am, ZHfstOspellerXmlMetadata.cc,
- ZHfstOspellerXmlMetadata.h: Move Xml metadata parsing and storing
- into own file and class
-
-2012-07-04 10:28 hardwick
-
- * hfst-ol.h: Increment *raw when reading bools
-
-2012-07-04 10:23 hardwick
-
- * hfst-ol.cc, hfst-ol.h: Added utility function for iterating c
- strings in raw memory,
- use it in every branch of symbol reading
-
-2012-07-04 07:21 hardwick
-
- * hfst-ol.cc: Fixed problems having to do with reading strings from
- a transducer in raw
- memory.
-
-2012-07-03 21:04 mie
-
- * ZHfstOspeller.cc, configure.ac: fix the tmpdir'd version again
-
-2012-07-03 19:42 mie
-
- * Makefile.am, ZHfstOspeller.cc, ZHfstOspeller.h, configure.ac,
- hfst-ol.cc: Version that extracts zhfst to memory iff it fits on
- one throw
-
-2012-07-02 22:19 hardwick
-
- * hfst-ol.cc, hfst-ol.h, ospell.h: Added raw memory constructors
- and table readers for slurping in transducers
- from char *
-
-2012-07-02 19:49 mie
-
- * main-cicling.cc, main-fsmnlp-2012.cc: Missing files lol
-
-2012-04-25 16:36 hardwick
-
- * test/editdist.py: ...and in that case the final identity states
- also are greater by one.
-
-2012-04-25 16:33 hardwick
-
- * test/editdist.py: When avoiding initial edits, there needs to be
- an extra inital state before
- we do any edits. So add the value of options.no_initial to the
- state range
- loop.
-
-2012-04-18 14:53 eaxelson
-
- * Makefile.am: Commented out hfst-ospell-cicling from Makefile.
-
-2012-04-18 14:48 hardwick
-
- * Makefile.am: Comment out missing cicling target
-
-2012-04-18 11:24 hardwick
-
- * main-fsmnlp-2012.cc: "else if" instead of incorrect "if" in
- option handling
-
-2012-04-17 13:23 hardwick
-
- * test/editdist.py: --no-string-initial should have transitions to
- state 1, not 0
-
-2012-03-31 14:06 hardwick
-
- * main-fsmnlp-2012.cc: Don't exit on empty lines
-
-2012-03-31 13:12 hardwick
-
- * main-fsmnlp-2012.cc: Don't print unhelpful warnings
-
-2012-03-31 08:55 hardwick
-
- * test/editdist.py: Silly default for minimum edit
-
-2012-03-19 08:26 hardwick
-
- * main-fsmnlp-2012.cc: fallback_spell()
-
-2012-03-19 07:40 hardwick
-
- * test/editdist.py: --no-string-initial-correction
-
-2012-03-05 08:58 hardwick
-
- * test/editdist.py: Corrected eliminations (hopefully), added
- --minimum-edit option
-
-2012-02-24 05:02 mie
-
- * Makefile.am, NEWS, main-fsmnlp-2012.cc: Profiled version for
- fsmnlp-2012 measurements§
-
-2011-11-08 11:18 hardwick
-
- * test/editdist.py: Don't skip state numbers when certain options
- are turned off
- (and don't print debugging lines)
-
-2011-11-08 11:16 hardwick
-
- * test/editdist.py: Corrections to redundancy elimination
-
-2011-11-01 13:55 hardwick
-
- * test/editdist.py: redundancy elimination was being performed one
- state too late (too little)
-
-2011-10-25 10:40 hardwick
-
- * test/editdist.py: Add option to disable redundancy elimination
-
-2011-10-25 10:35 hardwick
-
- * test/editdist.py: Initial support for redundancy elimination
- (refuse to do insertion after deletion or deletion after
- insertion)
-
-2011-10-11 19:30 hardwick
-
- * test/editdist.py: Bugfix: identity transitions were being
- forgotten in the last edit state
-
-2011-10-07 10:28 hardwick
-
- * hfst-ol.cc, test/editdist.py: Enforce @_UNKNOWN_SYMBOL_@ instead
- of @?@, which users didn't know about
-
-2011-10-03 14:23 moshagen
-
- * test/editdist.py: Fixed one remaining UTF-8 stderr printing bug.
-
-2011-09-28 08:52 hardwick
-
- * test/editdist.py: Fixed bug with the newline character not being
- stripped from excluded symbols
-
-2011-09-28 08:42 hardwick
-
- * test/editdist.py: Lines that start with ## are comments
-
-2011-09-28 08:24 hardwick
-
- * test/editdist.py: Updated help message & added exclusion of
- symbols by prepending a ~
-
-2011-09-28 07:44 hardwick
-
- * test/editdist.py: Wrap stderr with a utf-8 codec so we can print
- non-ascii symbols when verbose
-
-2011-09-28 07:32 hardwick
-
- * test/editdist.py: Write alphabet with weights when verbose
-
-2011-09-28 07:24 hardwick
-
- * test/editdist.py: Order of preference of alphabet definition is
- now
- configfile - commandline - transducer.
- If configfile gives a weight after a tab for symbols,
- they are used additively for all edits involving those symbols.
-
-2011-09-28 06:42 hardwick
-
- * test/editdist.py: Some clarifying comments
-
-2011-09-28 06:36 hardwick
-
- * test/editdist.py: Rescue identities from being considered
- substitutions
-
-2011-09-28 05:50 hardwick
-
- * test/editdist.py: utf-8 -decode the user-specified transitions
- from a conf file
- (so easy to forget one of these...)
-
-2011-09-13 22:03 mie
-
- * hfstospell.pc.in: use same includedir in pc as makefile
-
-2011-09-13 20:54 mie
-
- * ZHfstOspeller.h: don't declare strndup in public headers
-
-2011-09-06 07:46 mie
-
- * ol-exceptions.h: make ol exceptions in hfst_ol namespace, provide
- stdexception style what()
-
-2011-09-02 06:12 mie
-
- * main.cc: Add verbose, quiet
-
-2011-09-01 21:41 mie
-
- * ZHfstOspeller.cc, ZHfstOspeller.h, ospell.h: * parse all of the
- metadata if possible
- * use c++ ``struct''s for metadata
-
-2011-09-01 21:36 mie
-
- * ZHfstOspeller.cc: * Use temporary filenames by tmpsnam
- * Do not delete Transducers in data structures since it will
- segfault all
- enchant-based applications in dtor
-
-2011-08-31 08:56 hardwick
-
- * Makefile.am, hfst-ol.cc, hfst-ol.h, main.cc, ol-exceptions.h:
- libhfst-style exception macros and some more informative messages
-
-2011-08-31 08:26 hardwick
-
- * README: Document dependencies
-
-2011-08-17 11:14 moshagen
-
- * .: Ignore generated files.
-
-2011-08-17 08:20 moshagen
-
- * m4: Ignore generated files.
-
-2011-08-17 02:50 mie
-
- * ZHfstOspeller.cc, ZHfstOspeller.h, configure.ac: mac os x fixes:
- * strndup
- * libarchive installed without headers
-
-2011-08-08 02:49 mie
-
- * Makefile.am, ZHfstOspeller.cc, ZHfstOspeller.h, configure.ac, m4,
- main.cc: Preliminary zhfst support
-
-2011-07-21 01:05 mie
-
- * main-norvig.cc: Example from:
- http://norvig.com/spell-correct.html and similar tests
-
-2011-07-20 11:22 hardwick
-
- * hfst-ol.h: The test for final weighted transitions involves
- target_index == 1 instead of weight == INFINITE_WEIGHT
-
- (I wish I remembered why this was changed)
-
-2011-07-20 07:44 hardwick
-
- * hfst-ol.h: Fixed bug involving bytewise casting of longs to
- floats (I misunderstood
- what static_cast really does I guess).
-
-2011-05-24 10:19 hardwick
-
- * hfst-ol.cc: fread returns element count, not byte count
-
-2011-05-09 19:22 mie
-
- * configure.ac, main.cc: duplicate definitions
-
-2011-05-09 19:04 mie
-
- * hfst-ol.cc, main.cc: fix msvc problems
-
-2011-04-25 14:04 mie
-
- * hfst-ol.cc: check fread return value as advised by gcc
-
-2011-04-20 13:29 hardwick
-
- * ospell.cc: Removed unnecessary test
-
-2011-04-20 13:12 hardwick
-
- * hfst-ol.cc, hfst-ol.h, main.cc, ospell.cc, ospell.h: Understand
- hfst3 headers, don't demand weightedness at header-reading stage
-
-2011-03-03 10:45 moshagen
-
- * .: Ignore (autotools-)generated files.
-
-2011-02-22 01:20 mie
-
- * configure.ac: use hfstospell library name for compatibility or
- whatever
-
-2011-02-19 14:19 mie
-
- * Makefile, configure.ac, hfst-ol.cc, hfst-ol.h, ospell.cc: MSVC
- fixes:
- * include <string> when using strinG
- * use boolean operators instead of aliases?
-
-2011-02-03 01:07 mie
-
- * Makefile, Makefile.am, configure.ac, hfstospell.pc.in: add
- pkgconfig stuff
-
-2011-02-03 00:36 mie
-
- * AUTHORS, ChangeLog, INSTALL, Makefile, Makefile.am, NEWS,
- autogen.sh, configure.ac: autoconfiscate :-)
-
-2010-11-29 00:18 mie
-
- * Makefile, README, hfst-ol.cc, hfst-ol.h, ospell.cc, ospell.h: Add
- licences everywhere for release
-
-2010-11-07 00:29 mie
-
- * Makefile: make directories that do not exist
-
-2010-11-07 00:24 mie
-
- * Makefile: Install to destdir
-
-2010-11-02 19:35 moshagen
-
- * ., test/editdist.py: Ignore compiled libraries.
-
-2010-11-02 18:55 mie
-
- * Makefile: fix missing dash in mac dylib magic
-
-2010-11-02 18:44 mie
-
- * ospell.cc: Silently ignore if empty labels are missing
-
-2010-11-02 15:45 mie
-
- * Makefile: Make dynamic or share libraries
-
-2010-10-12 14:59 hardwick
-
- * test/editdist.py: Added some fancy rule autodetection
-
-2010-10-12 14:02 hardwick
-
- * test/editdist.py: Fixes to input format handling
-
-2010-10-12 10:22 hardwick
-
- * test/editdist.py: New input file syntax
-
-2010-09-07 14:22 hardwick
-
- * Makefile, hfst-ol.h, ospell.cc: More speed improvements
-
-2010-09-07 12:55 hardwick
-
- * Makefile, ospell.cc: Various optimizations
-
-2010-09-07 09:06 hardwick
-
- * hfst-ol.cc, hfst-ol.h, ospell.cc, ospell.h: Critical bugfix,
- output now believed to be correct
-
-2010-08-30 19:34 hardwick
-
- * test/editdist.py: Diagnostics and info about expected transducer
- model in test script
-
-2010-08-12 20:26 hardwick
-
- * test/editdist.py: More helpful help message for test script
-
-2010-08-12 10:52 moshagen
-
- * test/editdist.py: Should be executable.
-
-2010-08-12 01:15 hardwick
-
- * test/editdist.py: Support for OTHER symbol in test script
-
-2010-08-11 22:31 hardwick
-
- * Makefile, main.cc: Added profiler flag to debug compilation
- target and made demo utility exit
- on empty lines
-
-2010-08-11 22:03 hardwick
-
- * ospell.h: Trivial cosmetic changes
-
-2010-08-11 21:57 hardwick
-
- * hfst-ol.cc, hfst-ol.h: More header cleanup
-
-2010-08-11 21:49 hardwick
-
- * hfst-ol.cc, hfst-ol.h: Renamed variable
-
-2010-08-11 21:47 hardwick
-
- * hfst-ol.cc, hfst-ol.h, ospell.h: Misc. code cleanup and memory
- savings
-
-2010-08-11 20:43 hardwick
-
- * hfst-ol.h: Free memory holding transducer data after parsing
-
-2010-08-11 19:11 hardwick
-
- * ospell.cc, ospell.h: Some more const sprinkling
-
-2010-08-11 19:04 hardwick
-
- * hfst-ol.h, main.cc, ospell.h: Misc. nonfunctional cleanup
-
-2010-08-11 18:15 hardwick
-
- * test/editdist.py: Hack to make the test script handle unicode
-
-2010-08-11 18:10 hardwick
-
- * test/editdist.py: Added character swaps to edit distance script.
- You have to enable them with
- the -s flag - they generate A(A-1)*D new states and twice that
- many
- transitions, where A is the size of the alphabet and D is the
- edit distance.
- Pretty expensive. Is there a better way?
-
-2010-08-11 17:37 hardwick
-
- * test/editdist.py: Improvements to editdist.py - see
- test/editdist.py --help
-
-2010-08-11 16:52 hardwick
-
- * test/editdist.py: Put 1.0 weights on the test generator script
-
-2010-08-11 16:44 hardwick
-
- * test/editdist.py: Minor enhancement to test script
-
-2010-08-10 12:27 hardwick
-
- * main.cc, ospell.cc, ospell.h: Added helpful runtime error for
- alphabet translation problems, updated
- demo utility to make use of it
-
-2010-08-10 09:54 hardwick
-
- * hfst-ol.cc, hfst-ol.h: Better checking of read operations, added
- relevant exceptions
-
-2010-08-09 22:25 hardwick
-
- * README: Made example formatting in README more consistent - I may
- have broken
- Tommi's commit a bit, but I think it's ok now...
-
-2010-08-09 22:21 hardwick
-
- * README, main.cc: Minor improvement to demo
-
-2010-08-09 22:02 mie
-
- * Makefile, README: Static lib and fixes to xamples in readme
-
-2010-08-09 22:00 hardwick
-
- * ospell.h: Added comment
-
-2010-08-09 20:44 hardwick
-
- * ospell.h: Reversed previous commit, which did the opposite of
- what the commit message
- said it would. Committer will go to bed now...
-
-2010-08-09 20:41 hardwick
-
- * ospell.h: Return results in reverse order by weight, ie. in order
- of quality
- (instead of the opposite)
-
-2010-08-09 20:04 hardwick
-
- * ospell.h: Removed obsolete dependency on cassert
-
-2010-08-09 20:03 hardwick
-
- * hfst-ol.h: Fixed some comments
-
-2010-08-09 20:01 hardwick
-
- * ospell.cc: One more formatting fix
-
-2010-08-09 20:00 hardwick
-
- * ospell.cc: Fixed typo in comment
-
-2010-08-09 19:58 hardwick
-
- * main.cc, ospell.h: Moved getopt dependency from ospell.h to the
- demo utility proper
-
-2010-08-09 19:56 hardwick
-
- * hfst-ol.cc, hfst-ol.h, ospell.cc, ospell.h: Formatting
- improvements
-
-2010-08-09 15:43 hardwick
-
- * README, main.cc, ospell.cc, ospell.h: Introduced an exception for
- handling alphabet translation failure,
- fixed typo in help string, updated README
-
-2010-08-09 14:26 hardwick
-
- * main.cc, ospell.cc: Made some changes to correction-storing data
- structures to make sure each
- correction string only appears once
-
-2010-08-09 13:37 hardwick
-
- * hfst-ol.cc, hfst-ol.h, ospell.h, test/editdist.py, test/test.txt:
- Fatal bug(s) fixed, (more) correct flag diacritic functionality
-
-2010-08-06 12:10 hardwick
-
- * test/editdist.py, test/french.hfst, test/test2.txt: New test
- script
-
-2010-08-06 12:05 hardwick
-
- * ospell.cc: Fixed typo
-
-2010-08-06 11:59 hardwick
-
- * main.cc, ospell.cc, test/test.txt: Fixed a braindead bug that
- subtly broke everything, this should make
- some code redundant
-
-2010-08-02 13:46 hardwick
-
- * ospell.cc, ospell.h: A way to handle flag diacritics
-
-2010-07-08 15:33 hardwick
-
- * Makefile: Trivial Makefile fix for commandline tester
-
-2010-07-08 15:26 hardwick
-
- * README, hfst-ol.cc, hfst-ol.h: Replaced some ungracious exits
- with exceptions and made small change to README
-
-2010-07-08 15:15 hardwick
-
- * README, hfst-ol.cc, hfst-ol.h, main.cc, ospell.cc, ospell.h:
- Added README and some fixes
-
-2010-07-08 14:46 hardwick
-
- * hfst-ol.cc, hfst-ol.h, main.cc, ospell.cc, ospell.h: Implemented
- spellchecking and correction library functions; documentation,
- proper packaging and esp. functioning flag diacritics still to be
- done.
-
-2010-07-05 11:05 hardwick
-
- * ospell.h: Temporarily de-autotooled ospell
-
-2010-06-30 09:39 hardwick
-
- * ospell.cc, ospell.h: Incorporated queue in speller proper
-
-2010-06-29 06:31 hardwick
-
- * Makefile, hfst-ol.cc, hfst-ol.h, ospell.cc, ospell.h,
- test/french-symbols.txt, test/french.hfst.ol, test/test.hfst.ol,
- test/test2.hfst.ol: Fixed behaviour, added weightedness
- scaffolding
-
-2010-06-22 14:38 moshagen
-
- * main.cc: Corrected typo.
-
-2010-06-21 20:26 moshagen
-
- * .: Ignore generated binary.
-
-2010-06-21 17:29 hardwick
-
- * ., Makefile, hfst-ol.cc, hfst-ol.h, main.cc, ospell.cc, ospell.h,
- test, test/french-symbols.txt, test/french.hfst,
- test/french.hfst.ol, test/test.hfst.ol, test/test.txt,
- test/test2.hfst.ol, test/test2.txt: Initial commit of
- hfst-ospell.
- Basic functionality including OTHER symbol (@?@) and runtime
- alphabet
- translation is present; weighted transducers (probably to be the
- only option)
- and flag diacritic states for the mutator and lexicon
- forthcoming.
-
+9562377 (HEAD -> master, origin/master, origin/HEAD) Fixes for big endian conversions; Make use of AC_C_BIGENDIAN and WORDS_BIGENDIAN; UCHAR_MAX should be +1 to allow hitting index 255
+bf79d25 Add returns
+8573403 Merge branch 'big_endian_compatibility_suggestion' In hopes that this approach to fixing build problems on the problematic architectures will get tested for Debian builds. This is experimental, but at least shouldn't break anything - feel free to revert if unhelpful.
+b756b2e Killed an error. Don't know whether it was the correct solution - please review.
+3ca7756 max version for tinyxml
+662f79c (origin/big_endian_compatibility_suggestion) Forgot to actually assign header length to a variable when reading raw bytes
+abed65f A suggestion for big-endian compatibility
+43f5dc6 (tag: v0.4.2) Release 0.4.2.
+e9cc1da Prevent segfaults from no-errormodel.sh test This also makes the test succeed rather than XLFAIL, so removed it from XFAIL_TESTS (why not?)
+6d359c0 Remove unused $(HFST_FLAGS) variable that may be leading to some test failures
+b31fec1 markdown
+9fa49a2 not supporting default autostuff like promised...
+c53090d Merge branch 'master' of github.com:hfst/hfst-ospell
+4b07bf5 experimental ci
+f582bac (tag: v0.4.1) Release 0.4.1.
+29649b1 Set gitignores.
+31e6842 Bump patch version since revision has decreased
+d2fb71e set_time_cutoff(6.0)
+08bc7a9 When upping to 16 variants, also make room for all 16 variants
+080042a Added section header, correct tool name to man page.
+a95e4aa 0.4.0
+2565275 Release 0.4.0.
+3e31ef1 Update help message, copyright extends to 2016.
+6ca58bc Make no-errormodel.sh an xfail test.
+047e9d9 Only call clock() every millionth time for speed
+ae8a660 Fix time cutoff handling
+b222053 Update man pages
+96e17f7 Update man page
+d92e8a7 If the input token had any upper case, also try a first-upper variant of it
+5fee947 Special case nuvviDspeller
+b5946d9 Bugfix: class -> struct (libvoikko complained under OSX, and err’ed out).
+929f926 If we are looking for suggestions, don't use the cache
+04e4843 Fix a couple of typos in argument list and verbose prints. Add a missing statement that actually sets the time cutoff for the speller.
+cf8ee5c Fix a typo in wide string conversion function.
+30940be Time cutoff option during correction for library and demo app
+77bed9b Apply Brendan Molloy's alignment patch for ARM CPUs & reindent to namespace=0
+da6fcc5 Propagate first-upper and all-upper to suggestions
+b57d248 Add option --enable-hfst-ospell-office to configure (defaults to yes). Add support for using libarchive2 if libarchive3 is not available.
+26a4e37 Add --verbatim to disable the variations
+ece3455 Don't use UTF-8 length for UTF-16 buffer
+3b07200 I meant 20480
+df80db9 Use ICU to determine what's alphanumeric - should fix https://github.com/TinoDidriksen/spellers/issues/1
+bc4b3a2 If pkg-config fails, fall back on icu-config - blame Ubuntu 12.04
+3266e7b Silly mistake
+5226fb2 Use ICU instead
+be5f73a Use wide chars during case folding; Set locale
+90d7fcb Fix typo that broke UNKNOWN handling the the error source
+0ff3b88 Fix bug in novel symbol tokenization
+791ca3e Bounds check table accesses Now that new symbols from input can be added to the lexicon's alphabet, it's possible that during checking we try to access beyond the lexicon's table boundaries, so that needs to be prevented.
+0865615 CentOS 6 fixes: Move AC_INIT up to prevent _m4_divert_diversion error; Use older ax_check_compile_flag.m4
+833e6f6 autoconf-archive doesn't exist on RPM platforms, so bundle ax_check_compile_flag.m4
+e59a06e Determine and use highest supported C++ standard
+f8d3893 Reinstate legacy function Transducer::lookup() due to Voikko's use of it
+b645b28 Unknown and identity arcs can now appear both in the lexicon and error model This is a rather large overhaul of everything having to do with symbols, and causes an approximately 8% slowdown at this time. Some more efforts will be made to bring that down..
+28cd37b Make weight checking methods const
+d76526a Correct nbest handling for cached results too + remove extranous comparison class
+a10455e Special-case limit checking for nbest
+568d606 Correct limit setting
+22bd0b5 Remove debugging print
+ed57354 Use a list to emulate a sorted double-ended weight queue
+effe39d Removed keyword 'typename', it caused GCC with C++03 to bark.
+57891b8 Drop down to boring ol' C++03
+524f259 Fix declaration order and unicode issues
+1feb23e Add --default-weight, --outputfile, pair specification syntax for regex output
+d40e733 Add a --regex mode to produce a large regular expression with weighted replace rules
+612aa05 Add a first-symbol cache and do some reorg
+6a94a9b Fix -Wall -Wextra issues
+2fcd06e New protocol: Let client choose max suggestions, input must now match "^(\d+) (.+)$"
+dcb9474 Search up the normalization list until something gives suggestions
+ad14157 Disable log2 max-dist and instead set hard limit due to FST weights
+e7cfbc6 Tokenization is done by the host, so don't second guess
+e484ef1 Add hfst-ospell-office helper so the algorithm is in one place, instead of in every frontend
+14c72ca Arguably clearer scheme for combining limiting options
+0c5eeec Reinstate legacy spelling, slightly adding to but not altering interface
+4d1fa2b Add option --beam=W for restricting the search to a margin above the optimum. Any combination of n-best, weight limit and beam is possible, all restrict each other.
+29180f3 Use vector instead of deque now that breadth search isn't used for anything
+ea15127 Be a bit more systematic about search-limiting behaviour, also fail earlier, this gives a small speed improvement
+a9d9e55 Make argument parsing further agree with documentation, reinstating -X
+e2b9b7e nbest wasn't producing any runtime benefit due to incorrect comparison
+94064b5 Reinstate the : lost from the -n arg in getopt_long()
+b164f2b Change max_weight to max-weight in the help string and argument name
+48d6d15 Check that maxweight is positively set before using it, default is negative
+a782763 Version 0.3.1 - this introduces API/linking changes Support limiting the correction search to a maximum weight. The hfst-ospell command supports this via --max_weight / -w.
+76d9052 Make -n (max n corrections) work again
+4d6f7dd Have separate switches for each XML mangling library
+d39d419 Updated hfst-ospell man page.
+1f38ba3 Just error out when missing XML
+d7058e8 Allow selecting real-word errors
+8866483 Missing stdarg reqd in >gcc 4.8 or somesuch
+561760e Added configure and Makefile files for Windows compilation.
+f19070d A tiny performance improvement (about 5%, this is not what was discussed on irc, but every little helps)
+a674152 Fix missing newline in verbose message
+ba22ebb This is how I got the analysing speller to work
+f884489 Simple example of analysis spelling logic
+4019232 can analyse if speller or sugger
+f621a37 Fix bug #239 When correcting, keep track of mutator's output / lexicon's input; when analysing, keep track of lexicon's output
+bfd40d9 Fix bug #238 Now that alphabets are translated even if the error source contains symbols not found in the lexicon, the lexicon needs to be prepared to be asked whether there are transitions with NO_SYMBOL
+f7b6dcc documentation front page
+0a17be0 Dist changes, 0.3.0 API stable though pending other fixes for .1
+03d40cc Some beginnings for a documented stable API.
+a8dc221 This has been missing all along?
+7f70384 Allow setting max suggestions from command line tool
+edc36e7 Remove legacy, allow setting suggestion limits
+a61dcd2 Tentative support for analyse-as-you-error-correct
+8847e56 Errmodel with out of lexicon alphabet fails
+f702e28 Better test case for analyser
+c96a2e8 Add tests for analysing speller and missing errmodels
+104a24d Analysis interface and implementation for two-tape language models. The suggestion engine still erroneously gives the analysis side of suggestion LM though.
+75fc585 Now printing and reading to/from console works in hfst-ospell is it is compiled with -DWINDOWS.
+94031ca Added a rule for man page creation in Makefile.am.
+e293f96 Added a test for an empty zhfst file. It should fail, but presently hfst-ospell behaves as if all is ok, just that it isn't able to correct anything.
+add8ddd Corrected return value to what it should be for a SKIP result, which I assume was intended.
+09ea12d Added files to the distro.
+de621bd Added test to check whether error models with extra chars are correctly handled. The test passes with revision 3793 of ospell.cc, and fails with revision 3729.
+4860369 Silently ignore non-lexicon symbols in the error source, translating them to NO_SYMBOL on the output side.
+9e8ae05 revert
+8e74395 What happens if I do this?
+41bb8cf Moving implementations?
+9629091 Install men
+5a0fbdb A man page for debian's lint
+f4850b0 Probably libarchive deprecation will go away with different function name here too
+f76b1fa Move some of the code away from headers
+db0c56d some old changes on work desktop?
+16511d0 Ensure other potentially empty tags for libxmlpp
+832f9d0 Bumped version for tinyxml2 check to get lengthful Parse by Børre Gaup
+201acd8 unarchive to mem and tinyxml2 xml length fixes from Harri Pitkänen
+220e6cb Reverted the patch by Børre to clean the xml off of random byte noise after the end of the xml document. The issue was caused by zip compression incompatibilities between different plattforms, and the solution (for now) is to just skip compression altogether, using 'zip -Z store' instead when building the zhfst files. This creates a zip file readable everywhere (but without compression).
+c57f0bd Test for trailing spaces in xml structure
+ad60602 Patch by Børre Gaup to clean corrupted xml data returned from Libarchive when reading xml data from a zhfst file stored in RAM:
+b0ba291 Whitespace changes to make the file easier to my eyes, comments. Added tests for empty locale and empty title elements.
+42da63d Data and shell scripts to test empty titles and empty locale.
+6b5145f Check for empty locale node.
+0838caa Clean and ignore index.xml.
+26e6b99 One more test for empty data.
+2b1089f Added newline before error cause.
+a440da7 Empty descriptions will throw (there might be others left)
+58047b3 index.xml with empty descriptions
+2f93bcc Test.strings shall not be deleted when cleaning
+c4adeb0 Added check for the availability of pkg-config - without it configuration will fail in subtle but bad ways.
+cd24885 Document for next release candidate
+3365813 Missing model attribute parsing for errm
+5ca88f6 Use Elements instead of Nodes and other such fixes
+bd8eada Use automake conditionals to avoid pkg-config linking to libraries that are not in configure's code paths
+0322db3 update configuration for that
+2153668 Add tinyxml2 versions of XML parsing
+658ff68 conference demos as configure option
+b2ff3fb stats -> status
+e20d008 Allow to end correct correction
+c16d604 Merge lookup() from devbranch
+0761df3 This should be useful
+fb1c6c1 Fix the output tsv a bit more
+299b94d Slight udpate for new measure ments
+44f6276 Few starts
+f25c609 Allow selection of xml backend
+4be4b96 Switch <3 into >3 because it looks nicer.
+5d80f3f strndup() fixed by Tommi.
+e11a913 Pending proper understanding of why strndup is getting defined several times over, let's always use hfst_strndup() instead.
+248d05d Add include guard to custom strndup so it doesn't get compiled more than once
+af41cbf Move our custom strndup to ospell.h so it will be seen by all compilation units that need it
+bd813b0 Renamed the hfstospell package and variables to what they used to be, in case that can help solve a build issue with libvoikko+hfst.
+83b1a09 Wrap and load
+f03ef00 versioninfo 3:1:1
+d70a9e6 Preparing for 0.2.3 release.
+c8ba084 Changed version number to 0.2.3, and renamed the package and tool name to 'hfst-ospell', since that is what the command-line tool is actually called, and it is consistent with the rest of the hfst tool names.
+3d40399 Should fix #176 Flag state from the stack back was getting clobbered in some situations. Now we restore flag state after modifying it for new nodes.
+ac10ddd Fix dist stuff, wrap.
+3fa2b5d Use pkg-config instead of autoconf to check libarchive
+7e422b7 Prepare files for 0.2.2
+89a4a99 Fix few leaks
+44b313c Fixes to [#165] by Harri Pitkänen
+3ad822b Made some changes to depth-first searching to prevent just-added nodes getting removed. This appears fo fix a very long-standing and serious bug dating from r2763.
+36fa55c When data is in memory, allocate & copy it in ospell rather than expect caller to keep track of it (which it usually doesn't)
+8196d8a Add access to metadata
+1da34c4 When initialised from memory, don't assume responsibility for freeing it
+01a43fd Somewhat experimental (no checking) way of saving memory by ignoring alignment issues (4-5 x memory savings)
+db95a32 Remove leftover agenda variable
+2ee0efc Add stuff for survey article
+a439eaf Revert to breadth-first searching pending bugfix
+8166a16 Search depth-first by preference for hope
+876ebbf Forgot some important checks that we actually want to limit the results
+b1b6b24 Enforce an n-best limit for continuing the search, just for breadth-first for now
+78f2345 Don’t just break on empty lines
+ab3f591 Switch order of errmodels and acceptors in legacy read
+5baaaed set spelling caps on legacy read
+7c60d3e Test fallback
+a1b1e35 The VPATH approach to the test shell scripts was wrong. Now everything is working as it should.
+fcef585 Enabling VPATH building for 'make check'.
+f8d7a54 Whitespace change only in configure.ac. More generated files to ignore.
+b6f6482 Added some whitespace to ease readability. Replaced pattern rules with suffix rules to make automake happy.
+3027315 Fix compilation without xml, as suggested by Harri Pitkänen on libvoikko-devel
+9e2503d do not close unopened files, handle legacy fallback reading errors
+9e3739d print details of xml parsing errors
+5791425 Add the very basic test suite
+e39e40b Throw exception if reading a zip without existing zip file.
+3ebb0aa Optionalise the libraries
+4ab1a3a Update for automake-1.12 and AM_PROG_AR
+99f80d4 kill everyone if windows linebreaks
+9459c17 Fix documentation and set for 0.2.1 release
+ab90ec9 free more things and stuff
+bca66ba Move Xml metadata parsing and storing into own file and class
+686d305 Increment *raw when reading bools
+5e91e84 Added utility function for iterating c strings in raw memory, use it in every branch of symbol reading
+246c2b2 Fixed problems having to do with reading strings from a transducer in raw memory.
+15b3184 fix the tmpdir'd version again
+bc2d9af Version that extracts zhfst to memory iff it fits on one throw
+69ff631 Added raw memory constructors and table readers for slurping in transducers from char *
+4772d28 Missing files lol
+77c23cf ...and in that case the final identity states also are greater by one.
+58e9512 When avoiding initial edits, there needs to be an extra inital state before we do any edits. So add the value of options.no_initial to the state range loop.
+433e917 Commented out hfst-ospell-cicling from Makefile.
+73f8871 Comment out missing cicling target
+b6a3317 "else if" instead of incorrect "if" in option handling
+620392f --no-string-initial should have transitions to state 1, not 0
+9f8cf3c Don't exit on empty lines
+2ab0594 Don't print unhelpful warnings
+4fb22d1 Silly default for minimum edit
+2e58cd8 fallback_spell()
+ca9a3b0 --no-string-initial-correction
+76c87fe Corrected eliminations (hopefully), added --minimum-edit option
+713aff2 Profiled version for fsmnlp-2012 measurements§
+999ca00 Don't skip state numbers when certain options are turned off (and don't print debugging lines)
+e864859 Corrections to redundancy elimination
+49b4394 redundancy elimination was being performed one state too late (too little)
+bb93e12 Add option to disable redundancy elimination
+731db86 Initial support for redundancy elimination (refuse to do insertion after deletion or deletion after insertion)
+83f85ce Bugfix: identity transitions were being forgotten in the last edit state
+9ca92d4 Enforce @_UNKNOWN_SYMBOL_@ instead of @?@, which users didn't know about
+2b001e6 Fixed one remaining UTF-8 stderr printing bug.
+aee7a2a Fixed bug with the newline character not being stripped from excluded symbols
+92ab53a Lines that start with ## are comments
+fd5f560 Updated help message & added exclusion of symbols by prepending a ~
+45442ab Wrap stderr with a utf-8 codec so we can print non-ascii symbols when verbose
+5c27eca Write alphabet with weights when verbose
+f9b426f Order of preference of alphabet definition is now configfile - commandline - transducer. If configfile gives a weight after a tab for symbols, they are used additively for all edits involving those symbols.
+6629f88 Some clarifying comments
+33e3c95 Rescue identities from being considered substitutions
+4ce7bb0 utf-8 -decode the user-specified transitions from a conf file (so easy to forget one of these...)
+42ec180 use same includedir in pc as makefile
+40afbca don't declare strndup in public headers
+b5a6c7a make ol exceptions in hfst_ol namespace, provide stdexception style what()
+3a58fbb Add verbose, quiet
+c821e19 * parse all of the metadata if possible * use c++ ``struct''s for metadata
+3f5bb8b * Use temporary filenames by tmpsnam * Do not delete Transducers in data structures since it will segfault all enchant-based applications in dtor
+5449a28 libhfst-style exception macros and some more informative messages
+e9c551d Document dependencies
+105b6e2 mac os x fixes: * strndup * libarchive installed without headers
+a2b4ea5 Preliminary zhfst support
+d7bf3e8 Example from: http://norvig.com/spell-correct.html and similar tests
+a2b9d15 The test for final weighted transitions involves target_index == 1 instead of weight == INFINITE_WEIGHT
+3cb5d8a Fixed bug involving bytewise casting of longs to floats (I misunderstood what static_cast really does I guess).
+7ff6a13 fread returns element count, not byte count
+9c01f47 duplicate definitions
+a433823 fix msvc problems
+4f2c186 check fread return value as advised by gcc
+b324b05 Removed unnecessary test
+9f801d0 Understand hfst3 headers, don't demand weightedness at header-reading stage
+693eaea use hfstospell library name for compatibility or whatever
+c0a2985 MSVC fixes: * include <string> when using strinG * use boolean operators instead of aliases?
+4f16033 add pkgconfig stuff
+5ebbfec autoconfiscate :-)
+37ad031 Add licences everywhere for release
+4e7c098 make directories that do not exist
+5c450f7 Install to destdir
+a93f59d Ignore compiled libraries.
+7e0a5b4 fix missing dash in mac dylib magic
+3bd799d Silently ignore if empty labels are missing
+b6bc407 Make dynamic or share libraries
+3767ad9 Added some fancy rule autodetection
+5b87174 Fixes to input format handling
+9a7418c New input file syntax
+fd3eb19 More speed improvements
+a78427f Various optimizations
+dff1eab Critical bugfix, output now believed to be correct
+c227d79 Diagnostics and info about expected transducer model in test script
+196ff02 More helpful help message for test script
+1d1acfd Should be executable.
+c50bb9b Support for OTHER symbol in test script
+44e6e9c Added profiler flag to debug compilation target and made demo utility exit on empty lines
+2588216 Trivial cosmetic changes
+a059914 More header cleanup
+9c551c6 Renamed variable
+7035738 Misc. code cleanup and memory savings
+7339e8c Free memory holding transducer data after parsing
+915ef1d Some more const sprinkling
+59040cc Misc. nonfunctional cleanup
+5852517 Hack to make the test script handle unicode
+915a6e2 Added character swaps to edit distance script. You have to enable them with the -s flag - they generate A(A-1)*D new states and twice that many transitions, where A is the size of the alphabet and D is the edit distance. Pretty expensive. Is there a better way?
+f77baa8 Improvements to editdist.py - see test/editdist.py --help
+fa61c7d Put 1.0 weights on the test generator script
+2bea19e Minor enhancement to test script
+fb6b65f Added helpful runtime error for alphabet translation problems, updated demo utility to make use of it
+31a3561 Better checking of read operations, added relevant exceptions
+397a4b5 Made example formatting in README more consistent - I may have broken Tommi's commit a bit, but I think it's ok now...
+2e933a8 Minor improvement to demo
+a7cf1f3 Static lib and fixes to xamples in readme
+161d40d Added comment
+b383a42 Reversed previous commit, which did the opposite of what the commit message said it would. Committer will go to bed now...
+a4877dc Return results in reverse order by weight, ie. in order of quality (instead of the opposite)
+5827928 Removed obsolete dependency on cassert
+d8f388e Fixed some comments
+5d1d58a One more formatting fix
+403d7ed Fixed typo in comment
+d49e37d Moved getopt dependency from ospell.h to the demo utility proper
+5e1b2f1 Formatting improvements
+f11ed65 Introduced an exception for handling alphabet translation failure, fixed typo in help string, updated README
+73647b3 Made some changes to correction-storing data structures to make sure each correction string only appears once
+25207f7 Fatal bug(s) fixed, (more) correct flag diacritic functionality
+57441ea New test script
+82620ee Fixed typo
+778d477 Fixed a braindead bug that subtly broke everything, this should make some code redundant
+2a3ffcf A way to handle flag diacritics
+47d35eb Trivial Makefile fix for commandline tester
+564e7f2 Replaced some ungracious exits with exceptions and made small change to README
+d92e82c Added README and some fixes
+bd2c6c3 Implemented spellchecking and correction library functions; documentation, proper packaging and esp. functioning flag diacritics still to be done.
+d3ed0b7 Temporarily de-autotooled ospell
+7344a8e Incorporated queue in speller proper
+dbc3bae Fixed behaviour, added weightedness scaffolding
+e4851c1 Corrected typo.
+9df6f72 Initial commit of hfst-ospell. Basic functionality including OTHER symbol (@?@) and runtime alphabet translation is present; weighted transducers (probably to be the only option) and flag diacritic states for the mutator and lexicon forthcoming.
diff --git a/Makefile.am b/Makefile.am
index c5d7225..9423c5c 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -54,7 +54,7 @@ endif
libhfstospell_la_SOURCES=hfst-ol.cc ospell.cc \
ZHfstOspeller.cc ZHfstOspellerXmlMetadata.cc
libhfstospell_la_CXXFLAGS=$(AM_CXXFLAGS) $(CXXFLAGS) $(PKG_CXXFLAGS)
-libhfstospell_la_LDFLAGS=-no-undefined -version-info 5:0:0 \
+libhfstospell_la_LDFLAGS=-no-undefined -version-info 8:0:0 \
$(PKG_LIBS)
# link sample program against library here
@@ -65,7 +65,7 @@ hfst_ospell_CXXFLAGS=$(AM_CXXFLAGS) $(CXXFLAGS) \
if HFST_OSPELL_OFFICE
-hfst_ospell_office_SOURCES=office.cpp
+hfst_ospell_office_SOURCES=office.cc
hfst_ospell_office_LDADD=libhfstospell.la
hfst_ospell_office_CXXFLAGS=$(AM_CXXFLAGS) $(CXXFLAGS) $(PKG_CXXFLAGS)
@@ -110,104 +110,19 @@ endif # EXTRA_DEMOS
# install headers for library in hfst's includedir
include_HEADERS=hfst-ol.h ospell.h ol-exceptions.h \
- ZHfstOspeller.h ZHfstOspellerXmlMetadata.h
+ ZHfstOspeller.h ZHfstOspellerXmlMetadata.h \
+ hfstol-stdafx.h
# pkgconfig
pkgconfigdir=$(libdir)/pkgconfig
pkgconfig_DATA=hfstospell.pc
# tests
-if CAN_TEST
-TXTS=acceptor.basic.txt analyser.default.txt \
- errmodel.basic.txt errmodel.extrachars.txt errmodel.edit1.txt
-check_DATA=speller_basic.zhfst empty_descriptions.zhfst \
- empty_titles.zhfst empty_locale.zhfst \
- trailing_spaces.zhfst \
- acceptor.basic.hfst errmodel.basic.hfst \
- errmodel.extrachars.hfst bad_errormodel.zhfst \
- speller_analyser.zhfst no_errormodel.zhfst \
- speller_edit1.zhfst
-# Actual test scripts:
-TESTS=basic-zhfst.sh basic-edit1.sh \
- empty-descriptions.sh empty-titles.sh empty-locale.sh \
- trailing-spaces.sh bad-errormodel.sh empty-zhfst.sh \
- analyse-spell.sh no-errormodel.sh
-XFAIL_TESTS=empty-descriptions.sh empty-titles.sh empty-locale.sh empty-zhfst.sh no-errormodel.sh
-EXTRA_DIST=$(TXTS) $(TESTS) $(man1_MANS)\
- basic_test.xml empty_descriptions.xml empty_titles.xml \
- empty_locale.xml trailing_spaces.xml no_errmodel.xml \
- test.strings
-
-clean-local:
- -rm -rf $(check_DATA) index.xml
-endif # CAN_TEST
-
-# N.B. Do not parallel test, race condition exists
-# N.B. Do not parallel test, race condition exists
-empty_descriptions.zhfst: acceptor.basic.hfst errmodel.basic.hfst empty_descriptions.xml
- cp -f $(srcdir)/empty_descriptions.xml index.xml
- cp -f acceptor.basic.hfst acceptor.default.hfst
- cp -f errmodel.basic.hfst errmodel.default.hfst
- $(ZIP) $(ZIPFLAGS) $@ acceptor.default.hfst errmodel.default.hfst index.xml
- -rm -f index.xml acceptor.default.hfst errmodel.default.hfst
-
-empty_titles.zhfst: acceptor.basic.hfst errmodel.basic.hfst empty_titles.xml
- cp -f $(srcdir)/empty_titles.xml index.xml
- cp -f acceptor.basic.hfst acceptor.default.hfst
- cp -f errmodel.basic.hfst errmodel.default.hfst
- $(ZIP) $(ZIPFLAGS) $@ acceptor.default.hfst errmodel.default.hfst index.xml
- -rm -f index.xml acceptor.default.hfst errmodel.default.hfst
-
-empty_locale.zhfst: acceptor.basic.hfst errmodel.basic.hfst empty_locale.xml
- cp -f $(srcdir)/empty_locale.xml index.xml
- cp -f acceptor.basic.hfst acceptor.default.hfst
- cp -f errmodel.basic.hfst errmodel.default.hfst
- $(ZIP) $(ZIPFLAGS) $@ acceptor.default.hfst errmodel.default.hfst index.xml
- -rm -f index.xml acceptor.default.hfst errmodel.default.hfst
-
-trailing_spaces.zhfst: acceptor.basic.hfst errmodel.basic.hfst trailing_spaces.xml
- cp -f $(srcdir)/trailing_spaces.xml index.xml
- cp -f acceptor.basic.hfst acceptor.default.hfst
- cp -f errmodel.basic.hfst errmodel.default.hfst
- $(ZIP) $(ZIPFLAGS) $@ acceptor.default.hfst errmodel.default.hfst index.xml
- -rm -f index.xml acceptor.default.hfst errmodel.default.hfst
-
-speller_edit1.zhfst: acceptor.basic.hfst errmodel.edit1.hfst basic_test.xml
- cp $(srcdir)/basic_test.xml index.xml
- cp -f acceptor.basic.hfst acceptor.default.hfst
- cp -f errmodel.edit1.hfst errmodel.default.hfst
- $(ZIP) $(ZIPFLAGS) $@ acceptor.default.hfst errmodel.default.hfst index.xml
- -rm -f index.xml acceptor.default.hfst errmodel.default.hfst
-
-speller_basic.zhfst: acceptor.basic.hfst errmodel.basic.hfst basic_test.xml
- cp $(srcdir)/basic_test.xml index.xml
- cp -f acceptor.basic.hfst acceptor.default.hfst
- cp -f errmodel.basic.hfst errmodel.default.hfst
- $(ZIP) $(ZIPFLAGS) $@ acceptor.default.hfst errmodel.default.hfst index.xml
- -rm -f index.xml acceptor.default.hfst errmodel.default.hfst
-
-speller_analyser.zhfst: analyser.default.hfst errmodel.edit1.hfst basic_test.xml
- cp -f $(srcdir)/basic_test.xml index.xml
- cp -f analyser.default.hfst acceptor.default.hfst
- cp -f errmodel.edit1.hfst errmodel.default.hfst
- $(ZIP) $(ZIPFLAGS) $@ acceptor.default.hfst errmodel.default.hfst index.xml
- -rm -f index.xml acceptor.default.hfst errmodel.default.hfst
-
-bad_errormodel.zhfst: acceptor.basic.hfst errmodel.extrachars.hfst basic_test.xml
- cp -f $(srcdir)/basic_test.xml index.xml
- cp -f acceptor.basic.hfst acceptor.default.hfst
- cp -f errmodel.extrachars.hfst errmodel.default.hfst
- $(ZIP) $(ZIPFLAGS) $@ acceptor.default.hfst errmodel.default.hfst index.xml
- -rm -f index.xml acceptor.default.hfst errmodel.default.hfst
-
-no_errormodel.zhfst: acceptor.basic.hfst no_errmodel.xml
- cp -f $(srcdir)/no_errmodel.xml index.xml
- cp -f acceptor.basic.hfst acceptor.default.hfst
- $(ZIP) $(ZIPFLAGS) $@ acceptor.default.hfst index.xml
- -rm -f index.xml acceptor.default.hfst
-
-.txt.hfst:
- hfst-txt2fst -e "@0@" $(HFST_FLAGS) $< | hfst-fst2fst $(HFST_FLAGS) -f olw -o $@
+TESTS=tests/basic-zhfst.sh tests/basic-edit1.sh \
+ tests/empty-descriptions.sh tests/empty-titles.sh tests/empty-locale.sh \
+ tests/trailing-spaces.sh tests/bad-errormodel.sh tests/empty-zhfst.sh \
+ tests/analyse-spell.sh tests/no-errormodel.sh
+XFAIL_TESTS=tests/empty-descriptions.sh tests/empty-titles.sh tests/empty-locale.sh tests/empty-zhfst.sh
if CAN_DOXYGEN
doxygen:
diff --git a/NEWS b/NEWS
index f79eb36..4bb6ef5 100644
--- a/NEWS
+++ b/NEWS
@@ -2,7 +2,24 @@ NEWS for hfst-ospell
====================
This file contains all noteworthy changes in HFST-ospell development between
-releases. For full listing of changes see ChangeLog.
+releases. For full listing of changes see ChangeLog.
+
+Noteworthy changes in 0.4.3
+---------------------------
+
+* fixes for big endian conversions
+* use max version for tinyxml
+
+Noteworthy changes in 0.4.2
+---------------------------
+
+* small modifications to tests and documentation
+
+Noteworthy changes in 0.4.1
+---------------------------
+
+* set time cutoff to 6.0 seconds for ospell-office
+* minor bug fixes
Noteworthy changes in 0.4.0
---------------------------
diff --git a/README b/README
index b4a48c3..0d3fa3c 100644
--- a/README
+++ b/README
@@ -1,46 +1,40 @@
-.. -*- mode: rst -*-
-================================================
- Hfst-ospell library and toy commandline tester
-================================================
+# Hfst-ospell library and toy commandline tester
This is a minimal hfst optimized lookup format based spell checker library and
a demonstrational implementation of command line based spell checker. The
library is licenced under Apache licence version 2, other licences can be
obtained from University of Helsinki.
+[](https://travis-ci.org/hfst/hfst-ospell)
-Dependencies
-============
+## Dependencies
- - libxml++2
- - libarchive
+- libxml++2
+- libarchive
-Debian packages for dependencies
---------------------------------
+## Debian packages for dependencies
- - libxml++2-dev
- - libarchive-dev
+- libxml++2-dev
+- libarchive-dev
-Usage
-=====
+## Usage
-Usage in external programs::
+Usage in external programs:
#include <ospell.h>
-and compile your project with::
+and compile your project with:
$(pkg-config --cflags hfstospell)
-and link with::
+and link with:
$(pkg-config --libs hfstospell)
-Programming examples
---------------------
+## Programming examples
The library lives in a namespace called hfst_ol. Pass (weighted!) Transducer
-pointers to the Speller constructor, eg.::
+pointers to the Speller constructor, eg.:
FILE * error_source = fopen(error_filename, "r");
FILE * lexicon_file = fopen(lexicon_filename, "r");
@@ -62,7 +56,7 @@ pointers to the Speller constructor, eg.::
}
-And use the functions::
+And use the functions:
// returns true if line is found in lexicon
bool hfst_ol::Speller::check(char * line);
@@ -73,10 +67,9 @@ And use the functions::
to communicate with it. See main.cc for a concrete usage example.
-Command-line tool
------------------
+## Command-line tool
-Main.cc provides a demo utility with the following help message::
+Main.cc provides a demo utility with the following help message:
Usage: hfst-ospell [OPTIONS] ERRORSOURCE LEXICON
Run a composition of ERRORSOURCE and LEXICON on standard input and
@@ -91,11 +84,9 @@ Main.cc provides a demo utility with the following help message::
Report bugs to hfst-bugs at ling.helsinki.fi
-Use in real-world applications
-------------------------------
+# Use in real-world applications
The HFST based spellers can be used in real applications with help of
-`voikko <http://voikko.sf.net>`_. Voikko in turn can be used with enchant,
+[voikko](http://voikko.sf.net). Voikko in turn can be used with enchant,
libreoffice, and firefox.
-.. vim: set ft=rst:
diff --git a/ZHfstOspeller.cc b/ZHfstOspeller.cc
index 6f6386a..1bebaa3 100644
--- a/ZHfstOspeller.cc
+++ b/ZHfstOspeller.cc
@@ -41,60 +41,68 @@ namespace hfst_ol
{
#if HAVE_LIBARCHIVE
-#if ZHFST_EXTRACT_TO_MEM
-static
-char*
-extract_to_mem(archive* ar, archive_entry* entry, size_t* n)
- {
+inline std::string extract_to_mem(archive* ar, archive_entry* entry) {
size_t full_length = 0;
const struct stat* st = archive_entry_stat(entry);
size_t buffsize = st->st_size;
- char * buff = new char[buffsize];
- for (;;)
- {
- ssize_t curr = archive_read_data(ar, buff + full_length, buffsize - full_length);
- if (0 == curr)
- {
+ if (buffsize == 0) {
+ std::cerr << archive_error_string(ar) << std::endl;
+ throw ZHfstZipReadingError("Reading archive resulted in zero length entry");
+ }
+
+ std::string buff(buffsize, 0);
+ for (;;) {
+ ssize_t curr = archive_read_data(ar, &buff[0] + full_length, buffsize - full_length);
+ if (0 == curr) {
break;
- }
- else if (ARCHIVE_RETRY == curr)
- {
+ }
+ else if (ARCHIVE_RETRY == curr) {
continue;
- }
- else if (ARCHIVE_FAILED == curr)
- {
+ }
+ else if (ARCHIVE_FAILED == curr) {
throw ZHfstZipReadingError("Archive broken (ARCHIVE_FAILED)");
- }
- else if (curr < 0)
- {
+ }
+ else if (curr < 0) {
throw ZHfstZipReadingError("Archive broken...");
- }
- else
- {
+ }
+ else {
full_length += curr;
- }
- }
- *n = full_length;
+ }
+ }
+
+ if (full_length == 0) {
+ std::cerr << archive_error_string(ar) << std::endl;
+ throw ZHfstZipReadingError("Reading archive resulted in zero length");
+ }
+
return buff;
- }
-#endif
+}
-#if ZHFST_EXTRACT_TO_TMPDIR
-static
-char*
-extract_to_tmp_dir(archive* ar)
- {
+inline Transducer* transducer_to_mem(archive* ar, archive_entry* entry) {
+ std::string buff = extract_to_mem(ar, entry);
+ Transducer *trans = new Transducer(&buff[0]);
+ return trans;
+}
+
+inline char* extract_to_tmp_dir(archive* ar) {
char* rv = strdup("/tmp/zhfstospellXXXXXXXX");
int temp_fd = mkstemp(rv);
int rr = archive_read_data_into_fd(ar, temp_fd);
- if ((rr != ARCHIVE_EOF) && (rr != ARCHIVE_OK))
- {
+ if ((rr != ARCHIVE_EOF) && (rr != ARCHIVE_OK)) {
throw ZHfstZipReadingError("Archive not EOF'd or OK'd");
- }
+ }
close(temp_fd);
return rv;
- }
-#endif
+}
+
+inline Transducer* transducer_to_tmp_dir(archive* ar) {
+ char *filename = extract_to_tmp_dir(ar);
+ FILE* f = fopen(filename, "rb");
+ if (f == nullptr) {
+ throw ZHfstTemporaryWritingError("reading acceptor back from temp file");
+ }
+ return new Transducer(f);
+}
#endif // HAVE_LIBARCHIVE
@@ -264,7 +272,7 @@ ZHfstOspeller::read_zhfst(const string& filename)
{
throw ZHfstZipReadingError("Archive not OK");
}
- for (int rr = archive_read_next_header(ar, &entry);
+ for (int rr = archive_read_next_header(ar, &entry);
rr != ARCHIVE_EOF;
rr = archive_read_next_header(ar, &entry))
{
@@ -273,14 +281,34 @@ ZHfstOspeller::read_zhfst(const string& filename)
throw ZHfstZipReadingError("Archive not OK");
}
char* filename = strdup(archive_entry_pathname(entry));
- if (strncmp(filename, "acceptor.", strlen("acceptor.")) == 0)
- {
-#if ZHFST_EXTRACT_TO_TMPDIR
- char* temporary = extract_to_tmp_dir(ar);
-#elif ZHFST_EXTRACT_TO_MEM
- size_t total_length = 0;
- char* full_data = extract_to_mem(ar, entry, &total_length);
+ if (strncmp(filename, "acceptor.", strlen("acceptor.")) == 0) {
+ Transducer* trans = nullptr;
+
+#if ZHFST_EXTRACT_TO_MEM == 1
+ // Try to memory first...
+ try {
+ trans = transducer_to_mem(ar, entry);
+ }
+ catch (...) {
+ // If that failed, try to /tmp
+ //std::cerr << "Failed to memory - falling back to /tmp" << std::endl;
+ trans = transducer_to_tmp_dir(ar);
+ }
+#else
+ // Try to /tmp first...
+ try {
+ trans = transducer_to_tmp_dir(ar);
+ }
+ catch (...) {
+ // If that failed, try to memory
+ //std::cerr << "Failed to /tmp - falling back to memory" << std::endl;
+ trans = transducer_to_mem(ar, entry);
+ }
#endif
+ if (trans == nullptr) {
+ throw ZHfstZipReadingError("Failed to extract acceptor");
+ }
+
char* p = filename;
p += strlen("acceptor.");
size_t descr_len = 0;
@@ -295,30 +323,39 @@ ZHfstOspeller::read_zhfst(const string& filename)
descr_len++;
}
}
+
char* descr = hfst_strndup(p, descr_len);
-#if ZHFST_EXTRACT_TO_TMPDIR
- FILE* f = fopen(temporary, "r");
- if (f == NULL)
- {
- throw ZHfstTemporaryWritingError("reading acceptor back "
- "from temp file");
- }
- Transducer* trans = new Transducer(f);
-#elif ZHFST_EXTRACT_TO_MEM
- Transducer* trans = new Transducer(full_data);
- delete[] full_data;
-#endif
acceptors_[descr] = trans;
free(descr);
}
- else if (strncmp(filename, "errmodel.", strlen("errmodel.")) == 0)
- {
-#if ZHFST_EXTRACT_TO_TMPDIR
- char* temporary = extract_to_tmp_dir(ar);
-#elif ZHFST_EXTRACT_TO_MEM
- size_t total_length = 0;
- char* full_data = extract_to_mem(ar, entry, &total_length);
+ else if (strncmp(filename, "errmodel.", strlen("errmodel.")) == 0) {
+ Transducer* trans = nullptr;
+
+#if ZHFST_EXTRACT_TO_MEM == 1
+ // Try to memory first...
+ try {
+ trans = transducer_to_mem(ar, entry);
+ }
+ catch (...) {
+ // If that failed, try to /tmp
+ //std::cerr << "Failed to memory - falling back to /tmp" << std::endl;
+ trans = transducer_to_tmp_dir(ar);
+ }
+#else
+ // Try to /tmp first...
+ try {
+ trans = transducer_to_tmp_dir(ar);
+ }
+ catch (...) {
+ // If that failed, try to memory
+ //std::cerr << "Failed to /tmp - falling back to memory" << std::endl;
+ trans = transducer_to_mem(ar, entry);
+ }
#endif
+ if (trans == nullptr) {
+ throw ZHfstZipReadingError("Failed to extract error model");
+ }
+
const char* p = filename;
p += strlen("errmodel.");
size_t descr_len = 0;
@@ -333,34 +370,21 @@ ZHfstOspeller::read_zhfst(const string& filename)
descr_len++;
}
}
+
char* descr = hfst_strndup(p, descr_len);
-#if ZHFST_EXTRACT_TO_TMPDIR
- FILE* f = fopen(temporary, "r");
- if (NULL == f)
- {
- throw ZHfstTemporaryWritingError("reading errmodel back "
- "from temp file");
- }
- Transducer* trans = new Transducer(f);
-#elif ZHFST_EXTRACT_TO_MEM
- Transducer* trans = new Transducer(full_data);
- delete[] full_data;
-#endif
errmodels_[descr] = trans;
free(descr);
} // if acceptor or errmodel
- else if (strcmp(filename, "index.xml") == 0)
- {
-#if ZHFST_EXTRACT_TO_TMPDIR
- char* temporary = extract_to_tmp_dir(ar);
- metadata_.read_xml(temporary);
-#elif ZHFST_EXTRACT_TO_MEM
- size_t xml_len = 0;
- char* full_data = extract_to_mem(ar, entry, &xml_len);
- metadata_.read_xml(full_data, xml_len);
- delete[] full_data;
-#endif
-
+ else if (strcmp(filename, "index.xml") == 0) {
+ // Always try to memory first, as index.xml is tiny
+ try {
+ std::string full_data = extract_to_mem(ar, entry);
+ metadata_.read_xml(&full_data[0], full_data.size());
+ }
+ catch (...) {
+ char* temporary = extract_to_tmp_dir(ar);
+ metadata_.read_xml(temporary);
+ }
}
else
{
@@ -400,7 +424,7 @@ ZHfstOspeller::read_zhfst(const string& filename)
can_spell_ = true;
can_correct_ = true;
}
- else if ((acceptors_.size() > 0) &&
+ else if ((acceptors_.size() > 0) &&
(acceptors_.find("default") != acceptors_.end()))
{
current_speller_ = new Speller(0, acceptors_["default"]);
@@ -438,35 +462,4 @@ ZHfstOspeller::metadata_dump() const
return metadata_.debug_dump();
}
-
-ZHfstException::ZHfstException() :
- what_("unknown")
- {}
-ZHfstException::ZHfstException(const std::string& message) :
- what_(message)
- {}
-
-
-const char*
-ZHfstException::what()
- {
- return what_.c_str();
- }
-
-
-ZHfstMetaDataParsingError::ZHfstMetaDataParsingError(const std::string& message):
- ZHfstException(message)
- {}
-ZHfstXmlParsingError::ZHfstXmlParsingError(const std::string& message):
- ZHfstException(message)
- {}
-ZHfstZipReadingError::ZHfstZipReadingError(const std::string& message):
- ZHfstException(message)
- {}
-ZHfstTemporaryWritingError::ZHfstTemporaryWritingError(const std::string& message):
- ZHfstException(message)
- {}
-
} // namespace hfst_ol
-
-
diff --git a/ZHfstOspeller.h b/ZHfstOspeller.h
index b5b6e67..95b34b2 100644
--- a/ZHfstOspeller.h
+++ b/ZHfstOspeller.h
@@ -28,17 +28,20 @@
#ifndef HFST_OSPELL_ZHFSTOSPELLER_H_
#define HFST_OSPELL_ZHFSTOSPELLER_H_
+#include "hfstol-stdafx.h"
+
#if HAVE_CONFIG_H
# include <config.h>
#endif
+#include <stdexcept>
#include <map>
#include "ospell.h"
#include "hfst-ol.h"
#include "ZHfstOspellerXmlMetadata.h"
-namespace hfst_ol
+namespace hfst_ol
{
//! @brief ZHfstOspeller class holds one speller contained in one
//! zhfst file.
@@ -47,37 +50,37 @@ namespace hfst_ol
class ZHfstOspeller
{
public:
- //! @brief create speller with default values for undefined
+ //! @brief create speller with default values for undefined
//! language.
- ZHfstOspeller();
+ OSPELL_API ZHfstOspeller();
//! @brief destroy all automata used by the speller.
- ~ZHfstOspeller();
+ OSPELL_API ~ZHfstOspeller();
//! @brief assign a speller-suggestor circumventing the ZHFST format
- void inject_speller(Speller * s);
+ OSPELL_API void inject_speller(Speller * s);
//! @brief set upper limit to priority queue when performing
// suggestions or analyses.
- void set_queue_limit(unsigned long limit);
+ OSPELL_API void set_queue_limit(unsigned long limit);
//! @brief set upper limit for weights
- void set_weight_limit(Weight limit);
+ OSPELL_API void set_weight_limit(Weight limit);
//! @brief set search beam
- void set_beam(Weight beam);
+ OSPELL_API void set_beam(Weight beam);
//! @brief set time cutoff for correcting
- void set_time_cutoff(float time_cutoff);
+ OSPELL_API void set_time_cutoff(float time_cutoff);
//! @brief construct speller from named file containing valid
//! zhfst archive.
- void read_zhfst(const std::string& filename);
+ OSPELL_API void read_zhfst(const std::string& filename);
//! @brief check if the given word is spelled correctly
- bool spell(const std::string& wordform);
+ OSPELL_API bool spell(const std::string& wordform);
//! @brief construct an ordered set of corrections for misspelled
//! word form.
- CorrectionQueue suggest(const std::string& wordform);
+ OSPELL_API CorrectionQueue suggest(const std::string& wordform);
//! @brief analyse word form morphologically
//! @param wordform the string to analyse
//! @param ask_sugger whether to use the spelling correction model
// instead of the detection model
- AnalysisQueue analyse(const std::string& wordform,
+ AnalysisQueue analyse(const std::string& wordform,
bool ask_sugger = false);
//! @brief construct an ordered set of corrections with analyses
AnalysisCorrectionQueue suggest_analyses(const std::string&
@@ -133,20 +136,15 @@ namespace hfst_ol
//! Contains a human-readable error message that can be displayed to
//! end-user as additional info when either solving exception or exiting.
- class ZHfstException
+ class ZHfstException : public std::runtime_error
{
public:
- ZHfstException();
+ ZHfstException() : std::runtime_error("unknown") {}
//! @brief construct error with human readable message.
//!
//! the message will be displayed when recovering or dying from
//! exception
- explicit ZHfstException(const std::string& message);
- //!
- //! format error as user-readable message
- const char* what();
- private:
- std::string what_;
+ explicit ZHfstException(const std::string& message) : std::runtime_error(message) {}
};
//! @brief Generic error in metadata parsing.
@@ -155,9 +153,7 @@ namespace hfst_ol
class ZHfstMetaDataParsingError : public ZHfstException
{
public:
- explicit ZHfstMetaDataParsingError(const std::string& message);
- private:
- std::string what_;
+ explicit ZHfstMetaDataParsingError(const std::string& message) : ZHfstException(message) {}
};
//! @brief Exception for XML parser errors.
@@ -168,9 +164,7 @@ namespace hfst_ol
class ZHfstXmlParsingError : public ZHfstException
{
public:
- explicit ZHfstXmlParsingError(const std::string& message);
- private:
- std::string what_;
+ explicit ZHfstXmlParsingError(const std::string& message) : ZHfstException(message) {}
};
//! @brief Generic error while reading zip file.
@@ -180,9 +174,7 @@ namespace hfst_ol
class ZHfstZipReadingError : public ZHfstException
{
public:
- explicit ZHfstZipReadingError(const std::string& message);
- private:
- std::string what_;
+ explicit ZHfstZipReadingError(const std::string& message) : ZHfstException(message) {}
};
//! @brief Error when writing to temporary location.
@@ -192,9 +184,7 @@ namespace hfst_ol
class ZHfstTemporaryWritingError : public ZHfstException
{
public:
- explicit ZHfstTemporaryWritingError(const std::string& message);
- private:
- std::string what_;
+ explicit ZHfstTemporaryWritingError(const std::string& message) : ZHfstException(message) {}
};
} // namespace hfst_ol
diff --git a/ZHfstOspellerXmlMetadata.cc b/ZHfstOspellerXmlMetadata.cc
index 64c0502..727d23c 100644
--- a/ZHfstOspellerXmlMetadata.cc
+++ b/ZHfstOspellerXmlMetadata.cc
@@ -39,7 +39,7 @@ using std::map;
#include "ZHfstOspeller.h"
#include "ZHfstOspellerXmlMetadata.h"
-namespace hfst_ol
+namespace hfst_ol
{
ZHfstOspellerXmlMetadata::ZHfstOspellerXmlMetadata()
@@ -90,7 +90,7 @@ ZHfstOspellerXmlMetadata::verify_hfstspeller(xmlpp::Node* rootNode)
"root from XML file");
}
// check versions
- const xmlpp::Attribute* hfstversion =
+ const xmlpp::Attribute* hfstversion =
rootElement->get_attribute("hfstversion");
if (NULL == hfstversion)
{
@@ -204,7 +204,7 @@ ZHfstOspellerXmlMetadata::parse_title(xmlpp::Node* titleNode)
void
ZHfstOspellerXmlMetadata::parse_description(xmlpp::Node* descriptionNode)
{
- xmlpp::Element* descriptionElement =
+ xmlpp::Element* descriptionElement =
dynamic_cast<xmlpp::Element*>(descriptionNode);
const xmlpp::Attribute* lang = descriptionElement->get_attribute("lang");
if (NULL == descriptionElement->get_child_text())
@@ -238,7 +238,7 @@ ZHfstOspellerXmlMetadata::parse_version(xmlpp::Node* versionNode)
void
ZHfstOspellerXmlMetadata::parse_date(xmlpp::Node* dateNode)
{
- xmlpp::Element* dateElement =
+ xmlpp::Element* dateElement =
dynamic_cast<xmlpp::Element*>(dateNode);
info_.date_ = dateElement->get_child_text()->get_content();
}
@@ -246,7 +246,7 @@ ZHfstOspellerXmlMetadata::parse_date(xmlpp::Node* dateNode)
void
ZHfstOspellerXmlMetadata::parse_producer(xmlpp::Node* producerNode)
{
- xmlpp::Element* producerElement =
+ xmlpp::Element* producerElement =
dynamic_cast<xmlpp::Element*>(producerNode);
info_.producer_ = producerElement->get_child_text()->get_content();
}
@@ -271,7 +271,7 @@ ZHfstOspellerXmlMetadata::parse_contact(xmlpp::Node* contactNode)
void
ZHfstOspellerXmlMetadata::parse_acceptor(xmlpp::Node* acceptorNode)
{
- xmlpp::Element* acceptorElement =
+ xmlpp::Element* acceptorElement =
dynamic_cast<xmlpp::Element*>(acceptorNode);
xmlpp::Attribute* xid = acceptorElement->get_attribute("id");
if (xid == NULL)
@@ -286,7 +286,7 @@ ZHfstOspellerXmlMetadata::parse_acceptor(xmlpp::Node* acceptorNode)
char* descr = get_automaton_descr_from_id(xidValue.c_str());
acceptor_[descr].descr_ = descr;
acceptor_[descr].id_ = xidValue;
- const xmlpp::Attribute* trtype =
+ const xmlpp::Attribute* trtype =
acceptorElement->get_attribute("transtype");
if (trtype != NULL)
{
@@ -325,7 +325,7 @@ ZHfstOspellerXmlMetadata::parse_acceptor(xmlpp::Node* acceptorNode)
}
void
-ZHfstOspellerXmlMetadata::parse_title(xmlpp::Node* titleNode,
+ZHfstOspellerXmlMetadata::parse_title(xmlpp::Node* titleNode,
const string& descr)
{
xmlpp::Element* titleElement = dynamic_cast<xmlpp::Element*>(titleNode);
@@ -344,7 +344,7 @@ void
ZHfstOspellerXmlMetadata::parse_description(xmlpp::Node* descriptionNode,
const string& descr)
{
- xmlpp::Element* descriptionElement =
+ xmlpp::Element* descriptionElement =
dynamic_cast<xmlpp::Element*>(descriptionNode);
const xmlpp::Attribute* lang = descriptionElement->get_attribute("lang");
if (lang != NULL)
@@ -360,7 +360,7 @@ ZHfstOspellerXmlMetadata::parse_description(xmlpp::Node* descriptionNode,
void
ZHfstOspellerXmlMetadata::parse_errmodel(xmlpp::Node* errmodelNode)
{
- xmlpp::Element* errmodelElement =
+ xmlpp::Element* errmodelElement =
dynamic_cast<xmlpp::Element*>(errmodelNode);
xmlpp::Attribute* xid = errmodelElement->get_attribute("id");
if (xid == NULL)
@@ -416,7 +416,7 @@ ZHfstOspellerXmlMetadata::parse_errmodel(xmlpp::Node* errmodelNode)
}
void
-ZHfstOspellerXmlMetadata::parse_title(xmlpp::Node* titleNode,
+ZHfstOspellerXmlMetadata::parse_title(xmlpp::Node* titleNode,
size_t errm_count)
{
xmlpp::Element* titleElement = dynamic_cast<xmlpp::Element*>(titleNode);
@@ -435,7 +435,7 @@ void
ZHfstOspellerXmlMetadata::parse_description(xmlpp::Node* descriptionNode,
size_t errm_count)
{
- xmlpp::Element* descriptionElement =
+ xmlpp::Element* descriptionElement =
dynamic_cast<xmlpp::Element*>(descriptionNode);
const xmlpp::Attribute* lang = descriptionElement->get_attribute("lang");
if (lang != NULL)
@@ -485,7 +485,7 @@ ZHfstOspellerXmlMetadata::parse_xml(const xmlpp::Document* doc)
throw ZHfstMetaDataParsingError("No root node in index XML");
}
verify_hfstspeller(rootNode);
- // parse
+ // parse
xmlpp::Node::NodeList nodes = rootNode->get_children();
for (xmlpp::Node::NodeList::iterator node = nodes.begin();
node != nodes.end();
@@ -692,7 +692,7 @@ ZHfstOspellerXmlMetadata::parse_description(const tinyxml2::XMLElement& descript
{
info_.description_[info_.locale_] = descriptionNode.GetText();
}
-
+
}
void
@@ -773,7 +773,7 @@ ZHfstOspellerXmlMetadata::parse_acceptor(const tinyxml2::XMLElement& acceptorNod
}
free(descr);
}
-
+
void
ZHfstOspellerXmlMetadata::parse_title(const tinyxml2::XMLElement& titleNode,
const std::string& accName)
@@ -785,11 +785,11 @@ ZHfstOspellerXmlMetadata::parse_title(const tinyxml2::XMLElement& titleNode,
}
else
{
- acceptor_[accName].title_[info_.locale_] =
+ acceptor_[accName].title_[info_.locale_] =
titleNode.GetText();
}
}
-
+
void
ZHfstOspellerXmlMetadata::parse_description(const tinyxml2::XMLElement& descriptionNode,
@@ -802,7 +802,7 @@ ZHfstOspellerXmlMetadata::parse_description(const tinyxml2::XMLElement& descript
}
else
{
- acceptor_[accName].description_[info_.locale_] =
+ acceptor_[accName].description_[info_.locale_] =
descriptionNode.GetText();
}
}
@@ -880,7 +880,7 @@ ZHfstOspellerXmlMetadata::parse_description(const tinyxml2::XMLElement& descript
}
else
{
- errmodel_[errm_count].description_[info_.locale_] =
+ errmodel_[errm_count].description_[info_.locale_] =
descriptionNode.GetText();
}
}
@@ -926,7 +926,6 @@ ZHfstOspellerXmlMetadata::read_xml(const string& filename)
this->parse_xml(doc);
}
#else
-#error configure found no usable XML library
void
ZHfstOspellerXmlMetadata::read_xml(const char*, size_t)
{}
@@ -1014,7 +1013,7 @@ ZHfstOspellerXmlMetadata::debug_dump() const
retval.append("model: " + *model + "\n");
}
}
-
+
return retval;
}
diff --git a/ZHfstOspellerXmlMetadata.h b/ZHfstOspellerXmlMetadata.h
index 066174a..b6e1805 100644
--- a/ZHfstOspellerXmlMetadata.h
+++ b/ZHfstOspellerXmlMetadata.h
@@ -16,6 +16,8 @@
#ifndef HFST_OSPELL_ZHFSTOSPELLERXMLMETADATA_H_
#define HFST_OSPELL_ZHFSTOSPELLERXMLMETADATA_H_ 1
+#include "hfstol-stdafx.h"
+
#if HAVE_CONFIG_H
# include <config.h>
#endif
diff --git a/autogen.sh b/autogen.sh
index 916bace..f006c5c 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -98,7 +98,7 @@ ident ( ) {
fi
# extract version from CVS Id string
- __id="$Id: autogen.sh 920 2011-02-03 00:36:00Z mie $"
+ __id="$Id$"
__version="`echo $__id | sed 's/.*\([0-9][0-9][0-9][0-9]\)[-\/]\([0-9][0-9]\)[-\/]\([0-9][0-9]\).*/\1\2\3/'`"
if [ "x$__version" = "x" ] ; then
__version=""
diff --git a/configure.ac b/configure.ac
index d4ba779..5186cce 100644
--- a/configure.ac
+++ b/configure.ac
@@ -17,7 +17,7 @@
# autoconf requirements
AC_PREREQ([2.62])
-AC_INIT([hfstospell], [0.4.0], [hfst-bugs at helsinki.fi], [hfstospell], [http://hfst.sf.net])
+AC_INIT([hfstospell], [0.4.3], [hfst-bugs at helsinki.fi], [hfstospell], [http://hfst.github.io])
LT_PREREQ([2.2.6])
@@ -25,7 +25,7 @@ LT_PREREQ([2.2.6])
AC_CONFIG_AUX_DIR([build-aux])
AM_INIT_AUTOMAKE([1.11 -Wall -Werror foreign check-news color-tests silent-rules])
AM_SILENT_RULES([yes])
-AC_REVISION([$Revision: 4619 $])
+AC_REVISION([$Revision$])
AC_CONFIG_MACRO_DIR([m4])
AC_CONFIG_SRCDIR([ospell.cc])
AC_CONFIG_HEADERS([config.h])
@@ -34,7 +34,7 @@ AC_CONFIG_HEADERS([config.h])
HFSTOSPELL_NAME=hfstospell
HFSTOSPELL_MAJOR=0
HFSTOSPELL_MINOR=4
-HFSTOSPELL_EXTENSION=.0
+HFSTOSPELL_EXTENSION=.3
HFSTOSPELL_VERSION=$HFSTOSPELL_MAJOR.$HFSTOSPELL_MINOR$HFSTOSPELL_EXTENSION
AC_SUBST(HFSTOSPELL_MAJOR)
AC_SUBST(HFSTOSPELL_MINOR)
@@ -76,10 +76,10 @@ AC_ARG_WITH([extract],
[extract zhfst archives to tmpdir or mem @<:@default=mem@:>@])],
[with_extract=$withval], [with_extract=mem])
AS_IF([test "x$with_extract" = xmem], [AC_DEFINE([ZHFST_EXTRACT_TO_MEM], [1],
- [Define to extract zhfst archives to char buffer])],
+ [Define to extract zhfst archives to char buffer first])],
[AS_IF([test "x$with_extract" = xtmpdir],
- [AC_DEFINE([ZHFST_EXTRACT_TO_TMPDIR], [1],
- [Define to extract zhfst to tmp dir])],
+ [AC_DEFINE([ZHFST_EXTRACT_TO_MEM], [0],
+ [Define to extract zhfst to tmp dir first])],
[AC_MSG_ERROR([Use with-extract to mem or tmpdir])])])
# Checks for programs
@@ -91,11 +91,6 @@ LT_INIT
AC_PROG_INSTALL
AC_PROG_LN_S
AC_PROG_MAKE_SET
-AC_PATH_PROG([HFST_TXT2FST], [hfst-txt2fst], [false])
-AC_PATH_PROG([HFST_FST2FST], [hfst-fst2fst], [false])
-AC_PATH_PROG([ZIP], [zip], [false])
-AM_CONDITIONAL([CAN_TEST],
- [test x$HFST_TXT2FST != xfalse -a x$HFST_FST2FST != xfalse -a x$ZIP != xfalse])
# Checks for libraries
AS_IF([test x$enable_zhfst != xno],
@@ -116,7 +111,7 @@ AS_IF([test x$with_libxmlpp != xno],
enable_xml=no])])
AM_CONDITIONAL([WANT_LIBXMLPP], [test x$enable_xml = xlibxmlpp])
AS_IF([test x$with_tinyxml2 != xno -a x$with_libxmlpp = xno],
- [PKG_CHECK_MODULES([TINYXML2], [tinyxml2 >= 1.0.8],
+ [PKG_CHECK_MODULES([TINYXML2], [tinyxml2 >= 1.0.8 tinyxml2 < 3],
[AC_DEFINE([HAVE_TINYXML2], [1], [Use tinyxml])
enable_xml=tinyxml2],
[AC_MSG_ERROR([tinyxml missing])
@@ -141,6 +136,7 @@ AC_TYPE_SIZE_T
# Checks for structures
# Checks for compiler characteristics
+AC_C_BIGENDIAN
# Checks for library functions
AC_FUNC_MALLOC
@@ -154,7 +150,7 @@ AX_CHECK_COMPILE_FLAG([-std=c++17], [CXXFLAGS="$CXXFLAGS -std=c++17"], [
AX_CHECK_COMPILE_FLAG([-std=c++14], [CXXFLAGS="$CXXFLAGS -std=c++14"], [
AX_CHECK_COMPILE_FLAG([-std=c++1y], [CXXFLAGS="$CXXFLAGS -std=c++1y"], [
AX_CHECK_COMPILE_FLAG([-std=c++11], [CXXFLAGS="$CXXFLAGS -std=c++11"], [
- AX_CHECK_COMPILE_FLAG([-std=c++0x], [CXXFLAGS="$CXXFLAGS -std=c++0x"], [])
+ AC_MSG_ERROR([could not enable C++11 or newer])
])
])
])
@@ -176,4 +172,4 @@ cat <<EOF
* conference demos: $enable_extra_demos
EOF
AS_IF([test x$with_libxmlpp != xno -a x$with_tinyxml2 != xno],
- [AC_MSG_WARN([You can only have one xml library (e.g., --with-tinyxml2 --without-libxmlpp)])])
+ [AC_MSG_ERROR([You can only have one xml library (e.g., --with-tinyxml2 --without-libxmlpp)])])
diff --git a/hfst-ol.cc b/hfst-ol.cc
index e926cbe..23d5b25 100644
--- a/hfst-ol.cc
+++ b/hfst-ol.cc
@@ -14,6 +14,9 @@
#include "hfst-ol.h"
#include <string>
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif
namespace hfst_ol {
@@ -33,61 +36,144 @@ void skip_c_string(char ** raw)
++(*raw);
}
+bool is_big_endian()
+{
+#ifdef WORDS_BIGENDIAN
+ return true;
+#else
+ return false;
+#endif
+}
+
+uint16_t read_uint16_flipping_endianness(FILE * f)
+{
+ uint16_t result = 0;
+ unsigned char byte1 = static_cast<uint8_t>(getc(f));
+ unsigned char byte2 = static_cast<uint8_t>(getc(f));
+ result |= byte2;
+ result <<= 8;
+ result |= byte1;
+ return result;
+}
+
+uint16_t read_uint16_flipping_endianness(char * raw)
+{
+ uint16_t result = 0;
+ result |= *(raw + 1);
+ result <<= 8;
+ result |= *raw;
+ return result;
+}
+
+uint32_t read_uint32_flipping_endianness(FILE * f)
+{
+ uint32_t result = 0;
+ unsigned char byte1 = static_cast<uint8_t>(getc(f));
+ unsigned char byte2 = static_cast<uint8_t>(getc(f));
+ unsigned char byte3 = static_cast<uint8_t>(getc(f));
+ unsigned char byte4 = static_cast<uint8_t>(getc(f));
+ result |= byte4;
+ result <<= 8;
+ result |= byte3;
+ result <<= 8;
+ result |= byte2;
+ result <<= 8;
+ result |= byte1;
+ return result;
+}
+
+uint32_t read_uint32_flipping_endianness(char * raw)
+{
+ uint32_t result = 0;
+ result |= *(raw + 3);
+ result <<= 8;
+ result |= *(raw + 2);
+ result <<= 8;
+ result |= *(raw + 1);
+ result <<= 8;
+ result |= *raw;
+ return result;
+}
+
+float read_float_flipping_endianness(FILE * f)
+{
+ union {
+ float f;
+ char c[4];
+ } flipper;
+
+ (void)f;
+ flipper.f = 0;
+
+ return flipper.f;
+}
+
void
TransducerHeader::read_property(bool& property, FILE* f)
{
- unsigned int prop;
- if (fread(&prop,sizeof(unsigned int),1,f) != 1) {
- HFST_THROW_MESSAGE(HeaderParsingException,
- "Header ended unexpectedly\n");
- }
- if (prop == 0)
- {
- property = false;
- return;
- }
- else
- {
- property = true;
- return;
+ if (is_big_endian()) {
+ // four little-endian bytes
+ int c = getc(f);
+ property = !(c == 0);
+ c = getc(f);
+ c = getc(f);
+ c = getc(f);
+ } else {
+ unsigned int prop;
+ if (fread(&prop,sizeof(uint32_t),1,f) != 1) {
+ HFST_THROW_MESSAGE(HeaderParsingException,
+ "Header ended unexpectedly\n");
+ }
+ if (prop == 0)
+ {
+ property = false;
+ }
+ else
+ {
+ property = true;
+ }
}
}
void
TransducerHeader::read_property(bool& property, char** raw)
{
- unsigned int prop = *((unsigned int *) *raw);
- (*raw) += sizeof(unsigned int);
- if (prop == 0)
- {
- property = false;
- return;
- }
- else
- {
- property = true;
- return;
+ if (is_big_endian()) {
+ property = !(**raw == 0);
+ } else {
+ unsigned int prop = *((unsigned int *) *raw);
+ property = !(prop == 0);
}
+ (*raw) += sizeof(uint32_t);
}
void TransducerHeader::skip_hfst3_header(FILE * f)
{
const char* header1 = "HFST";
unsigned int header_loc = 0; // how much of the header has been found
- int c;
+ int c = 0;
for(header_loc = 0; header_loc < strlen(header1) + 1; header_loc++)
{
c = getc(f);
if(c != header1[header_loc]) {
+ //std::cerr << header_loc << ": " << int(c) << " != " << header1[header_loc] << std::endl;
break;
}
}
if(header_loc == strlen(header1) + 1) // we found it
{
- unsigned short remaining_header_len;
- if (fread(&remaining_header_len,
- sizeof(remaining_header_len), 1, f) != 1 ||
- getc(f) != '\0') {
+ uint16_t remaining_header_len = 0;
+ if (is_big_endian()) {
+ remaining_header_len = read_uint16_flipping_endianness(f);
+ } else {
+ if (fread(&remaining_header_len,
+ sizeof(remaining_header_len), 1, f) != 1) {
+ HFST_THROW_MESSAGE(HeaderParsingException,
+ "Found broken HFST3 header\n");
+ }
+ }
+ //std::cerr << "remaining_header_len " << remaining_header_len << std::endl;
+ if (getc(f) != '\0') {
HFST_THROW_MESSAGE(HeaderParsingException,
"Found broken HFST3 header\n");
}
@@ -106,7 +192,7 @@ void TransducerHeader::skip_hfst3_header(FILE * f)
if (type_field != std::string::npos) {
if (header_tail.find("HFST_OL") != type_field + 5 &&
header_tail.find("HFST_OLW") != type_field + 5) {
- delete headervalue;
+ delete[] headervalue;
HFST_THROW_MESSAGE(
TransducerTypeException,
"Transducer has incorrect type, should be "
@@ -131,14 +217,21 @@ void TransducerHeader::skip_hfst3_header(char ** raw)
for(header_loc = 0; header_loc < strlen(header1) + 1; header_loc++)
{
if(**raw != header1[header_loc]) {
+ //std::cerr << header_loc << ": " << int(**raw) << " != " << header1[header_loc] << std::endl;
break;
}
++(*raw);
}
if(header_loc == strlen(header1) + 1) // we found it
{
- unsigned short remaining_header_len = *((unsigned short *) *raw);
- (*raw) += sizeof(unsigned short) + 1 + remaining_header_len;
+ uint16_t remaining_header_len = 0;
+ if (is_big_endian()) {
+ remaining_header_len = read_uint16_flipping_endianness(*raw);
+ } else {
+ remaining_header_len = *((unsigned short *) *raw);
+ }
+ //std::cerr << "remaining_header_len " << remaining_header_len << std::endl;
+ (*raw) += sizeof(uint16_t) + 1 + remaining_header_len;
} else // nope. put back what we've taken
{
--(*raw); // first the non-matching character
@@ -152,23 +245,41 @@ void TransducerHeader::skip_hfst3_header(char ** raw)
TransducerHeader::TransducerHeader(FILE* f)
{
skip_hfst3_header(f); // skip header iff it is present
- /* The following conditional clause does all the numerical reads
- and throws an exception if any fails to return 1 */
- if (fread(&number_of_input_symbols,
- sizeof(SymbolNumber),1,f) != 1||
- fread(&number_of_symbols,
- sizeof(SymbolNumber),1,f) != 1||
- fread(&size_of_transition_index_table,
- sizeof(TransitionTableIndex),1,f) != 1||
- fread(&size_of_transition_target_table,
- sizeof(TransitionTableIndex),1,f) != 1||
- fread(&number_of_states,
- sizeof(TransitionTableIndex),1,f) != 1||
- fread(&number_of_transitions,
- sizeof(TransitionTableIndex),1,f) != 1) {
- HFST_THROW_MESSAGE(HeaderParsingException,
- "Header ended unexpectedly\n");
+ if (is_big_endian()) {
+ // no error checking yet
+ number_of_input_symbols = read_uint16_flipping_endianness(f);
+ number_of_symbols = read_uint16_flipping_endianness(f);
+ size_of_transition_index_table = read_uint32_flipping_endianness(f);
+ size_of_transition_target_table = read_uint32_flipping_endianness(f);
+ number_of_states = read_uint32_flipping_endianness(f);
+ number_of_transitions = read_uint32_flipping_endianness(f);
+ } else {
+ /* The following conditional clause does all the numerical reads
+ and throws an exception if any fails to return 1 */
+ if (fread(&number_of_input_symbols,
+ sizeof(SymbolNumber),1,f) != 1||
+ fread(&number_of_symbols,
+ sizeof(SymbolNumber),1,f) != 1||
+ fread(&size_of_transition_index_table,
+ sizeof(TransitionTableIndex),1,f) != 1||
+ fread(&size_of_transition_target_table,
+ sizeof(TransitionTableIndex),1,f) != 1||
+ fread(&number_of_states,
+ sizeof(TransitionTableIndex),1,f) != 1||
+ fread(&number_of_transitions,
+ sizeof(TransitionTableIndex),1,f) != 1) {
+ HFST_THROW_MESSAGE(HeaderParsingException,
+ "Header ended unexpectedly\n");
+ }
}
+ /*
+ std::cerr << "number_of_input_symbols " << number_of_input_symbols << std::endl;
+ std::cerr << "number_of_symbols " << number_of_symbols << std::endl;
+ std::cerr << "size_of_transition_index_table " << size_of_transition_index_table << std::endl;
+ std::cerr << "size_of_transition_target_table " << size_of_transition_target_table << std::endl;
+ std::cerr << "number_of_states " << number_of_states << std::endl;
+ std::cerr << "number_of_transitions " << number_of_transitions << std::endl;
+ //*/
read_property(weighted,f);
read_property(deterministic,f);
read_property(input_deterministic,f);
@@ -183,18 +294,41 @@ TransducerHeader::TransducerHeader(FILE* f)
TransducerHeader::TransducerHeader(char** raw)
{
skip_hfst3_header(raw); // skip header iff it is present
- number_of_input_symbols = *(SymbolNumber*) *raw;
- (*raw) += sizeof(SymbolNumber);
- number_of_symbols = *(SymbolNumber*) *raw;
- (*raw) += sizeof(SymbolNumber);
- size_of_transition_index_table = *(TransitionTableIndex*) *raw;
- (*raw) += sizeof(TransitionTableIndex);
- size_of_transition_target_table = *(TransitionTableIndex*) *raw;
- (*raw) += sizeof(TransitionTableIndex);
- number_of_states = *(TransitionTableIndex*) *raw;
- (*raw) += sizeof(TransitionTableIndex);
- number_of_transitions = *(TransitionTableIndex*) *raw;
- (*raw) += sizeof(TransitionTableIndex);
+ if (is_big_endian()) {
+ number_of_input_symbols = read_uint16_flipping_endianness(*raw);
+ (*raw) += sizeof(SymbolNumber);
+ number_of_symbols = read_uint16_flipping_endianness(*raw);
+ (*raw) += sizeof(SymbolNumber);
+ size_of_transition_index_table = read_uint32_flipping_endianness(*raw);
+ (*raw) += sizeof(TransitionTableIndex);
+ size_of_transition_target_table = read_uint32_flipping_endianness(*raw);
+ (*raw) += sizeof(TransitionTableIndex);
+ number_of_states = read_uint32_flipping_endianness(*raw);
+ (*raw) += sizeof(TransitionTableIndex);
+ number_of_transitions = read_uint32_flipping_endianness(*raw);
+ (*raw) += sizeof(TransitionTableIndex);
+ } else {
+ number_of_input_symbols = *(SymbolNumber*) *raw;
+ (*raw) += sizeof(SymbolNumber);
+ number_of_symbols = *(SymbolNumber*) *raw;
+ (*raw) += sizeof(SymbolNumber);
+ size_of_transition_index_table = *(TransitionTableIndex*) *raw;
+ (*raw) += sizeof(TransitionTableIndex);
+ size_of_transition_target_table = *(TransitionTableIndex*) *raw;
+ (*raw) += sizeof(TransitionTableIndex);
+ number_of_states = *(TransitionTableIndex*) *raw;
+ (*raw) += sizeof(TransitionTableIndex);
+ number_of_transitions = *(TransitionTableIndex*) *raw;
+ (*raw) += sizeof(TransitionTableIndex);
+ }
+ /*
+ std::cerr << "number_of_input_symbols " << number_of_input_symbols << std::endl;
+ std::cerr << "number_of_symbols " << number_of_symbols << std::endl;
+ std::cerr << "size_of_transition_index_table " << size_of_transition_index_table << std::endl;
+ std::cerr << "size_of_transition_target_table " << size_of_transition_target_table << std::endl;
+ std::cerr << "number_of_states " << number_of_states << std::endl;
+ std::cerr << "number_of_transitions " << number_of_transitions << std::endl;
+ //*/
read_property(weighted,raw);
read_property(deterministic,raw);
read_property(input_deterministic,raw);
@@ -209,24 +343,24 @@ TransducerHeader::TransducerHeader(char** raw)
SymbolNumber
TransducerHeader::symbol_count()
{
- return number_of_symbols;
+ return number_of_symbols;
}
SymbolNumber
TransducerHeader::input_symbol_count()
{
- return number_of_input_symbols;
+ return number_of_input_symbols;
}
TransitionTableIndex
-TransducerHeader::index_table_size(void)
+TransducerHeader::index_table_size()
{
- return size_of_transition_index_table;
+ return size_of_transition_index_table;
}
-TransitionTableIndex
+TransitionTableIndex
TransducerHeader::target_table_size()
{
- return size_of_transition_target_table;
+ return size_of_transition_target_table;
}
bool
@@ -258,7 +392,7 @@ TransducerHeader::probe_flag(HeaderFlag flag)
bool
FlagDiacriticOperation::isFlag() const
{
- return feature != NO_SYMBOL;
+ return feature != NO_SYMBOL;
}
FlagDiacriticOperator
@@ -305,7 +439,7 @@ void TransducerAlphabet::read(FILE * f, SymbolNumber number_of_symbols)
if (byte == EOF) {
HFST_THROW(AlphabetParsingException);
}
- *sym = byte;
+ *sym = static_cast<char>(byte);
++sym;
}
*sym = 0;
@@ -366,7 +500,7 @@ void TransducerAlphabet::read(FILE * f, SymbolNumber number_of_symbols)
string_to_symbol[std::string(line)] = k;
}
free(line);
- flag_state_size = feature_bucket.size();
+ flag_state_size = static_cast<SymbolNumber>(feature_bucket.size());
}
void TransducerAlphabet::read(char ** raw, SymbolNumber number_of_symbols)
@@ -443,7 +577,7 @@ void TransducerAlphabet::read(char ** raw, SymbolNumber number_of_symbols)
string_to_symbol[std::string(*raw)] = k;
skip_c_string(raw);
}
- flag_state_size = feature_bucket.size();
+ flag_state_size = static_cast<SymbolNumber>(feature_bucket.size());
}
TransducerAlphabet::TransducerAlphabet(FILE* f, SymbolNumber number_of_symbols):
@@ -465,7 +599,7 @@ TransducerAlphabet::TransducerAlphabet(char** raw,
void TransducerAlphabet::add_symbol(std::string & sym)
{
- string_to_symbol[sym] = kt.size();
+ string_to_symbol[sym] = static_cast<SymbolNumber>(kt.size());
kt.push_back(sym);
}
@@ -478,7 +612,7 @@ void TransducerAlphabet::add_symbol(char * sym)
KeyTable*
TransducerAlphabet::get_key_table()
{
- return &kt;
+ return &kt;
}
OperationMap*
@@ -535,6 +669,9 @@ void IndexTable::read(FILE * f,
if (fread(indices,table_size, 1, f) != 1) {
HFST_THROW(IndexTableReadingException);
}
+ if (is_big_endian()) {
+ convert_to_big_endian();
+ }
}
void IndexTable::read(char ** raw,
@@ -544,6 +681,23 @@ void IndexTable::read(char ** raw,
indices = (char*)(malloc(table_size));
memcpy((void *) indices, (const void *) *raw, table_size);
(*raw) += table_size;
+ if (is_big_endian()) {
+ convert_to_big_endian();
+ }
+}
+
+void IndexTable::convert_to_big_endian()
+{
+ //std::cerr << "IndexTable::convert_to_big_endian " << size << std::endl;
+ // We have a sequence of uint_16's and uint_32's in memory,
+ // and we're going to flip endianness of each one.
+ for(size_t i = 0; i < size; ++i) {
+ // first the uint_16
+ std::swap(*(indices + i * TransitionIndex::SIZE + 0), *(indices + i * TransitionIndex::SIZE + 1));
+ // then the uint_32
+ std::swap(*(indices + i * TransitionIndex::SIZE + 2), *(indices + i * TransitionIndex::SIZE + 5));
+ std::swap(*(indices + i * TransitionIndex::SIZE + 3), *(indices + i * TransitionIndex::SIZE + 4));
+ }
}
void TransitionTable::read(FILE * f,
@@ -554,6 +708,9 @@ void TransitionTable::read(FILE * f,
if (fread(transitions, table_size, 1, f) != 1) {
HFST_THROW(TransitionTableReadingException);
}
+ if (is_big_endian()) {
+ convert_to_big_endian();
+ }
}
void TransitionTable::read(char ** raw,
@@ -563,6 +720,27 @@ void TransitionTable::read(char ** raw,
transitions = (char*)(malloc(table_size));
memcpy((void *) transitions, (const void *) *raw, table_size);
(*raw) += table_size;
+ if (is_big_endian()) {
+ convert_to_big_endian();
+ }
+}
+
+void TransitionTable::convert_to_big_endian()
+{
+ //std::cerr << "TransitionTable::convert_to_big_endian " << size << std::endl;
+ // We have a sequence of [ uint_16, uint_16, uint_32, float ] in memory,
+ // and we're going to flip endianness of each one.
+ for(size_t i = 0; i < size; ++i) {
+ // first the uint_16s
+ std::swap(*(transitions + i * Transition::SIZE + 0), *(transitions + i * Transition::SIZE + 1));
+ std::swap(*(transitions + i * Transition::SIZE + 2), *(transitions + i * Transition::SIZE + 3));
+ // then the uint_32
+ std::swap(*(transitions + i * Transition::SIZE + 4), *(transitions + i * Transition::SIZE + 7));
+ std::swap(*(transitions + i * Transition::SIZE + 5), *(transitions + i * Transition::SIZE + 6));
+ // then the float
+ std::swap(*(transitions + i * Transition::SIZE + 8), *(transitions + i * Transition::SIZE + 11));
+ std::swap(*(transitions + i * Transition::SIZE + 9), *(transitions + i * Transition::SIZE + 10));
+ }
}
void LetterTrie::add_string(const char * p, SymbolNumber symbol_key)
@@ -602,14 +780,14 @@ LetterTrie::~LetterTrie()
i != letters.end(); ++i)
{
if (*i)
- {
+ {
delete *i;
}
}
}
Encoder::Encoder(KeyTable * kt, SymbolNumber number_of_input_symbols):
- ascii_symbols(UCHAR_MAX,NO_SYMBOL)
+ ascii_symbols(UCHAR_MAX+1,NO_SYMBOL)
{
read_input_symbols(kt, number_of_input_symbols);
}
@@ -621,9 +799,9 @@ void Encoder::read_input_symbol(const char * s, const int s_num)
}
if ((strlen(s) == 1) && (unsigned char)(*s) <= 127)
{
- ascii_symbols[(unsigned char)(*s)] = s_num;
+ ascii_symbols[(unsigned char)(*s)] = static_cast<SymbolNumber>(s_num);
}
- letters.add_string(s, s_num);
+ letters.add_string(s, static_cast<SymbolNumber>(s_num));
}
void Encoder::read_input_symbol(std::string const & s, const int s_num)
@@ -647,15 +825,15 @@ TransitionIndex::target() const
return first_transition_index;
}
-bool
-TransitionIndex::final(void) const
+bool
+TransitionIndex::final() const
{
return input_symbol == NO_SYMBOL &&
first_transition_index != NO_TABLE_INDEX;
}
Weight
-TransitionIndex::final_weight(void) const
+TransitionIndex::final_weight() const
{
union to_weight
{
@@ -667,37 +845,37 @@ TransitionIndex::final_weight(void) const
}
SymbolNumber
-TransitionIndex::get_input(void) const
+TransitionIndex::get_input() const
{
return input_symbol;
}
-TransitionTableIndex
-Transition::target(void) const
+TransitionTableIndex
+Transition::target() const
{
return target_index;
}
SymbolNumber
-Transition::get_output(void) const
+Transition::get_output() const
{
return output_symbol;
}
SymbolNumber
-Transition::get_input(void) const
+Transition::get_input() const
{
return input_symbol;
}
Weight
-Transition::get_weight(void) const
+Transition::get_weight() const
{
return transition_weight;
}
bool
-Transition::final(void) const
+Transition::final() const
{
return input_symbol == NO_SYMBOL &&
output_symbol == NO_SYMBOL &&
@@ -711,7 +889,7 @@ IndexTable::IndexTable(FILE* f,
{
read(f, number_of_table_entries);
}
-
+
IndexTable::IndexTable(char ** raw,
TransitionTableIndex number_of_table_entries):
indices(NULL),
@@ -720,7 +898,7 @@ IndexTable::IndexTable(char ** raw,
read(raw, number_of_table_entries);
}
-IndexTable::~IndexTable(void)
+IndexTable::~IndexTable()
{
if (indices) {
free(indices);
@@ -742,7 +920,7 @@ TransitionTableIndex
IndexTable::target(TransitionTableIndex i) const
{
if (i < size) {
- return hfst_deref((TransitionTableIndex *)
+ return hfst_deref((TransitionTableIndex *)
(indices + TransitionIndex::SIZE * i +
sizeof(SymbolNumber)));
} else {
@@ -775,7 +953,7 @@ TransitionTable::TransitionTable(FILE * f,
{
read(f, transition_count);
}
-
+
TransitionTable::TransitionTable(char ** raw,
TransitionTableIndex transition_count):
transitions(NULL),
@@ -784,7 +962,7 @@ TransitionTable::TransitionTable(char ** raw,
read(raw, transition_count);
}
-TransitionTable::~TransitionTable(void)
+TransitionTable::~TransitionTable()
{
if (transitions) {
free(transitions);
diff --git a/hfst-ol.h b/hfst-ol.h
index e8a9887..63002d6 100644
--- a/hfst-ol.h
+++ b/hfst-ol.h
@@ -22,6 +22,7 @@
#ifndef HFST_OSPELL_HFST_OL_H_
#define HFST_OSPELL_HFST_OL_H_
+#include "hfstol-stdafx.h"
#include <vector>
#include <map>
#include <climits>
@@ -35,8 +36,8 @@
namespace hfst_ol {
-typedef unsigned short SymbolNumber;
-typedef unsigned int TransitionTableIndex;
+typedef uint16_t SymbolNumber;
+typedef uint32_t TransitionTableIndex;
typedef std::vector<SymbolNumber> SymbolVector;
typedef std::vector<std::string> KeyTable;
typedef std::map<std::string, SymbolNumber> StringSymbolMap;
@@ -71,6 +72,13 @@ enum HeaderFlag {Weighted, Deterministic, Input_deterministic, Minimized,
Has_input_epsilon_transitions, Has_input_epsilon_cycles,
Has_unweighted_input_epsilon_cycles};
+// Will probably turn into a compile-time constant
+bool is_big_endian(void);
+uint16_t read_uint16_flipping_endianness(FILE * f);
+uint16_t read_uint16_flipping_endianness(char * raw);
+uint32_t read_uint32_flipping_endianness(char * raw);
+float read_float_flipping_endianness(FILE * f);
+
// Utility function for dealing with raw memory
void skip_c_string(char ** raw);
@@ -137,7 +145,7 @@ private:
const SymbolNumber feature;
const ValueNumber value;
public:
- //!
+ //!
//! Construct flag diacritic of from \@ @a op . @a feat . @a val \@.
FlagDiacriticOperation(const FlagDiacriticOperator op,
const SymbolNumber feat,
@@ -147,17 +155,17 @@ public:
// dummy constructor
FlagDiacriticOperation():
operation(P), feature(NO_SYMBOL), value(0) {}
-
+
//!
//! check if flag
bool isFlag(void) const;
//!
//! Operation something I don't understand really.
FlagDiacriticOperator Operation(void) const;
- //!
+ //!
//! No clue
SymbolNumber Feature(void) const;
- //!
+ //!
//! Not a slightest idea
ValueNumber Value(void) const;
@@ -177,30 +185,30 @@ private:
SymbolNumber orig_symbol_count;
StringSymbolMap string_to_symbol;
void process_symbol(char * line);
-
+
void read(FILE * f, SymbolNumber number_of_symbols);
void read(char ** raw, SymbolNumber number_of_symbols);
-
+
public:
- //!
+ //!
//! read alphabets from file @a f
TransducerAlphabet(FILE *f, SymbolNumber number_of_symbols);
- //!
+ //!
//! read alphabes from raw data @a raw
TransducerAlphabet(char ** raw, SymbolNumber number_of_symbols);
-
+
void add_symbol(std::string & sym);
void add_symbol(char * sym);
//!
//! get alphabet's keytable mapping
KeyTable * get_key_table(void);
- //!
+ //!
//! get flag operation map stuff
OperationMap * get_operation_map(void);
//!
//! get state's size
SymbolNumber get_state_size(void);
- //!
+ //!
//! get position of unknown symbol
SymbolNumber get_unknown(void) const;
SymbolNumber get_identity(void) const;
@@ -210,7 +218,7 @@ public:
//! get mapping from strings to symbols
StringSymbolMap * get_string_to_symbol(void);
bool has_string(std::string const & s) const;
- //!
+ //!
//! get if given symbol is a flag
bool is_flag(SymbolNumber symbol);
};
@@ -229,10 +237,10 @@ private:
public:
LetterTrie(void):
- letters(UCHAR_MAX, static_cast<LetterTrie*>(NULL)),
- symbols(UCHAR_MAX,NO_SYMBOL)
+ letters(UCHAR_MAX+1, static_cast<LetterTrie*>(NULL)),
+ symbols(UCHAR_MAX+1,NO_SYMBOL)
{}
- //!
+ //!
//! add a string to alphabets with a key
void add_string(const char * p,SymbolNumber symbol_key);
//!
@@ -254,7 +262,7 @@ private:
public:
//!
- //! create encoder from keytable
+ //! create encoder from keytable
Encoder(KeyTable * kt, SymbolNumber number_of_input_symbols);
SymbolNumber find_key(char ** p);
void read_input_symbol(const char * s, const int s_num);
@@ -271,12 +279,12 @@ class TransitionIndex
protected:
SymbolNumber input_symbol; //!< transition's input symbol
TransitionTableIndex first_transition_index; //!< first transition location
-
+
public:
-
+
//!
//! Each TransitionIndex has an input symbol and a target index.
- static const size_t SIZE =
+ static const size_t SIZE =
sizeof(SymbolNumber) + sizeof(TransitionTableIndex);
//!
@@ -313,9 +321,9 @@ protected:
public:
- //! Each transition has an input symbol, an output symbol and
+ //! Each transition has an input symbol, an output symbol and
//! a target index.
- static const size_t SIZE =
+ static const size_t SIZE =
2 * sizeof(SymbolNumber) + sizeof(TransitionTableIndex) + sizeof(Weight);
//!
@@ -337,7 +345,7 @@ public:
transition_weight(INFINITE_WEIGHT)
{}
- //!
+ //!
//! get transitions target
TransitionTableIndex target(void) const;
//!
@@ -361,12 +369,12 @@ class IndexTable
{
private:
char * indices;
-
+ TransitionTableIndex size;
void read(FILE * f,
TransitionTableIndex number_of_table_entries);
void read(char ** raw,
TransitionTableIndex number_of_table_entries);
- TransitionTableIndex size;
+ void convert_to_big_endian(void);
public:
//!
@@ -384,10 +392,10 @@ public:
//!
//! target state location for the index
TransitionTableIndex target(TransitionTableIndex i) const;
- //!
+ //!
//! whether it's final transition
bool final(TransitionTableIndex i) const;
- //!
+ //!
//! transition's weight
Weight final_weight(TransitionTableIndex i) const;
};
@@ -401,7 +409,8 @@ protected:
//!
//! raw transition data
char * transitions;
-
+ TransitionTableIndex size;
+
//!
//! read known amount of transitions from file @a f
void read(FILE * f,
@@ -409,9 +418,9 @@ protected:
//! read known amount of transitions from raw dara @a data
void read(char ** raw,
TransitionTableIndex number_of_table_entries);
- TransitionTableIndex size;
+ void convert_to_big_endian(void);
public:
- //!
+ //!
//! read transition table from file @a f
TransitionTable(FILE * f,
TransitionTableIndex transition_count);
@@ -421,10 +430,10 @@ public:
TransitionTableIndex transition_count);
~TransitionTable(void);
- //!
+ //!
//! transition's input symbol
SymbolNumber input_symbol(TransitionTableIndex i) const;
- //!
+ //!
//! transition's output symbol
SymbolNumber output_symbol(TransitionTableIndex i) const;
//!
@@ -449,5 +458,5 @@ void debug_print(printable p)
}
} // namespace hfst_ol
-
+
#endif // HFST_OSPELL_HFST_OL_H_
diff --git a/hfst-ospell-office.1 b/hfst-ospell-office.1
index 47cb4f4..40650df 100644
--- a/hfst-ospell-office.1
+++ b/hfst-ospell-office.1
@@ -13,7 +13,7 @@ Check the input as-is without any transformations
.SH "REPORTING BUGS"
Report bugs to mail at tinodidriksen.com and/or hfst\-bugs at helsinki.fi
.PP
-hfstospell 0.4.0
+hfstospell 0.4.1
.br
Jan 11 2016 12:57:25
.br
diff --git a/hfst-ospell.1 b/hfst-ospell.1
index 6646106..a57e125 100644
--- a/hfst-ospell.1
+++ b/hfst-ospell.1
@@ -51,7 +51,7 @@ Use this lexicon (must also give erro model as option)
.SH "REPORTING BUGS"
Report bugs to hfst\-bugs at helsinki.fi
.PP
-hfstospell 0.4.0
+hfstospell 0.4.1
.br
Jan 11 2016 12:57:25
.br
diff --git a/hfstol-stdafx.h b/hfstol-stdafx.h
new file mode 100644
index 0000000..b6667e7
--- /dev/null
+++ b/hfstol-stdafx.h
@@ -0,0 +1,44 @@
+/* -*- Mode: C++ -*- */
+// Copyright 2010 University of Helsinki
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HFST_OSPELL_STDAFX_H_
+#define HFST_OSPELL_STDAFX_H_
+
+#ifdef _MSC_VER
+ // warning C4512: assignment operator could not be generated
+ #pragma warning (disable: 4512)
+ // warning C4456: declaration hides previous local declaration
+ #pragma warning (disable: 4456)
+ // warning C4458: declaration hides class member
+ #pragma warning (disable: 4458)
+ // warning C4996: POSIX names deprecated
+ #pragma warning (disable: 4996)
+#endif
+
+#ifdef _WIN32
+ #ifdef LIBHFSTOSPELL_EXPORTS
+ #define OSPELL_API __declspec(dllexport)
+ #else
+ #define OSPELL_API __declspec(dllimport)
+ #endif
+#else
+ #ifdef LIBHFSTOSPELL_EXPORTS
+ #define OSPELL_API __attribute__ ((visibility ("default")))
+ #else
+ #define OSPELL_API
+ #endif
+#endif
+
+#endif
diff --git a/main-cicling.cc b/main-cicling.cc
index a22a325..67170d0 100644
--- a/main-cicling.cc
+++ b/main-cicling.cc
@@ -54,7 +54,6 @@ bool print_version(void)
std::cerr <<
"\n" <<
PACKAGE_STRING << std::endl <<
- __DATE__ << " " __TIME__ << std::endl <<
"copyright (C) 2009 University of Helsinki\n";
return true;
}
diff --git a/main-fsmnlp-2012.cc b/main-fsmnlp-2012.cc
index 06de881..3226881 100644
--- a/main-fsmnlp-2012.cc
+++ b/main-fsmnlp-2012.cc
@@ -73,7 +73,6 @@ bool print_version(void)
std::cerr <<
"\n" <<
PACKAGE_STRING << std::endl <<
- __DATE__ << " " __TIME__ << std::endl <<
"copyright (C) 2009 - 2011 University of Helsinki\n";
return true;
}
diff --git a/main-norvig.cc b/main-norvig.cc
index 7b46ef8..1b1e206 100644
--- a/main-norvig.cc
+++ b/main-norvig.cc
@@ -54,7 +54,6 @@ bool print_version(void)
std::cerr <<
"\n" <<
PACKAGE_STRING << std::endl <<
- __DATE__ << " " __TIME__ << std::endl <<
"copyright (C) 2009 University of Helsinki\n";
return true;
}
diff --git a/main-survey.cc b/main-survey.cc
index d9426a5..c690a0a 100644
--- a/main-survey.cc
+++ b/main-survey.cc
@@ -95,7 +95,6 @@ print_version(void)
std::cerr <<
"\n" <<
PACKAGE_STRING << std::endl <<
- __DATE__ << " " __TIME__ << std::endl <<
"copyright (C) 2009 - 2011 University of Helsinki\n";
return true;
}
diff --git a/main.cc b/main.cc
index dce4f96..f63b8ba 100644
--- a/main.cc
+++ b/main.cc
@@ -151,7 +151,6 @@ bool print_version(void)
std::cout <<
"\n" <<
PACKAGE_STRING << std::endl <<
- __DATE__ << " " __TIME__ << std::endl <<
"copyright (C) 2009 - 2016 University of Helsinki\n";
return true;
}
diff --git a/office.cpp b/office.cc
similarity index 99%
rename from office.cpp
rename to office.cc
index 522db86..4896b75 100644
--- a/office.cpp
+++ b/office.cc
@@ -47,8 +47,6 @@
#include <unicode/uchar.h>
#include <unicode/unistr.h>
-#include "ol-exceptions.h"
-#include "ospell.h"
#include "ZHfstOspeller.h"
using hfst_ol::ZHfstOspeller;
diff --git a/ol-exceptions.h b/ol-exceptions.h
index 2116c08..0762fa9 100644
--- a/ol-exceptions.h
+++ b/ol-exceptions.h
@@ -1,6 +1,7 @@
#ifndef _OL_EXCEPTIONS_H
#define _OL_EXCEPTIONS_H
+#include "hfstol-stdafx.h"
#include <string>
#include <sstream>
@@ -23,7 +24,7 @@ struct OspellException
//!
//! construct exception with name, file and location
-OspellException(const std::string &name,const std::string &file,size_t line):
+ OspellException(const std::string &name,const std::string &file,size_t line):
name(name),
file(file),
line(line)
diff --git a/ospell.cc b/ospell.cc
index f60c0d0..82593fa 100644
--- a/ospell.cc
+++ b/ospell.cc
@@ -37,27 +37,6 @@ int nByte_utf8(unsigned char c)
}
}
-bool
-StringWeightComparison::operator()(StringWeightPair lhs, StringWeightPair rhs)
-{ // return true when we want rhs to appear before lhs
- if (reverse) {
- return (lhs.second < rhs.second);
- } else {
- return (lhs.second > rhs.second);
- }
-}
-
-bool
-StringPairWeightComparison::operator()(StringPairWeightPair lhs,
- StringPairWeightPair rhs)
-{ // return true when we want rhs to appear before lhs
- if (reverse) {
- return (lhs.second < rhs.second);
- } else {
- return (lhs.second > rhs.second);
- }
-}
-
void WeightQueue::push(Weight w)
{
for (WeightQueue::iterator it = begin(); it != end(); ++it) {
@@ -238,7 +217,7 @@ Speller::Speller(Transducer* mutator_ptr, Transducer* lexicon_ptr):
SymbolNumber
Speller::get_state_size()
{
- return lexicon->get_state_size();
+ return static_cast<SymbolNumber>(lexicon->get_state_size());
}
@@ -280,7 +259,14 @@ void Speller::lexicon_consume(void)
// no more input
return;
}
- SymbolNumber this_input = alphabet_translator[input[input_state]];
+ SymbolNumber this_input;
+ if (mutator != NULL) {
+ this_input = alphabet_translator[input[input_state]];
+ } else {
+ // To support zhfst spellers without error models, we allow
+ // for the case with plain lexicon symbols
+ this_input = input[input_state];
+ }
if(!lexicon->has_transitions(
next_node.lexicon_state + 1, this_input)) {
// we have no regular transitions for this
@@ -681,7 +667,7 @@ bool Transducer::has_non_epsilons_or_flags(const TransitionTableIndex i)
return((this_input != 0 && this_input != NO_SYMBOL) &&
!is_flag(this_input));
} else {
- SymbolNumber max_symbol = get_key_table()->size();
+ SymbolNumber max_symbol = static_cast<SymbolNumber>(get_key_table()->size());
for (SymbolNumber sym = 1; sym < max_symbol; ++sym) {
if (indices.input_symbol(i + sym) == sym) {
return true;
@@ -756,6 +742,7 @@ Transducer::is_weighted(void)
AnalysisQueue Speller::analyse(char * line, int nbest)
{
+ (void)nbest;
mode = Lookup;
if (!init_input(line)) {
return AnalysisQueue();
@@ -1090,7 +1077,7 @@ void Speller::build_alphabet_translator(void)
// A symbol in the error source isn't present in the
// lexicon, so we add it.
std::string sym = from_keys->operator[](i);
- SymbolNumber lexicon_key = lexicon->get_key_table()->size();
+ SymbolNumber lexicon_key = static_cast<SymbolNumber>(lexicon->get_key_table()->size());
lexicon->get_encoder()->read_input_symbol(sym, lexicon_key);
lexicon->get_alphabet()->add_symbol(sym);
alphabet_translator.push_back(lexicon_key);
@@ -1115,10 +1102,14 @@ bool Speller::init_input(char * line)
SymbolNumber k = NO_SYMBOL;
char ** inpointer = &line;
char * oldpointer;
-
+
while (**inpointer != '\0') {
oldpointer = *inpointer;
- k = mutator->get_encoder()->find_key(inpointer);
+ if (mutator != NULL) {
+ k = mutator->get_encoder()->find_key(inpointer);
+ } else {
+ k = lexicon->get_encoder()->find_key(inpointer);
+ }
if (k == NO_SYMBOL) { // no tokenization from alphabet
int bytes_to_tokenize = nByte_utf8(static_cast<unsigned char>(*oldpointer));
if (bytes_to_tokenize == 0) {
@@ -1137,14 +1128,16 @@ bool Speller::init_input(char * line)
SymbolNumber k_lexicon = lexicon->get_alphabet()->get_string_to_symbol()
->operator[](new_symbol_string);
lexicon->get_encoder()->read_input_symbol(new_symbol, k_lexicon);
- if (!mutator->get_alphabet()->has_string(new_symbol_string)) {
- mutator->get_alphabet()->add_symbol(new_symbol_string);
- }
- k = mutator->get_alphabet()->get_string_to_symbol()->
- operator[](new_symbol_string);
- mutator->get_encoder()->read_input_symbol(new_symbol, k);
- if (k >= alphabet_translator.size()) {
- add_symbol_to_alphabet_translator(k_lexicon);
+ if (mutator != NULL) {
+ if (!mutator->get_alphabet()->has_string(new_symbol_string)) {
+ mutator->get_alphabet()->add_symbol(new_symbol_string);
+ }
+ k = mutator->get_alphabet()->get_string_to_symbol()->
+ operator[](new_symbol_string);
+ mutator->get_encoder()->read_input_symbol(new_symbol, k);
+ if (k >= alphabet_translator.size()) {
+ add_symbol_to_alphabet_translator(k_lexicon);
+ }
}
input.push_back(k);
continue;
diff --git a/ospell.h b/ospell.h
index a4c8a01..7bd73be 100644
--- a/ospell.h
+++ b/ospell.h
@@ -16,6 +16,7 @@
#ifndef HFST_OSPELL_OSPELL_H_
#define HFST_OSPELL_OSPELL_H_ 1
+#include "hfstol-stdafx.h"
#include <string>
#include <deque>
#include <queue>
@@ -85,7 +86,15 @@ public:
//!
//! compare two string weight pairs for weights
- bool operator() (StringWeightPair lhs, StringWeightPair rhs);
+ bool operator() (const StringWeightPair& lhs, const StringWeightPair& rhs) const
+ { // return true when we want rhs to appear before lhs
+ if (reverse) {
+ return (lhs.second < rhs.second);
+ }
+ else {
+ return (lhs.second > rhs.second);
+ }
+ }
};
//! @brief comparison for complex analysis queues
@@ -104,7 +113,15 @@ public:
//!
//! compare two analysis corrections for weights
- bool operator() (StringPairWeightPair lhs, StringPairWeightPair rhs);
+ bool operator() (const StringPairWeightPair& lhs, const StringPairWeightPair& rhs) const
+ { // return true when we want rhs to appear before lhs
+ if (reverse) {
+ return (lhs.second < rhs.second);
+ }
+ else {
+ return (lhs.second > rhs.second);
+ }
+ }
};
typedef std::priority_queue<StringWeightPair,
diff --git a/acceptor.basic.txt b/tests/acceptor.basic.txt
similarity index 100%
rename from acceptor.basic.txt
rename to tests/acceptor.basic.txt
diff --git a/analyse-spell.sh b/tests/analyse-spell.sh
similarity index 54%
rename from analyse-spell.sh
rename to tests/analyse-spell.sh
index 6b1f312..4f5dcdd 100755
--- a/analyse-spell.sh
+++ b/tests/analyse-spell.sh
@@ -1,7 +1,7 @@
#!/bin/bash
if test -x ./hfst-ospell ; then
- if ! cat $srcdir/test.strings | ./hfst-ospell -a speller_analyser.zhfst ; then
+ if ! cat $srcdir/tests/test.strings | ./hfst-ospell -a $srcdir/tests/speller_analyser.zhfst ; then
exit 1
fi
else
diff --git a/analyser.default.txt b/tests/analyser.default.txt
similarity index 100%
rename from analyser.default.txt
rename to tests/analyser.default.txt
diff --git a/bad-errormodel.sh b/tests/bad-errormodel.sh
similarity index 54%
rename from bad-errormodel.sh
rename to tests/bad-errormodel.sh
index 836ba9c..4a065f9 100755
--- a/bad-errormodel.sh
+++ b/tests/bad-errormodel.sh
@@ -1,7 +1,7 @@
#!/bin/bash
if test -x ./hfst-ospell ; then
- if ! cat $srcdir/test.strings | ./hfst-ospell -v bad_errormodel.zhfst ; then
+ if ! cat $srcdir/tests/test.strings | ./hfst-ospell -v $srcdir/tests/bad_errormodel.zhfst ; then
exit 1
fi
else
diff --git a/tests/bad_errormodel.zhfst b/tests/bad_errormodel.zhfst
new file mode 100644
index 0000000..49134ec
Binary files /dev/null and b/tests/bad_errormodel.zhfst differ
diff --git a/basic-edit1.sh b/tests/basic-edit1.sh
similarity index 55%
rename from basic-edit1.sh
rename to tests/basic-edit1.sh
index f67be34..00b6525 100755
--- a/basic-edit1.sh
+++ b/tests/basic-edit1.sh
@@ -1,7 +1,7 @@
#!/bin/bash
if test -x ./hfst-ospell ; then
- if ! cat $srcdir/test.strings | ./hfst-ospell speller_edit1.zhfst ; then
+ if ! cat $srcdir/tests/test.strings | ./hfst-ospell $srcdir/tests/speller_edit1.zhfst ; then
exit 1
fi
else
diff --git a/basic-zhfst.sh b/tests/basic-zhfst.sh
similarity index 55%
rename from basic-zhfst.sh
rename to tests/basic-zhfst.sh
index 4393dff..a455a1a 100755
--- a/basic-zhfst.sh
+++ b/tests/basic-zhfst.sh
@@ -1,7 +1,7 @@
#!/bin/bash
if test -x ./hfst-ospell ; then
- if ! cat $srcdir/test.strings | ./hfst-ospell speller_basic.zhfst ; then
+ if ! cat $srcdir/tests/test.strings | ./hfst-ospell $srcdir/tests/speller_basic.zhfst ; then
exit 1
fi
else
diff --git a/basic_test.xml b/tests/basic_test.xml
similarity index 100%
rename from basic_test.xml
rename to tests/basic_test.xml
diff --git a/empty-descriptions.sh b/tests/empty-descriptions.sh
similarity index 53%
rename from empty-descriptions.sh
rename to tests/empty-descriptions.sh
index 707779f..4d224f5 100755
--- a/empty-descriptions.sh
+++ b/tests/empty-descriptions.sh
@@ -1,7 +1,7 @@
#!/bin/bash
if test -x ./hfst-ospell ; then
- if ! cat $srcdir/test.strings | ./hfst-ospell -v empty_descriptions.zhfst ; then
+ if ! cat $srcdir/tests/test.strings | ./hfst-ospell -v $srcdir/tests/empty_descriptions.zhfst ; then
exit 1
fi
else
diff --git a/empty-locale.sh b/tests/empty-locale.sh
similarity index 55%
rename from empty-locale.sh
rename to tests/empty-locale.sh
index 009e408..19cb30b 100755
--- a/empty-locale.sh
+++ b/tests/empty-locale.sh
@@ -1,7 +1,7 @@
#!/bin/bash
if test -x ./hfst-ospell ; then
- if ! cat $srcdir/test.strings | ./hfst-ospell -v empty_locale.zhfst ; then
+ if ! cat $srcdir/tests/test.strings | ./hfst-ospell -v $srcdir/tests/empty_locale.zhfst ; then
exit 1
fi
else
diff --git a/empty-titles.sh b/tests/empty-titles.sh
similarity index 55%
rename from empty-titles.sh
rename to tests/empty-titles.sh
index 523b321..e0f9e06 100755
--- a/empty-titles.sh
+++ b/tests/empty-titles.sh
@@ -1,7 +1,7 @@
#!/bin/bash
if test -x ./hfst-ospell ; then
- if ! cat $srcdir/test.strings | ./hfst-ospell -v empty_titles.zhfst ; then
+ if ! cat $srcdir/tests/test.strings | ./hfst-ospell -v $srcdir/tests/empty_titles.zhfst ; then
exit 1
fi
else
diff --git a/empty-zhfst.sh b/tests/empty-zhfst.sh
similarity index 58%
rename from empty-zhfst.sh
rename to tests/empty-zhfst.sh
index eb2a6ab..824ec5e 100755
--- a/empty-zhfst.sh
+++ b/tests/empty-zhfst.sh
@@ -1,8 +1,9 @@
#!/bin/bash
if test -x ./hfst-ospell ; then
+ rm -f empty.zhfst
touch empty.zhfst
- if ! cat $srcdir/test.strings | ./hfst-ospell -v empty.zhfst ; then
+ if ! cat $srcdir/tests/test.strings | ./hfst-ospell -v empty.zhfst ; then
exit 1
fi
else
diff --git a/empty_descriptions.xml b/tests/empty_descriptions.xml
similarity index 100%
rename from empty_descriptions.xml
rename to tests/empty_descriptions.xml
diff --git a/tests/empty_descriptions.zhfst b/tests/empty_descriptions.zhfst
new file mode 100644
index 0000000..a512e1e
Binary files /dev/null and b/tests/empty_descriptions.zhfst differ
diff --git a/empty_locale.xml b/tests/empty_locale.xml
similarity index 100%
rename from empty_locale.xml
rename to tests/empty_locale.xml
diff --git a/tests/empty_locale.zhfst b/tests/empty_locale.zhfst
new file mode 100644
index 0000000..0de003b
Binary files /dev/null and b/tests/empty_locale.zhfst differ
diff --git a/empty_titles.xml b/tests/empty_titles.xml
similarity index 100%
rename from empty_titles.xml
rename to tests/empty_titles.xml
diff --git a/tests/empty_titles.zhfst b/tests/empty_titles.zhfst
new file mode 100644
index 0000000..5c926c6
Binary files /dev/null and b/tests/empty_titles.zhfst differ
diff --git a/errmodel.basic.txt b/tests/errmodel.basic.txt
similarity index 100%
rename from errmodel.basic.txt
rename to tests/errmodel.basic.txt
diff --git a/errmodel.edit1.txt b/tests/errmodel.edit1.txt
similarity index 100%
rename from errmodel.edit1.txt
rename to tests/errmodel.edit1.txt
diff --git a/errmodel.extrachars.txt b/tests/errmodel.extrachars.txt
similarity index 100%
rename from errmodel.extrachars.txt
rename to tests/errmodel.extrachars.txt
diff --git a/no-errormodel.sh b/tests/no-errormodel.sh
similarity index 55%
rename from no-errormodel.sh
rename to tests/no-errormodel.sh
index 2d3c5ff..84436b5 100755
--- a/no-errormodel.sh
+++ b/tests/no-errormodel.sh
@@ -1,7 +1,7 @@
#!/bin/bash
if test -x ./hfst-ospell ; then
- if ! cat $srcdir/test.strings | ./hfst-ospell no_errormodel.zhfst ; then
+ if ! cat $srcdir/tests/test.strings | ./hfst-ospell $srcdir/tests/no_errormodel.zhfst ; then
exit 1
fi
else
diff --git a/no_errmodel.xml b/tests/no_errmodel.xml
similarity index 100%
rename from no_errmodel.xml
rename to tests/no_errmodel.xml
diff --git a/tests/no_errormodel.zhfst b/tests/no_errormodel.zhfst
new file mode 100644
index 0000000..d9fb191
Binary files /dev/null and b/tests/no_errormodel.zhfst differ
diff --git a/tests/speller_analyser.zhfst b/tests/speller_analyser.zhfst
new file mode 100644
index 0000000..38f6f08
Binary files /dev/null and b/tests/speller_analyser.zhfst differ
diff --git a/tests/speller_basic.zhfst b/tests/speller_basic.zhfst
new file mode 100644
index 0000000..d76e8d4
Binary files /dev/null and b/tests/speller_basic.zhfst differ
diff --git a/tests/speller_edit1.zhfst b/tests/speller_edit1.zhfst
new file mode 100644
index 0000000..d2b2eac
Binary files /dev/null and b/tests/speller_edit1.zhfst differ
diff --git a/test.strings b/tests/test.strings
similarity index 100%
rename from test.strings
rename to tests/test.strings
diff --git a/trailing-spaces.sh b/tests/trailing-spaces.sh
similarity index 54%
rename from trailing-spaces.sh
rename to tests/trailing-spaces.sh
index e0f7b99..9cdc7d4 100755
--- a/trailing-spaces.sh
+++ b/tests/trailing-spaces.sh
@@ -1,7 +1,7 @@
#!/bin/bash
if test -x ./hfst-ospell ; then
- if ! cat $srcdir/test.strings | ./hfst-ospell -v trailing_spaces.zhfst ; then
+ if ! cat $srcdir/tests/test.strings | ./hfst-ospell -v $srcdir/tests/trailing_spaces.zhfst ; then
exit 1
fi
else
diff --git a/trailing_spaces.xml b/tests/trailing_spaces.xml
similarity index 100%
rename from trailing_spaces.xml
rename to tests/trailing_spaces.xml
diff --git a/tests/trailing_spaces.zhfst b/tests/trailing_spaces.zhfst
new file mode 100644
index 0000000..7d676e6
Binary files /dev/null and b/tests/trailing_spaces.zhfst differ
diff --git a/windows-Makefile.am b/windows-Makefile.am
index 6f3af6c..0561974 100644
--- a/windows-Makefile.am
+++ b/windows-Makefile.am
@@ -269,10 +269,10 @@ err.hfstol: errmodel.default.hfst
-ln -sf $< $@
.txt.hfst:
- hfst-txt2fst $(HFST_FLAGS) $< | hfst-fst2fst $(HFST_FLAGS) -f olw -o $@
+ hfst-txt2fst $< | hfst-fst2fst -f olw -o $@
.txt.hfstol:
- hfst-txt2fst $(HFST_FLAGS) $< | hfst-fst2fst $(HFST_FLAGS) -f olw -o $@
+ hfst-txt2fst $< | hfst-fst2fst -f olw -o $@
hfst-ospell.1: hfst-ospell
help2man --no-discard-stderr $< > $@
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/hfst-ospell.git
More information about the debian-science-commits
mailing list