[Debian-ha-commits] [resource-agents] 03/04: Imported Upstream version 3.9.6

Richard Winters devrik-guest at moszumanska.debian.org
Thu Apr 16 08:08:15 UTC 2015


This is an automated email from the git hooks/post-receive script.

devrik-guest pushed a commit to branch master
in repository resource-agents.

commit 6cae755581fa936d0a1877376c34cceaebe806e6
Author: Richard B Winters <rik at mmogp.com>
Date:   Thu Apr 16 03:55:03 2015 -0400

    Imported Upstream version 3.9.6
---
 ChangeLog                                     | 261 ++++++++++
 Makefile.am                                   |   7 +
 configure.ac                                  |  69 ++-
 doc/dev-guides/ra-dev-guide.txt               |  31 ++
 doc/man/Makefile.am                           |   6 +
 doc/man/ra2refentry.xsl                       | 141 +++++-
 heartbeat/CTDB                                |  44 +-
 heartbeat/Delay                               |   2 +-
 heartbeat/Dummy                               |   6 +
 heartbeat/Filesystem                          | 153 ++++--
 heartbeat/IPaddr2                             |  42 +-
 heartbeat/IPsrcaddr                           |  14 +-
 heartbeat/IPv6addr.c                          |   2 +-
 heartbeat/LVM                                 |  53 +-
 heartbeat/MailTo                              |   4 +-
 heartbeat/Makefile.am                         |   8 +
 heartbeat/Route                               |  22 +-
 heartbeat/SendArp                             |   4 +-
 heartbeat/Squid                               |  14 +-
 heartbeat/Stateful                            |  25 +-
 heartbeat/VirtualDomain                       | 334 ++++++++++---
 heartbeat/Xen                                 | 571 +++++++++++----------
 heartbeat/Xinetd                              |  24 +-
 heartbeat/anything                            |  34 +-
 heartbeat/apache                              |  30 +-
 heartbeat/asterisk                            |   2 +-
 heartbeat/clvm                                | 410 +++++++++++++++
 heartbeat/conntrackd                          |  16 +-
 heartbeat/db2                                 |  60 ++-
 heartbeat/dhcpd                               |  37 +-
 heartbeat/dnsupdate                           | 272 ++++++++++
 heartbeat/docker                              | 425 ++++++++++++++++
 heartbeat/ethmonitor                          | 290 +++++++----
 heartbeat/exportfs                            | 241 ++++++---
 heartbeat/findif.sh                           |  82 ++-
 heartbeat/galera                              | 691 ++++++++++++++++++++++++++
 heartbeat/iSCSILogicalUnit                    | 573 ++++++++++++---------
 heartbeat/iSCSITarget                         | 661 +++++++++++++-----------
 heartbeat/iscsi                               |  63 ++-
 heartbeat/kamailio                            | 688 +++++++++++++++++++++++++
 heartbeat/lxc                                 |  79 ++-
 heartbeat/mysql                               | 327 ++----------
 heartbeat/mysql-common.sh                     | 287 +++++++++++
 heartbeat/named                               |  16 +-
 heartbeat/nfsnotify                           | 315 ++++++++++++
 heartbeat/nfsserver                           | 506 +++++++++++++++----
 heartbeat/nginx                               |  20 +-
 heartbeat/ocf-binaries.in                     |   2 +-
 heartbeat/ocf-shellfuncs.in                   |  65 ++-
 heartbeat/oracle                              | 120 ++++-
 heartbeat/oralsnr                             |   2 +-
 heartbeat/pgsql                               | 101 ++--
 heartbeat/portblock                           |  79 ++-
 heartbeat/postfix                             |  22 +-
 heartbeat/rsyncd                              |  16 +-
 heartbeat/sg_persist                          | 673 +++++++++++++++++++++++++
 heartbeat/slapd                               |  24 +-
 heartbeat/symlink                             |  12 +-
 heartbeat/tomcat                              |  37 +-
 heartbeat/vsftpd                              | 253 ++++++++++
 include/agent_config.h.in                     |   1 +
 ldirectord/Makefile.am                        |   2 +-
 ldirectord/ldirectord.cf                      |   8 +-
 ldirectord/ldirectord.in                      |  53 +-
 ldirectord/{ => systemd}/Makefile.am          |  25 +-
 ldirectord/systemd/ldirectord.service.in      |  15 +
 resource-agents.spec.in                       |  40 +-
 rgmanager/src/resources/Makefile.am           |   5 +-
 rgmanager/src/resources/apache.metadata       |  10 +
 rgmanager/src/resources/apache.sh             |  16 +-
 rgmanager/src/resources/bind-mount.sh         | 167 +++++++
 rgmanager/src/resources/db2.sh                | 133 +++++
 rgmanager/src/resources/fs.sh.in              |   2 +-
 rgmanager/src/resources/lvm_by_vg.sh          |   2 +-
 rgmanager/src/resources/mysql.sh              |  10 +-
 rgmanager/src/resources/named.sh              |  11 +-
 rgmanager/src/resources/nfsserver.sh          |  80 ++-
 rgmanager/src/resources/oracledb.sh           |   5 +-
 rgmanager/src/resources/orainstance.sh        |   2 +-
 rgmanager/src/resources/oralistener.sh        |   6 +
 rgmanager/src/resources/postgres-8.metadata   |  10 +
 rgmanager/src/resources/postgres-8.sh         |  20 +-
 rgmanager/src/resources/utils/config-utils.sh |  42 +-
 rgmanager/src/resources/utils/fs-lib.sh       |  36 +-
 rgmanager/src/resources/vm.sh                 | 128 ++++-
 tools/ocft/Filesystem                         |   3 +
 tools/ocft/Makefile.am                        |   4 +
 tools/ocft/VirtualDomain                      |  70 +++
 tools/ocft/{apache => Xen}                    |  40 +-
 tools/ocft/apache                             |   4 +-
 tools/ocft/caselib.in                         |  12 +
 tools/ocft/{mysql => exportfs-multidir}       |  68 +--
 tools/ocft/iscsi                              |   6 +-
 tools/ocft/mysql                              |   4 +-
 tools/ocft/named                              |   4 +-
 tools/ocft/ocft.in                            |  34 +-
 tools/ocft/pgsql                              |   4 +-
 tools/ocft/sg_persist                         | 225 +++++++++
 98 files changed, 8645 insertions(+), 2030 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 3eeb3ee..3805327 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,264 @@
+* Thu Jan 29 2015 resource-agents contributors
+- stable release 3.9.6
+- VirtualDomain: add migrate_options parameter
+- VirtualDomain: enforce C locale in status
+- ocf-shellfuncs: add printenv to RA trace
+- nginx: allow different URLs for level 10 and 20 monitor
+
+* Tue Jan 20 2015 resource-agents contributors
+- release candidate 3.9.6 rc1
+- VirtualDomain: add sync_config_on_stop to sync the config to
+  other nodes
+- mysql.sh: Allow MySQL to run as user other than mysql
+- fs-lib.sh: Fix missed detection of write failure
+- iscsi: run iscsi discovery only when necessary
+- VirtualDomain: save the config before virsh undefine
+  (bnc#891915)
+- sg_persist: new RA
+- ldirectord: Add Install section for systemd unit file.
+- kamilio: new RA
+- findif.sh: Use the most specific matching route
+- LVM: volgrpname is unique (meta-data update)
+- ldirectord: Get correct user for sending email (bnc#910497)
+- iSCSILogicalUnit: do not set write_back when creating
+  backstore/block
+- portblock: Feature: reset_tcp_on_unblock_stop
+- nfsserver: prevent error messages on platforms without rpcuser
+- Xen: fix regression with xm and quoting (lf#2671)
+- lxc: re-add LXCpre1.0 logic
+- lxc: fix for missing lxc-ps command
+- Route: some unique attributes are not unique (meta-data)
+- IPaddr2 findif: accept dotted quad netmask
+- VirtualDomain: For Xen, prefer xl to xen-list (boo#901453)
+- Xen: Use xl list $domain return code in status check
+  (boo#901453)
+- pgsql: PostgreSQL 9.3 compatibility for unix_socket_directories
+- lvm_by_vg.sh: Only strip tag on stop if we are owner
+- ocf-shellfuncs: set HA_LOGD depending on HA_use_logd
+- mysql: avoid use of check_binary in common validation function.
+- mysql: report error when validation fails during monitor yet
+  pid is still active
+- docker: new RA
+- mysql: do not report success on 'stop' if validation fails
+- anything: fix output redirection
+- mysql: fix unexpected operation error that caused by MySQL
+  client timeout.
+- Middle: anything: Prevent stop failure, even if the job takes
+  time to stop.
+- doc: Add pcs to man page example section
+- multiple RA: exit reason support
+- Build: add --compat-habindir option for HA_BIN backward
+  compatibility
+- iSCSITarget: monitor operation must pass before start is
+  considered complete
+- iSCSILogicalUnit: monitor operation must pass before start is
+  considered complete
+- iSCSILogicalUnit:  Fixes the check for pre-existing LUN ACLs
+- iSCSITarget: fixes syntax error caused by targetcli update
+- Filesystem: when loading kernel modules wait for filesystem to
+  initialize
+- ethmonitor: add infiniband status monitoring support
+- Xinetd: refine exit codes when xinetd is stopped
+- Xen: Properly quote domain name in call to create
+- Xen: Use xenstore-ls to get status if available
+- Xen: Replace use of xm with xl (bnc#882548)
+- oracle: fix setting monitor user profile for other languages
+  (nls)
+- apache: Revised fix for init script reference on SUSE
+  (bnc#884674)
+- VirtualDomain: Try xenstore-ls if no emulator is set
+  (bnc#885292)
+- galera: new RA
+- VirtualDomain: Add support for qemu-dm as emulator (bnc#885292)
+- CTDB: add ctdb_rundir parameter and create on startup
+- VirtualDomain: new parameter save_config_on_stop to enable
+  saving running VM's configuration
+- fs-lib.sh: Force kill processes with access to shared libraries
+  on mount point
+- iSCSILogicalUnit: add targetcli support
+- Filesystem: Add force_unmount option
+- nfsserver: regenerate statd state file before starting nfs
+- nfsnotify: new RA
+- nfsserver: introducing nfs_no_notify option for suppressing
+  reboot notifications
+- nfsserver: Keep statd directories synced with ha backup
+- oracle: reset MONUSR password if expired
+- oracle: try as sysdba if monitor as MONUSR fails
+- oracle: Make monitoring user configurable (bnc#850589)
+- oralistener.sh: Do not fail during stop if listener has already
+  stopped
+- nfsserver: Add options for explicitly setting nfs daemon ports
+- vsftpd: new RA
+- fs-lib.sh: Fix usage of findmnt to work for filesystems
+  containing bind mounts
+- exportfs: allow multiple exports
+- ldirectord: Update .spec file with systemd support (bnc#863250)
+- ldirectord: Add systemd unit file (bnc#863250)
+- nfsserver: Do not require shared info directory when cloned
+- nfsserver: Allow dynamically setting rpc.nfsd options.
+- iscsi: iscsi status fails with open-iscsi with support for
+  flash (bnc#878039)
+- Med: rgmanager/fs: typo preventing passing some mount opts
+- nfsserver.sh: add an optional option for the rpc.statd
+  listening port
+- Feature: addition of bind-mount resource agent for rgmanager
+- exportfs: Do not fail during monitor and stop operations if dir
+  has trailing slash
+- nfsserver: Do not fail monitor if lock services are enabled
+- nfsserver: let systemd service files handle lock notifications
+- vm.sh: 'no_kill' option for preventing auto destruction of vm
+  after timeout period
+- vm.sh: Monitor kvm resources without requiring libvirtd to be
+  up
+- nfsserver: preserve statd directory permissions during
+  sm-notify or else lock recovery fails
+- kamailio: new RA
+- nfsserver.sh: On stop, kill sm-notify so /var/lib/nfs/statd can
+  be unmounted
+- db2: Report OCF_ERR_GENERIC instead of OCF_NOT_RUNNING when
+  instance is not completely up
+- named.sh: Fix issues running named daemon as a non-root user
+- db2.sh: rgmanager wrapper for the heartbeat db2 agent
+- CTDB: do not fail monitor operation when ctdb socket does not
+  exist
+- exportfs: set unlock_on_stop_default=1 (bnc#864263)
+- dnsupdate: new RA
+- clvm: new RA
+- mysql: handle $secs_behind = NULL
+- dhcpd: Added a restart-function
+- high: conntrackd: allow probe to return OCF_RUNNING_MASTER
+- VirtualDomain: check process table for qemu-system-* too, not
+  just qemu-kvm
+- VirtualDomain: avoid running "virsh uri" if hypervisor is set
+- apache.sh: Add ability to set custom httpd binary
+- db2: Allow db2 agent to work without crm_master binary
+- VirtualDomain: Attempt to determine vm status even when libvirt
+  is unavailable
+- VirtualDomain: Fixes parsing domain name from xml file.
+- fs-lib.sh: Fixes failure to unmount local fs when process runs
+  with cwd inside fs mount
+- ldirectord: Fix sockaddr_in6 redefined error
+- Pure-FTPd: Create pid directory if needed
+- VirtualDomain: support more virsh domstate output formats
+- varnish: Added support for ulimit -l and ulimit -n
+- lxc: support up-to-date lxc-ps versions
+- tomcat: Override default tomcat config with resource options
+- nfsserver: nfsserver not starting due to missing etab file
+- tomcat: Avoid unnecessary force kill of tomcat on stop
+- tomcat: Avoid race condition in reading pid file on stop
+- slapd: find the correct default slapd config in fedora
+- tomcat: Detect start script location
+- pgsql: Support for non-standard port and library locations
+- fix netfs unmount/self_fence integration
+- Med: oracledb.sh: Remove quotes around listener name
+- IPaddr2/findif.sh: Do a sanity check only on start and
+  validate-all
+- ldirectord: Disable HTTPS SSL certificate hostname checking
+- tomcat: Monitor rotatelogs process and restart when it is
+  stopped
+- jboss: Monitor rotatelogs process and restart when it is
+  stopped
+- mysql: Fix for the issue of detecting an unconfigured slave
+  with empty master_host since setting empty master_host is not
+  allowed with 5.5
+- IPaddr2/findif.sh: Robust parameter checking for 'nic'
+- VirtualDomain: Ensure it is possible to manage a libvirt domain
+  defined outside of VirtualDomain
+- VirtualDomain: Fix ability to use default libvirt hypervisor
+  without explicitly setting agent attribute
+- tomcat-6.sh: Fixes setting TOMCAT_USER correctly
+- tomcat-s.sh: Do not fail on stop if config validation fails.
+- fs-lib.sh: Removes usage of fuser -kvm from fs-lib.sh based
+  agents.
+- ldirectord: Use an alarm for LDAP check to ensure it times out
+- exportfs: stop with no directory should succeed
+- Xen: retry domain lookup in repeating monitor and stop
+- Fixes lvm metadata corruption caused when activating by lv
+  using tags.
+- ldirector: fix using service name instead of port number
+  (bnc#836759)
+- Filesystem: remove SLES10 compatibility code
+- Raid1: disallow md raid arrays as clone resources to avoid data
+  corruption
+- apache: put back config file existence test
+- VirtualDomain: Do not attempt graceful shutdown if force_stop
+  is enabled
+- Med: oracledb.sh: Fix process name grep in exit_idle
+- VirtualDomain: use virsh create instead of define to start it
+  non-persistent
+- eDir88: multiple IP support
+- apache: better handling of not installed apache
+- jboss: add jboss_base_dir parameter to support multi-instances
+  on JBoss 6
+- jboss: stop the JBoss daemon by sending a signal on JBoss 6
+- jboss: add run_command parameter to change the start up script
+- jboss: add jboss_version parameter to support JBoss 6
+- tomcat: use root as the default for tomcat_user instead of
+  RUNASIS mode
+- tomcat: multiple tomcat instances based on CATALINE_BASE
+- pound: add parameter maxfiles to set ulimit
+- apache: remove unnecessary and imperfect checks from
+  validate_all (bnc#827927)
+- pgsql: set only one node into sync mode when using 3 nodes or
+  higher
+- Med: oracledb.sh: Set RESTART_RETRIES back to 0
+- Raid1: stop arrays even with block device file missing
+  (bnc#821861)
+- oracle/oralsnr: use /bin/sh in sudo calls (bnc#825517)
+- apache: Properly check meta_timeout variable during graceful
+  stop
+- Raid1: set MDADM_NO_UDEV appropriately if udev is not used
+- Raid1: wait for udevd to settle (bnc#821861)
+- iSCSILogicalUnit: add tgt specific parameters bstype, bsoflags,
+  device_type
+- apache: Attempt graceful stop before -TERM signal
+- LVM: Warn user if initrd is older than lvm.conf when using
+  exclusive activation with tags
+- LVM: Exclusive activation without clvmd using filtering with
+  tags.
+- named: Attempt to autogen /etc/rndc.key using rndc-confgen tool
+- mysql: really use log setting (bnc#823095)
+- mysql: test properly for failed process start (bnc#823095)
+- Med: Don't preserve SELinux context when copying files to
+  /var/lib/nfs/sm
+- Med: Cleanup oracledb.sh rgmanager agent and add support for
+  Oracle 11g
+- Med: Cleanup oralistener.sh and add support for Oracle 11g
+- Med: Cleanup orainstance.sh and add support for Oracle 11g
+- fs-lib.sh: Faster filesystem start/stop through use of
+  'findmnt' command for 'is_mounted' function
+- named.sh: Addition of update-source option.
+- LVM: Retry exclusive activation after deactivating vg
+  cluster-wide
+- LVM: Retry deactivating vg allowing udev to settle
+- ip.sh: Fixes usage of ipv6 addresses with uppercase lettering
+- LVM: Verify setup on start plus verify exclusive activation is
+  possible.
+- Route: add IPv6 support
+- build: Place resource state information in /var/run/... by
+  default
+- DRBD: remove deprecated drbd agent.
+- fs-lib.sh: Always honor self_fence option when
+  force_unmount="on"
+- postgres-8: Shutdown postgres with SIGINT before forcing
+  SIGQUIT
+- tools: send_arp.libnet: reuse ARP packets (debian#701914)
+- ethmonitor: correctly detect when the network is unplugged.
+- Raid1: do not test for device existence in the stop operation
+  (bnc#821861)
+- mysql: Attempt to auto-detect mysql binary default location.
+- VirtualDomain: Support saving and restoring virtual machine
+  snapshot state files
+- nfsserver: Cleanup shared nfs dir mount
+- nfsserver: Enable file locking daemon when systemd is being
+  used
+- nfsserver: Maintain SELinux permissions on failover for nfs v3
+  lock state
+- nfsserver: Improve support for v3 file lock recovery
+- nfsserver: Add systemd unit-file support when init scripts are
+  not present
+- VirtualDomain: Properly detect defined lxc domains
+
 * Thu Feb  7 2013 Linux-HA contributors
 - stable release 3.9.5
 - IPaddr2: support nic:iflabel format in nic parameter
diff --git a/Makefile.am b/Makefile.am
index 0d75a9a..1769c6e 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -43,6 +43,12 @@ else
 LINUX_HA = with
 endif
 
+if WITH_COMPAT_HABINDIR
+COMPAT_HABINDIR = without
+else
+COMPAT_HABINDIR = with
+endif
+
 EXTRA_DIST		= autogen.sh .version make/release.mk \
 			  make/git-version-gen make/gitlog-to-changelog \
 			  AUTHORS COPYING COPYING.GPLv3 COPYING.LGPL ChangeLog \
@@ -142,6 +148,7 @@ $(SPEC): $(SPEC).in
 		-e "s#@rcver@#$$rcver#g" \
 		-e "s#@rgmanager@#$(RGMANAGER)#g" \
 		-e "s#@linux-ha@#$(LINUX_HA)#g" \
+		-e "s#@compat-habindir@#$(COMPAT_HABINDIR)#g" \
 	$< > $@-t; \
 	if [ -z "$$dirty" ]; then sed -i -e "s#%glo.*dirty.*##g" $@-t; fi; \
 	if [ -z "$$alphatag" ]; then sed -i -e "s#%glo.*alphatag.*##g" $@-t; fi; \
diff --git a/configure.ac b/configure.ac
index 9e55960..ec3ec6e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -56,6 +56,26 @@ AC_ARG_WITH(pkg-name,
     [  --with-pkg-name=name     Override package name (if you're a packager needing to pretend) ],
     [ PACKAGE_NAME="$withval" ])
 
+AC_PATH_PROGS(PKGCONFIG, pkg-config)
+if test x"${PKGCONFIG}" = x""; then
+   AC_MSG_ERROR(You need pkgconfig installed in order to build ${PACKAGE})
+fi
+
+AC_ARG_WITH([systemdsystemunitdir],
+     [AS_HELP_STRING([--with-systemdsystemunitdir=DIR], [Directory for systemd service files])],,
+     [with_systemdsystemunitdir=auto])
+AS_IF([test "x$with_systemdsystemunitdir" = "xyes" -o "x$with_systemdsystemunitdir" = "xauto"], [
+     def_systemdsystemunitdir=$($PKGCONFIG --variable=systemdsystemunitdir systemd)
+
+     AS_IF([test "x$def_systemdsystemunitdir" = "x"],
+         [AS_IF([test "x$with_systemdsystemunitdir" = "xyes"],
+                [AC_MSG_ERROR([systemd support requested but pkg-config unable to query systemd package])])
+          with_systemdsystemunitdir=no],
+         [with_systemdsystemunitdir="$def_systemdsystemunitdir"])])
+AS_IF([test "x$with_systemdsystemunitdir" != "xno"],
+      [AC_SUBST([systemdsystemunitdir], [$with_systemdsystemunitdir])])
+AM_CONDITIONAL([HAVE_SYSTEMD], [test "x$with_systemdsystemunitdir" != "xno"])
+
 dnl 
 dnl AM_INIT_AUTOMAKE([1.11.1 foreign dist-bzip2 dist-xz])
 dnl
@@ -170,13 +190,11 @@ AC_ARG_WITH(initdir,
 OCF_ROOT_DIR="/usr/lib/ocf"
 AC_ARG_WITH(ocf-root,
     [  --with-ocf-root=DIR      directory for OCF scripts [${OCF_ROOT_DIR}]],
-    [ if test x"$withval" = xprefix; then OCF_ROOT_DIR=${prefix}; else
-	 OCF_ROOT_DIR="$withval"; fi ])
+    [ OCF_ROOT_DIR="$withval" ])
 HA_RSCTMPDIR=${localstatedir}/run/resource-agents
 AC_ARG_WITH(rsctmpdir,
     [  --with-rsctmpdir=DIR      directory for resource agents state files [${HA_RSCTMPDIR}]],
-    [ if test x"$withval" = xprefix; then HA_RSCTMPDIR=${prefix}; else
-	 HA_RSCTMPDIR="$withval"; fi ])
+    [ HA_RSCTMPDIR="$withval" ])
 
 AC_ARG_ENABLE([libnet],
  [  --enable-libnet 	Use libnet for ARP based funcationality, [default=try]], 
@@ -211,6 +229,13 @@ fi
 AM_CONDITIONAL(BUILD_LINUX_HA, test $BUILD_LINUX_HA -eq 1)
 AM_CONDITIONAL(BUILD_RGMANAGER, test $BUILD_RGMANAGER -eq 1)
 
+AC_ARG_WITH(compat-habindir,
+    [  --with-compat-habindir      use HA_BIN directory with compatibility for the Heartbeat stack [${libexecdir}]],
+    [],
+    [with_compat_habindir=no])
+AM_CONDITIONAL(WITH_COMPAT_HABINDIR, test "x$with_compat_habindir" != "xno")
+
+
 dnl ===============================================
 dnl General Processing
 dnl ===============================================
@@ -275,6 +300,10 @@ case $libdir in
     ;;
 esac
 
+if test "x$with_compat_habindir" != "xno" ; then
+  libexecdir=${libdir}
+fi
+
 dnl Expand autoconf variables so that we dont end up with '${prefix}' 
 dnl in #defines and python scripts
 dnl NOTE: Autoconf deliberately leaves them unexpanded to allow
@@ -333,6 +362,7 @@ case "$host_os" in
 *solaris*)
 		REBOOT_OPTIONS="-n"
 		POWEROFF_OPTIONS="-n"
+		LDFLAGS+=" -lssp -lssp_nonshared"
 		;;
 *linux*)	
 		AC_DEFINE_UNQUOTED(ON_LINUX, 1, Compiling for Linux platform)
@@ -401,26 +431,18 @@ HA_VARLIBHBDIR=${localstatedir}/lib/heartbeat
 AC_DEFINE_UNQUOTED(HA_VARLIBHBDIR,"$HA_VARLIBHBDIR", Whatever this used to mean)
 AC_SUBST(HA_VARLIBHBDIR)
 
-OCF_RA_DIR="${OCF_ROOT_DIR}/resource.d/"
+OCF_RA_DIR="${OCF_ROOT_DIR}/resource.d"
 AC_DEFINE_UNQUOTED(OCF_RA_DIR,"$OCF_RA_DIR", Location for OCF RAs)
 AC_SUBST(OCF_RA_DIR)
 
-if test "${prefix}" = "/usr"; then
- OCF_RA_DIR_PREFIX="$OCF_RA_DIR"
-else
- OCF_RA_DIR_PREFIX="${prefix}/$OCF_RA_DIR"
-fi
+OCF_RA_DIR_PREFIX="$OCF_RA_DIR"
 AC_SUBST(OCF_RA_DIR_PREFIX)
 
-OCF_LIB_DIR="${OCF_ROOT_DIR}/lib/"
+OCF_LIB_DIR="${OCF_ROOT_DIR}/lib"
 AC_DEFINE_UNQUOTED(OCF_LIB_DIR,"$OCF_LIB_DIR", Location for shared code for OCF RAs)
 AC_SUBST(OCF_LIB_DIR)
 
-if test "${prefix}" = "/usr"; then
- OCF_LIB_DIR_PREFIX="$OCF_LIB_DIR"
-else
- OCF_LIB_DIR_PREFIX="${prefix}/$OCF_LIB_DIR"
-fi
+OCF_LIB_DIR_PREFIX="$OCF_LIB_DIR"
 AC_SUBST(OCF_LIB_DIR_PREFIX)
 
 dnl ===============================================
@@ -451,7 +473,6 @@ AC_PATH_PROGS(PING, ping, /bin/ping)
 AC_PATH_PROGS(IFCONFIG, ifconfig, /sbin/ifconfig)
 AC_PATH_PROGS(MAILCMD, mailx mail, mail)
 AC_PATH_PROGS(EGREP, egrep)
-AC_PATH_PROGS(PKGCONFIG, pkg-config)
 
 AC_SUBST(MAILCMD)
 AC_SUBST(EGREP)
@@ -487,10 +508,6 @@ dnl ===============================================
 AC_CHECK_LIB(socket, socket)			
 AC_CHECK_LIB(gnugetopt, getopt_long)		dnl if available
 
-if test x"${PKGCONFIG}" = x""; then
-   AC_MSG_ERROR(You need pkgconfig installed in order to build ${PACKAGE})
-fi
-
 if test "x${enable_thread_safe}" = "xyes"; then
         GPKGNAME="gthread-2.0"
 else
@@ -730,8 +747,7 @@ dnl to sub processes.
 CC_ERRORS=""
 CC_EXTRAS=""
 
-if export | fgrep " CFLAGS=" > /dev/null; then
-	export -n CFLAGS || true # We don't want to bomb out if this fails
+if export -p | fgrep " CFLAGS=" > /dev/null; then
 	SAVED_CFLAGS="$CFLAGS"
 	unset CFLAGS
 	CFLAGS="$SAVED_CFLAGS"
@@ -768,7 +784,7 @@ else
 		-Wno-strict-aliasing
 		-Wpointer-arith 
 		-Wstrict-prototypes
-    		-Wunsigned-char
+		-Wunsigned-char
 		-Wwrite-strings"
 
 # Additional warnings it might be nice to enable one day
@@ -861,6 +877,8 @@ ldirectord/ldirectord   					\
 	ldirectord/init.d/ldirectord   				\
 	ldirectord/init.d/ldirectord.debian			\
 	ldirectord/init.d/ldirectord.debian.default		\
+	ldirectord/systemd/Makefile	\
+	ldirectord/systemd/ldirectord.service	\
 	ldirectord/logrotate.d/Makefile				\
 	ldirectord/OCF/Makefile					\
 	ldirectord/OCF/ldirectord				\
@@ -883,7 +901,7 @@ dnl *****************
 AC_MSG_RESULT([])
 AC_MSG_RESULT([$PACKAGE configuration:])
 AC_MSG_RESULT([  Version                  = ${VERSION}])
-AC_MSG_RESULT([  Build Version            = 32dd79a119834cd3482b01d4bba623e7b8c0d8b4])
+AC_MSG_RESULT([  Build Version            = 02beac55c1da0ad99a5a19bd3b2333bcff7e916c])
 AC_MSG_RESULT([  Features                 =${PKG_FEATURES}])
 AC_MSG_RESULT([])
 AC_MSG_RESULT([  Prefix                   = ${prefix}])
@@ -895,6 +913,7 @@ AC_MSG_RESULT([  Arch-independent files   = ${datadir}])
 AC_MSG_RESULT([  Documentation            = ${docdir}])
 AC_MSG_RESULT([  State information        = ${localstatedir}])
 AC_MSG_RESULT([  System configuration     = ${sysconfdir}])
+AC_MSG_RESULT([  HA_BIN directory prefix  = ${libexecdir}])
 AC_MSG_RESULT([  RA state files           = ${HA_RSCTMPDIR}])
 AC_MSG_RESULT([  AIS Plugins              = ${LCRSODIR}])
 AC_MSG_RESULT([])
diff --git a/doc/dev-guides/ra-dev-guide.txt b/doc/dev-guides/ra-dev-guide.txt
index 11e9a5d..7191bdd 100644
--- a/doc/dev-guides/ra-dev-guide.txt
+++ b/doc/dev-guides/ra-dev-guide.txt
@@ -69,6 +69,9 @@ Meta parameters are passed into the resource agent in the
 underscores. Thus, the +target-role+ attribute maps to an environment
 variable named +OCF_RESKEY_CRM_meta_target_role+.
 
+The <<_script_variables>> section contains other system environment
+variables.
+
 === Actions
 
 Any resource agent must support one command-line argument which
@@ -93,6 +96,7 @@ actions:
 * +usage+ or +help+ -- displays a usage message when the resource
   agent is invoked from the command line, rather than by the cluster
   manager.
+* +notify+ -- inform resource about changes in state of other clones.
 * +status+ -- historical (deprecated) synonym for +monitor+.
 
 === Timeouts
@@ -156,6 +160,7 @@ OCF Resource Agent Developers Guide.
     <action name="stop"         timeout="20" />
     <action name="monitor"      timeout="20"
                                 interval="10" depth="0" />
+    <action name="notify"       timeout="20" />
     <action name="reload"       timeout="20" />
     <action name="migrate_to"   timeout="20" />
     <action name="migrate_from" timeout="20" />
@@ -450,6 +455,7 @@ stop)		foobar_stop;;
 status|monitor)	foobar_monitor;;
 promote)	foobar_promote;;
 demote)		foobar_demote;;
+notify)		foobar_notify;;
 reload)		ocf_log info "Reloading..."
 	        foobar_start
 		;;
@@ -1093,6 +1099,16 @@ primarily for convenience purposes. For additional variables
 available while the agent is being executed, refer to
 <<_environment_variables>> and <<_return_codes>>.
 
+=== +$OCF_RA_VERSION_MAJOR+
+
+The major version number of the resource agent API that the cluster
+manager is currently using.
+
+=== +$OCF_RA_VERSION_MINOR+
+
+The minor version number of the resource agent API that the cluster
+manager is currently using.
+
 === +$OCF_ROOT+
 
 The root of the OCF resource agent hierarchy. This should never be
@@ -1106,6 +1122,12 @@ The directory where the resource agents shell function library,
 variable may, however, be overridden from the command line while
 testing a new or modified resource agent.
 
+=== +$OCF_EXIT_REASON_PREFIX+
+
+Used as a prefix when printing error messages from the resource agent.
+Script functions use this automaticly so no explicit use is required
+for shell based scripts.
+
 === +$OCF_RESOURCE_INSTANCE+
 
 The resource instance name. For primitive (non-clone, non-stateful)
@@ -1113,6 +1135,15 @@ resources, this is simply the resource name. For clones and stateful
 resources, this is the primitive name, followed by a colon an the
 clone instance number (such as +p_foobar:0+).
 
+=== +$OCF_RESOURCE_TYPE+
+
+The resource type of the current resource, e.g. IPaddr2.
+
+=== +$OCF_RESOURCE_PROVIDER+
+
+The resource provider, e.g. heartbeat. This may not be in all cluster
+managers of Resource Agent API version 1.0.
+
 === +$__OCF_ACTION+
 
 The currently invoked action. This is exactly the first command-line
diff --git a/doc/man/Makefile.am b/doc/man/Makefile.am
index 3bf569a..e082bac 100644
--- a/doc/man/Makefile.am
+++ b/doc/man/Makefile.am
@@ -94,13 +94,16 @@ man_MANS	       = ocf_heartbeat_AoEtarget.7 \
                           ocf_heartbeat_anything.7 \
                           ocf_heartbeat_apache.7 \
                           ocf_heartbeat_asterisk.7 \
+                          ocf_heartbeat_clvm.7 \
                           ocf_heartbeat_conntrackd.7 \
                           ocf_heartbeat_db2.7 \
                           ocf_heartbeat_dhcpd.7 \
+                          ocf_heartbeat_docker.7 \
                           ocf_heartbeat_eDir88.7 \
                           ocf_heartbeat_ethmonitor.7 \
                           ocf_heartbeat_exportfs.7 \
                           ocf_heartbeat_fio.7 \
+                          ocf_heartbeat_galera.7 \
                           ocf_heartbeat_iSCSILogicalUnit.7 \
                           ocf_heartbeat_iSCSITarget.7 \
                           ocf_heartbeat_iface-bridge.7 \
@@ -108,10 +111,12 @@ man_MANS	       = ocf_heartbeat_AoEtarget.7 \
                           ocf_heartbeat_ids.7 \
                           ocf_heartbeat_iscsi.7 \
                           ocf_heartbeat_jboss.7 \
+                          ocf_heartbeat_kamailio.7 \
                           ocf_heartbeat_lxc.7 \
                           ocf_heartbeat_mysql.7 \
                           ocf_heartbeat_mysql-proxy.7 \
                           ocf_heartbeat_named.7 \
+                          ocf_heartbeat_nfsnotify.7 \
                           ocf_heartbeat_nfsserver.7 \
                           ocf_heartbeat_nginx.7 \
                           ocf_heartbeat_oracle.7 \
@@ -127,6 +132,7 @@ man_MANS	       = ocf_heartbeat_AoEtarget.7 \
                           ocf_heartbeat_scsi2reservation.7 \
                           ocf_heartbeat_sfex.7 \
                           ocf_heartbeat_slapd.7 \
+                          ocf_heartbeat_sg_persist.7 \
                           ocf_heartbeat_symlink.7 \
                           ocf_heartbeat_syslog-ng.7 \
                           ocf_heartbeat_tomcat.7 \
diff --git a/doc/man/ra2refentry.xsl b/doc/man/ra2refentry.xsl
index 41a60aa..ac148ef 100644
--- a/doc/man/ra2refentry.xsl
+++ b/doc/man/ra2refentry.xsl
@@ -50,7 +50,8 @@
   <xsl:apply-templates select="$this" mode="description"/>
   <xsl:apply-templates select="$this" mode="parameters"/>
   <xsl:apply-templates select="$this" mode="actions"/>
-  <xsl:apply-templates select="$this" mode="example"/>
+  <xsl:apply-templates select="$this" mode="examplecrmsh"/>
+  <xsl:apply-templates select="$this" mode="examplepcs"/>
   <xsl:apply-templates select="$this" mode="seealso"/>
  </xsl:template>
 
@@ -403,10 +404,10 @@
   </xsl:template>
 
 
-  <!-- Mode Example -->
-  <xsl:template match="resource-agent" mode="example">
+  <!-- Mode Example CRM Shell-->
+  <xsl:template match="resource-agent" mode="examplecrmsh">
     <refsection>
-      <title>Example</title>
+      <title>Example CRM Shell</title>
       <para>
 	<xsl:text>The following is an example configuration for a </xsl:text>
 	<xsl:value-of select="@name"/>
@@ -428,7 +429,7 @@
 	    <xsl:text> \
   params \
 </xsl:text>
-	    <xsl:apply-templates select="parameters" mode="example"/>
+	    <xsl:apply-templates select="parameters" mode="examplecrmsh"/>
 	  </xsl:when>
 	  <xsl:otherwise>
 	  <xsl:value-of select="@name"/><xsl:text> \</xsl:text>
@@ -440,7 +441,7 @@
 	  <xsl:text>
   meta allow-migrate="true" \</xsl:text>
 	</xsl:if>
-	<xsl:apply-templates select="actions" mode="example"/>
+	<xsl:apply-templates select="actions" mode="examplecrmsh"/>
       </programlisting>
       <!-- Insert a master/slave set definition if the resource
       agent supports promotion and demotion -->
@@ -457,15 +458,15 @@
     </refsection>
   </xsl:template>
 
-  <xsl:template match="parameters" mode="example">
-    <xsl:apply-templates select="parameter[@required = 1]" mode="example"/>
+  <xsl:template match="parameters" mode="examplecrmsh">
+    <xsl:apply-templates select="parameter[@required = 1]" mode="examplecrmsh"/>
   </xsl:template>
 
-  <xsl:template match="parameter" mode="example">
+  <xsl:template match="parameter" mode="examplecrmsh">
     <xsl:text>    </xsl:text>
     <xsl:value-of select="@name"/>
     <xsl:text>=</xsl:text>
-    <xsl:apply-templates select="content" mode="example"/>
+    <xsl:apply-templates select="content" mode="examplecrmsh"/>
     <xsl:text> \</xsl:text>
     <xsl:if test="following-sibling::parameter/@required = 1">
       <xsl:text>
@@ -473,7 +474,7 @@
     </xsl:if>
   </xsl:template>
 
-  <xsl:template match="content" mode="example">
+  <xsl:template match="content" mode="examplecrmsh">
     <xsl:choose>
       <xsl:when test="@default != ''">
 	<xsl:text>"</xsl:text>
@@ -486,23 +487,23 @@
     </xsl:choose>
   </xsl:template>
 
-  <xsl:template match="actions" mode="example">
+  <xsl:template match="actions" mode="examplecrmsh">
     <!-- In the CRM shell example, show only the monitor action -->
-    <xsl:apply-templates select="action[@name = 'monitor']" mode="example"/>
+    <xsl:apply-templates select="action[@name = 'monitor']" mode="examplecrmsh"/>
   </xsl:template>
 
-  <xsl:template match="action" mode="example">
+  <xsl:template match="action" mode="examplecrmsh">
     <xsl:text>
   op </xsl:text>
     <xsl:value-of select="@name"/>
     <xsl:text> </xsl:text>
-    <xsl:apply-templates select="@*" mode="example"/>
+    <xsl:apply-templates select="@*" mode="examplecrmsh"/>
     <xsl:if test="following-sibling::action/@name = 'monitor'">
       <xsl:text>\</xsl:text>
     </xsl:if>
   </xsl:template>
 
-  <xsl:template match="action/@*" mode="example">
+  <xsl:template match="action/@*" mode="examplecrmsh">
     <xsl:choose>
       <xsl:when test="name() = 'name'"><!-- suppress --></xsl:when>
       <xsl:otherwise>
@@ -517,9 +518,113 @@
     </xsl:if>
   </xsl:template>
 
-  <xsl:template match="longdesc" mode="example"/>
+  <xsl:template match="longdesc" mode="examplecrmsh"/>
 
-  <xsl:template match="shortdesc" mode="example"/>
+  <xsl:template match="shortdesc" mode="examplecrmsh"/>
+
+  <!-- Mode Example PCS-->
+  <xsl:template match="resource-agent" mode="examplepcs">
+    <refsection>
+      <title>Example PCS</title>
+      <para>
+	<xsl:text>The following is an example configuration for a </xsl:text>
+	<xsl:value-of select="@name"/>
+	<xsl:text> resource using </xsl:text>
+	<citerefentry><refentrytitle>pcs</refentrytitle><manvolnum>8</manvolnum></citerefentry>
+      </para>
+      <programlisting>
+	<xsl:text>pcs resource create p_</xsl:text>
+	<xsl:value-of select="@name"/>
+	<xsl:text> </xsl:text>
+	<xsl:value-of select="$class"/>
+	<xsl:text>:</xsl:text>
+	<xsl:value-of select="$provider"/>
+	<xsl:text>:</xsl:text>
+	<xsl:choose>
+	  <xsl:when test="parameters/parameter[@required = 1]">
+	    <xsl:value-of select="@name"/>
+	    <xsl:text> \
+</xsl:text>
+	    <xsl:apply-templates select="parameters" mode="examplepcs"/>
+	  </xsl:when>
+	  <xsl:otherwise>
+	  <xsl:value-of select="@name"/><xsl:text> \</xsl:text>
+	  </xsl:otherwise>
+	</xsl:choose>
+	<xsl:apply-templates select="actions" mode="examplepcs"/>
+
+     <!-- Insert a master/slave set definition if the resource
+      agent supports promotion and demotion -->
+      <xsl:if test="actions/action/@name = 'promote' and actions/action/@name = 'demote'">
+	  <xsl:text>--master</xsl:text>
+      </xsl:if>
+      </programlisting>
+
+    </refsection>
+  </xsl:template>
+
+  <xsl:template match="parameters" mode="examplepcs">
+    <xsl:apply-templates select="parameter[@required = 1]" mode="examplepcs"/>
+  </xsl:template>
+
+  <xsl:template match="parameter" mode="examplepcs">
+    <xsl:text>  </xsl:text>
+    <xsl:value-of select="@name"/>
+    <xsl:text>=</xsl:text>
+    <xsl:apply-templates select="content" mode="examplepcs"/>
+    <xsl:text> \</xsl:text>
+    <xsl:if test="following-sibling::parameter/@required = 1">
+      <xsl:text>
+</xsl:text>
+    </xsl:if>
+  </xsl:template>
+
+  <xsl:template match="content" mode="examplepcs">
+    <xsl:choose>
+      <xsl:when test="@default != ''">
+	<xsl:text>"</xsl:text>
+	<xsl:value-of select="@default"/>
+	<xsl:text>"</xsl:text>
+      </xsl:when>
+      <xsl:otherwise>
+	<replaceable><xsl:value-of select="@type"/></replaceable>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+  <xsl:template match="actions" mode="examplepcs">
+    <!-- In the CRM shell example, show only the monitor action -->
+    <xsl:apply-templates select="action[@name = 'monitor']" mode="examplepcs"/>
+  </xsl:template>
+
+  <xsl:template match="action" mode="examplepcs">
+    <xsl:text>
+  op </xsl:text>
+    <xsl:value-of select="@name"/>
+    <xsl:text> </xsl:text>
+    <xsl:apply-templates select="@*" mode="examplepcs"/>
+    <xsl:if test="following-sibling::action/@name = 'monitor'">
+      <xsl:text>\</xsl:text>
+    </xsl:if>
+  </xsl:template>
+
+  <xsl:template match="action/@*" mode="examplepcs">
+    <xsl:choose>
+      <xsl:when test="name() = 'name'"><!-- suppress --></xsl:when>
+      <xsl:otherwise>
+	<xsl:value-of select="name()"/>
+	<xsl:text>="</xsl:text>
+	<xsl:value-of select="current()"/>
+	<xsl:text>" </xsl:text>
+      </xsl:otherwise>
+    </xsl:choose>
+    <xsl:if test="following-sibling::*">
+      <xsl:text> </xsl:text>
+    </xsl:if>
+  </xsl:template>
+
+  <xsl:template match="longdesc" mode="examplepcs"/>
+  <xsl:template match="shortdesc" mode="examplepcs"/>
 
   <xsl:template match="resource-agent" mode="seealso">
     <refsection>
diff --git a/heartbeat/CTDB b/heartbeat/CTDB
index 8c82322..3419b54 100755
--- a/heartbeat/CTDB
+++ b/heartbeat/CTDB
@@ -80,6 +80,11 @@ if [ ! -d "$var_prefix" ] && [ -d "/var/ctdb" ]; then
 	var_prefix="/var/ctdb"
 fi
 
+run_prefix="/run"
+if [ ! -d "$var_prefix" ] && [ -d "/var/run" ]; then
+	var_prefix="/var/run"
+fi
+
 : ${OCF_RESKEY_ctdb_manages_samba:=no}
 : ${OCF_RESKEY_ctdb_manages_winbind:=no}
 : ${OCF_RESKEY_ctdb_service_smb:=""}
@@ -95,6 +100,7 @@ fi
 : ${OCF_RESKEY_ctdb_socket:=${var_prefix}/ctdb.socket}
 : ${OCF_RESKEY_ctdb_dbdir:=${var_prefix}}
 : ${OCF_RESKEY_ctdb_logfile:=/var/log/ctdb/log.ctdb}
+: ${OCF_RESKEY_ctdb_rundir:=${run_prefix}/ctdb}
 : ${OCF_RESKEY_ctdb_debuglevel:=2}
 
 : ${OCF_RESKEY_smb_conf:=/etc/samba/smb.conf}
@@ -265,6 +271,15 @@ value "syslog".
 <content type="string" default="/var/log/ctdb/log.ctdb" />
 </parameter>
 
+<parameter name="ctdb_rundir" unique="0" required="0">
+<longdesc lang="en">
+Full path to ctdb runtime directory, used for storage of socket
+lock state.
+</longdesc>
+<shortdesc lang="en">CTDB runtime directory location</shortdesc>
+<content type="string" default="${OCF_RESKEY_ctdb_rundir}" />
+</parameter>
+
 <parameter name="ctdb_debuglevel" unique="0" required="0">
 <longdesc lang="en">
 What debug level to run at (0-10). Higher means more verbose.
@@ -529,7 +544,7 @@ ctdb_start() {
 	mkdir -p $persistent_db_dir 2>/dev/null
 	for pdbase in $(ls $persistent_db_dir/*.tdb.[0-9] 2>/dev/null$) ; do
 		/usr/bin/tdbdump $pdbase >/dev/null 2>/dev/null || {
-			ocf_log err "Persistent database $pdbase is corrupted!  CTDB will not start."
+			ocf_exit_reason "Persistent database $pdbase is corrupted!  CTDB will not start."
 			return $OCF_ERR_GENERIC
 		}
 	done
@@ -537,7 +552,7 @@ ctdb_start() {
 	# Add necessary configuration to smb.conf
 	init_smb_conf
 	if [ $? -ne 0 ]; then
-		ocf_log err "Failed to update $OCF_RESKEY_smb_conf."
+		ocf_exit_reason "Failed to update $OCF_RESKEY_smb_conf."
 		return $OCF_ERR_GENERIC
 	fi
 
@@ -554,6 +569,9 @@ ctdb_start() {
 		mkdir -p $(dirname $OCF_RESKEY_ctdb_logfile)
 	fi
 
+	# ensure ctdb's rundir exists, otherwise it will fail to start
+	mkdir -p $OCF_RESKEY_ctdb_rundir 2>/dev/null
+
 	# public addresses file (should not be present, but need to set for correctness if it is)
 	local pub_addr_option=""
 	[ -f "${OCF_RESKEY_ctdb_config_dir}/public_addresses" ] && \
@@ -578,7 +596,7 @@ ctdb_start() {
 		# cleanup smb.conf
 		cleanup_smb_conf
 		
-		ocf_log err "Failed to execute $OCF_RESKEY_ctdbd_binary."
+		ocf_exit_reason "Failed to execute $OCF_RESKEY_ctdbd_binary."
 		return $OCF_ERR_GENERIC
 	else
 		# Wait a bit for CTDB to stabilize
@@ -592,7 +610,7 @@ ctdb_start() {
 			if [ $? -ne 0 ]; then
 				# CTDB will be running, kill it before returning
 				ctdb_stop
-				ocf_log err "Can't invoke $OCF_RESKEY_ctdb_binary --socket=$OCF_RESKEY_ctdb_socket status"
+				ocf_exit_reason "Can't invoke $OCF_RESKEY_ctdb_binary --socket=$OCF_RESKEY_ctdb_socket status"
 				return $OCF_ERR_GENERIC
 			fi
 			if ! echo $status | grep -qs 'UNHEALTHY (THIS'; then
@@ -608,7 +626,7 @@ ctdb_start() {
 	# ctdbd will (or can) actually still be running at this point, so kill it
 	ctdb_stop
 	
-	ocf_log err "Timeout waiting for CTDB to stabilize"
+	ocf_exit_reason "Timeout waiting for CTDB to stabilize"
 	return $OCF_ERR_GENERIC
 }
 
@@ -660,8 +678,10 @@ ctdb_monitor() {
 	if [ $? -ne 0 ]; then
 		if echo $status | grep -qs 'Connection refused'; then
 			return $OCF_NOT_RUNNING
+		elif echo $status | grep -qs 'No such file or directory'; then
+			return $OCF_NOT_RUNNING
 		else
-			ocf_log err "CTDB status call failed: $status"
+			ocf_exit_reason "CTDB status call failed: $status"
 			return $OCF_ERR_GENERIC
 		fi
 	fi
@@ -669,7 +689,7 @@ ctdb_monitor() {
 		return $OCF_SUCCESS
 	fi
 
-	ocf_log err "CTDB status is bad: $status"
+	ocf_exit_reason "CTDB status is bad: $status"
 	return $OCF_ERR_GENERIC
 }
 
@@ -682,12 +702,12 @@ ctdb_validate() {
 	done
 
 	if [ -z "$CTDB_SYSCONFIG" ]; then
-		ocf_log err "Can't find CTDB config file (expecting /etc/sysconfig/ctdb, /etc/default/ctdb or similar)"
+		ocf_exit_reason "Can't find CTDB config file (expecting /etc/sysconfig/ctdb, /etc/default/ctdb or similar)"
 		return $OCF_ERR_INSTALLED
 	fi
 
 	if ocf_is_true "$OCF_RESKEY_ctdb_manages_samba" && [ ! -f "$OCF_RESKEY_smb_conf" ]; then
-		ocf_log err "Samba config file '$OCF_RESKEY_smb_conf' does not exist."
+		ocf_exit_reason "Samba config file '$OCF_RESKEY_smb_conf' does not exist."
 		return $OCF_ERR_INSTALLED
 	fi
 
@@ -696,19 +716,19 @@ ctdb_validate() {
 	fi
 
 	if [ ! -f "$OCF_RESKEY_ctdb_config_dir/nodes" ]; then
-		ocf_log err "$OCF_RESKEY_ctdb_config_dir/nodes does not exist."
+		ocf_exit_reason "$OCF_RESKEY_ctdb_config_dir/nodes does not exist."
 		return $OCF_ERR_ARGS
 	fi
 
 	if [ -z "$OCF_RESKEY_ctdb_recovery_lock" ]; then
-		ocf_log err "ctdb_recovery_lock not specified."
+		ocf_exit_reason "ctdb_recovery_lock not specified."
 		return $OCF_ERR_CONFIGURED
 	fi
 
 	lock_dir=$(dirname "$OCF_RESKEY_ctdb_recovery_lock")
 	touch "$lock_dir/$$" 2>/dev/null
 	if [ $? != 0 ]; then
-		ocf_log err "Directory for lock file '$OCF_RESKEY_ctdb_recovery_lock' does not exist, or is not writable."
+		ocf_exit_reason "Directory for lock file '$OCF_RESKEY_ctdb_recovery_lock' does not exist, or is not writable."
 		return $OCF_ERR_ARGS
 	fi
 	rm "$lock_dir/$$"
diff --git a/heartbeat/Delay b/heartbeat/Delay
index f505391..9cfa939 100755
--- a/heartbeat/Delay
+++ b/heartbeat/Delay
@@ -184,7 +184,7 @@ Delay_Validate_All() {
 # _Return_ on validation success
 	return $OCF_SUCCESS
   else
-	echo "Some of the instance parameters are invalid"
+	ocf_exit_reason "Some of the instance parameters are invalid"
 # _Exit_ on validation failure
 	exit $OCF_ERR_ARGS
   fi
diff --git a/heartbeat/Dummy b/heartbeat/Dummy
index dd79814..a0929fa 100755
--- a/heartbeat/Dummy
+++ b/heartbeat/Dummy
@@ -126,6 +126,11 @@ dummy_monitor() {
 	if false ; then
 		return $OCF_ERR_GENERIC
 	fi
+
+	if ! ocf_is_probe && [ "$__OCF_ACTION" = "monitor" ]; then
+		# set exit string only when NOT_RUNNING occurs during an actual monitor operation.
+		ocf_exit_reason "No process state file found"
+	fi
 	return $OCF_NOT_RUNNING
 }
 
@@ -135,6 +140,7 @@ dummy_validate() {
     state_dir=`dirname "$OCF_RESKEY_state"`
     touch "$state_dir/$$"
     if [ $? != 0 ]; then
+	ocf_exit_reason "State file \"$OCF_RESKEY_state\" is not writable"
 	return $OCF_ERR_ARGS
     fi
     rm "$state_dir/$$"
diff --git a/heartbeat/Filesystem b/heartbeat/Filesystem
index 1902c62..cefe52b 100755
--- a/heartbeat/Filesystem
+++ b/heartbeat/Filesystem
@@ -196,6 +196,26 @@ Only set this to "true" if you know what you are doing!
 <content type="boolean" default="false" />
 </parameter>
 
+<parameter name="force_unmount">
+<longdesc lang="en">
+This option allows specifying how to handle processes that are
+currently accessing the mount directory.
+
+"true"  : Default value, kill processes accessing mount point
+"safe"  : Kill processes accessing mount point using methods that
+          avoid functions that could potentially block during process
+          detection 
+"false" : Do not kill any processes.
+
+The 'safe' option uses shell logic to walk the /procs/ directory
+for pids using the mount point while the default option uses the
+fuser cli tool. fuser is known to perform operations that can potentially
+block if unresponsive nfs mounts are in use on the system.
+</longdesc>
+<shortdesc lang="en">Kill processes before unmount</shortdesc>
+<content type="boolean" default="true" />
+</parameter>
+
 </parameters>
 
 <actions>
@@ -318,6 +338,58 @@ is_fsck_needed() {
 	esac
 }
 
+fstype_supported()
+{
+	local support="$FSTYPE"
+	local rc
+
+	if [ "X${HOSTOS}" != "XOpenBSD" ];then
+		# skip checking /proc/filesystems for obsd
+		return $OCF_SUCCESS
+	fi
+
+	if [ -z "$FSTYPE" -o "$FSTYPE" = none ]; then
+		: No FSTYPE specified, rely on the system has the right file-system support already 
+		return $OCF_SUCCESS
+	fi
+
+	# support fuse-filesystems (e.g. GlusterFS)
+	case $FSTYPE in
+		glusterfs) support="fuse";;
+	esac
+
+	grep -w "$support"'$' /proc/filesystems >/dev/null
+	if [ $? -eq 0 ]; then
+		# found the fs type
+		return $OCF_SUCCESS
+	fi
+
+	# if here, we should attempt to load the module and then
+	# check the if the filesystem support exists again.
+	$MODPROBE $support >/dev/null
+	if [ $? -ne 0 ]; then
+		ocf_exit_reason "Couldn't find filesystem $FSTYPE in /proc/filesystems and failed to load kernal module"
+		return $OCF_ERR_INSTALLED
+	fi
+
+	# It is possible for the module to load and not be complete initialized
+	# before we check /proc/filesystems again. Give this a few trys before
+	# giving up entirely.
+	for try in $(seq 5); do
+		grep -w "$support"'$' /proc/filesystems >/dev/null
+		if [ $? -eq 0 ] ; then
+			# yes. found the filesystem after doing the modprobe
+			return $OCF_SUCCESS
+		fi
+		ocf_log debug "Unable to find support for $FSTYPE in /proc/filesystems after modprobe, trying again"
+		sleep 1
+	done
+
+	ocf_exit_reason "Couldn't find filesystem $FSTYPE in /proc/filesystems"
+	return $OCF_ERR_INSTALLED
+}
+
+
 #
 # START: Start up the filesystem
 #
@@ -329,24 +401,7 @@ Filesystem_start()
 		return $OCF_SUCCESS
 	fi
 
-	if [ "X${HOSTOS}" != "XOpenBSD" ];then
-		if [ -z "$FSTYPE" -o "$FSTYPE" = none ]; then
-			: No FSTYPE specified, rely on the system has the right file-system support already 
-		else
-			local support="$FSTYPE"
-			# support fuse-filesystems (e.g. GlusterFS)
-			case $FSTYPE in
-				glusterfs) support="fuse";;
-			esac
-			grep -w "$support"'$' /proc/filesystems >/dev/null ||
-				$MODPROBE $support >/dev/null
-			grep -w "$support"'$' /proc/filesystems >/dev/null
-			if [ $? -ne 0 ] ; then
-				ocf_log err "Couldn't find filesystem $FSTYPE in /proc/filesystems"
-				return $OCF_ERR_INSTALLED
-			fi
-		fi
-	fi
+	fstype_supported || exit $OCF_ERR_INSTALLED
 
 	# Check the filesystem & auto repair.  
 	# NOTE: Some filesystem types don't need this step...  Please modify
@@ -354,7 +409,7 @@ Filesystem_start()
 
 	if [ $blockdevice = "yes" ]; then
 		if [ "$DEVICE" != "/dev/null" -a ! -b "$DEVICE" ] ; then
-			ocf_log err "Couldn't find device [$DEVICE]. Expected /dev/??? to exist"
+			ocf_exit_reason "Couldn't find device [$DEVICE]. Expected /dev/??? to exist"
 			exit $OCF_ERR_INSTALLED
 		fi
 
@@ -369,7 +424,7 @@ Filesystem_start()
 			# NOTE: if any errors at all are detected, it returns non-zero
 			# if the error is >= 4 then there is a big problem
 			if [ $? -ge 4 ]; then
-				ocf_log err "Couldn't sucessfully fsck filesystem for $DEVICE"
+				ocf_exit_reason "Couldn't sucessfully fsck filesystem for $DEVICE"
 				return $OCF_ERR_GENERIC
 			fi
 		fi
@@ -378,7 +433,7 @@ Filesystem_start()
 	[ -d "$MOUNTPOINT" ] ||
 		ocf_run mkdir -p $MOUNTPOINT
 	if [ ! -d "$MOUNTPOINT" ] ; then
-		ocf_log err "Couldn't find directory  [$MOUNTPOINT] to use as a mount point"
+		ocf_exit_reason "Couldn't find directory  [$MOUNTPOINT] to use as a mount point"
 		exit $OCF_ERR_INSTALLED
 	fi
 
@@ -393,13 +448,32 @@ Filesystem_start()
 	esac
 
 	if [ $? -ne 0 ]; then
-		ocf_log err "Couldn't mount filesystem $DEVICE on $MOUNTPOINT"
+		ocf_exit_reason "Couldn't mount filesystem $DEVICE on $MOUNTPOINT"
 		return $OCF_ERR_GENERIC
 	fi
 	return $OCF_SUCCESS
 }
 # end of Filesystem_start
 
+get_pids()
+{
+	local dir=$1
+	local procs
+	local mmap_procs
+
+	if ocf_is_true  "$FORCE_UNMOUNT"; then
+		if [ "X${HOSTOS}" = "XOpenBSD" ];then
+			fstat | grep $dir | awk '{print $3}'
+		else
+			$FUSER -m $dir 2>/dev/null
+		fi
+	elif [ "$FORCE_UNMOUNT" = "safe" ]; then
+		procs=$(find /proc/[0-9]*/ -type l -lname "${dir}/*" -or -lname "${dir}" 2>/dev/null | awk -F/ '{print $3}')
+		mmap_procs=$(grep " ${dir}" /proc/[0-9]*/maps | awk -F/ '{print $3}')
+		printf "${procs}\n${mmap_procs}" | sort | uniq
+	fi
+}
+
 signal_processes() {
 	local dir=$1
 	local sig=$2
@@ -407,15 +481,9 @@ signal_processes() {
 	# fuser returns a non-zero return code if none of the
 	# specified files is accessed or in case of a fatal 
 	# error.
-	pids=$(
-		if [ "X${HOSTOS}" = "XOpenBSD" ];then
-			fstat | grep $dir | awk '{print $3}'
-		else
-			$FUSER -m $dir 2>/dev/null
-		fi
-	)
+	pids=$(get_pids "$dir")
 	if [ -z "$pids" ]; then
-		ocf_log info "No processes on $dir were signalled"
+		ocf_log info "No processes on $dir were signalled. force_unmount is set to '$FORCE_UNMOUNT'"
 		return
 	fi
 	for pid in $pids; do
@@ -439,7 +507,7 @@ fs_stop() {
 		while [ $cnt -gt 0 ]; do
 			try_umount $SUB &&
 				return $OCF_SUCCESS
-			ocf_log err "Couldn't unmount $SUB; trying cleanup with $sig"
+			ocf_exit_reason "Couldn't unmount $SUB; trying cleanup with $sig"
 			signal_processes $SUB $sig
 			cnt=$((cnt-1))
 			sleep 1
@@ -490,7 +558,7 @@ Filesystem_stop()
 			fs_stop $SUB $timeout
 			rc=$?
 			if [ $rc -ne $OCF_SUCCESS ]; then
-				ocf_log err "Couldn't unmount $SUB, giving up!"
+				ocf_exit_reason "Couldn't unmount $SUB, giving up!"
 			fi
 		done
 	fi
@@ -543,7 +611,7 @@ Filesystem_monitor_10()
 	dd_opts="iflag=direct bs=4k count=1"
 	err_output=`dd if=$DEVICE $dd_opts 2>&1 >/dev/null`
 	if [ $? -ne 0 ]; then
-		ocf_log err "Failed to read device $DEVICE"
+		ocf_exit_reason "Failed to read device $DEVICE"
 		ocf_log err "dd said: $err_output"
 		return $OCF_ERR_GENERIC
 	fi
@@ -566,18 +634,18 @@ Filesystem_monitor_20()
 	[ -d "$status_dir" ] || mkdir -p "$status_dir"
 	err_output=`echo "${OCF_RESOURCE_INSTANCE}" | dd of=${STATUSFILE} $dd_opts 2>&1`
 	if [ $? -ne 0 ]; then
-		ocf_log err "Failed to write status file ${STATUSFILE}"
+		ocf_exit_reason "Failed to write status file ${STATUSFILE}"
 		ocf_log err "dd said: $err_output"
 		return $OCF_ERR_GENERIC
 	fi
 	test -f ${STATUSFILE}
 	if [ $? -ne 0 ]; then
-		ocf_log err "Cannot stat the status file ${STATUSFILE}"
+		ocf_exit_reason "Cannot stat the status file ${STATUSFILE}"
 		return $OCF_ERR_GENERIC
 	fi
 	cat ${STATUSFILE} > /dev/null
 	if [ $? -ne 0 ]; then
-		ocf_log err "Cannot read the status file ${STATUSFILE}"
+		ocf_exit_reason "Cannot read the status file ${STATUSFILE}"
 		return $OCF_ERR_GENERIC
 	fi
 	return $OCF_SUCCESS
@@ -595,7 +663,7 @@ Filesystem_monitor()
 		10) Filesystem_monitor_10; rc=$?;;
 		20) Filesystem_monitor_20; rc=$?;;
 		*)
-			ocf_log err "unsupported monitor level $OCF_CHECK_LEVEL"
+			ocf_exit_reason "unsupported monitor level $OCF_CHECK_LEVEL"
 			rc=$OCF_ERR_CONFIGURED
 		;;
 		esac
@@ -687,6 +755,11 @@ if [ $# -ne 1 ]; then
 fi
 
 # Check the OCF_RESKEY_ environment variables...
+FORCE_UNMOUNT="yes"
+if [ -n "${OCF_RESKEY_force_unmount}" ]; then
+	FORCE_UNMOUNT=$OCF_RESKEY_force_unmount
+fi
+
 DEVICE=$OCF_RESKEY_device
 FSTYPE=$OCF_RESKEY_fstype
 if [ ! -z "$OCF_RESKEY_options" ]; then
@@ -707,7 +780,7 @@ case $OP in
 esac
 
 if [ x = x"$DEVICE" ]; then
-	ocf_log err "Please set OCF_RESKEY_device to the device to be managed"
+	ocf_exit_reason "Please set OCF_RESKEY_device to the device to be managed"
 	exit $OCF_ERR_CONFIGURED
 fi
 
@@ -719,7 +792,7 @@ set_blockdevice_var
 # But the output of `mount` and /proc/mounts do not.
 if [ -z "$OCF_RESKEY_directory" ]; then
 	if [ X$OP = "Xstart" -o $blockdevice = "no" ]; then
-		ocf_log err "Please specify the directory"
+		ocf_exit_reason "Please specify the directory"
 		exit $OCF_ERR_CONFIGURED 
 	fi
 else
@@ -781,7 +854,7 @@ esac
 if ocf_is_clone; then
 	case $CLUSTERSAFE in
 	0)
-		ocf_log err "DANGER! $FSTYPE on $DEVICE is NOT cluster-aware!"
+		ocf_exit_reason "DANGER! $FSTYPE on $DEVICE is NOT cluster-aware!"
 		ocf_log err "DO NOT RUN IT AS A CLONE!"
 		ocf_log err "Politely refusing to proceed to avoid data corruption."
 		exit $OCF_ERR_CONFIGURED
diff --git a/heartbeat/IPaddr2 b/heartbeat/IPaddr2
index 014744d..a580683 100755
--- a/heartbeat/IPaddr2
+++ b/heartbeat/IPaddr2
@@ -345,12 +345,12 @@ ip_init() {
 	local rc
 
 	if [ X`uname -s` != "XLinux" ]; then
-		ocf_log err "IPaddr2 only supported Linux."
+		ocf_exit_reason "IPaddr2 only supported Linux."
 		exit $OCF_ERR_INSTALLED
 	fi
 
 	if [ X"$OCF_RESKEY_ip" = "X" ]; then
-		ocf_log err "IP address (the ip parameter) is mandatory"
+		ocf_exit_reason "IP address (the ip parameter) is mandatory"
 		exit $OCF_ERR_CONFIGURED
 	fi
 
@@ -362,7 +362,7 @@ ip_init() {
         then
           : YAY!
         else
-		ocf_log err "You must be root for $__OCF_ACTION operation."
+		ocf_exit_reason "You must be root for $__OCF_ACTION operation."
 		exit $OCF_ERR_PERM
 	fi
 
@@ -385,14 +385,14 @@ ip_init() {
 	IP_INC_NO=`expr ${OCF_RESKEY_CRM_meta_clone:-0} + 1`
 
 	if ocf_is_true ${OCF_RESKEY_lvs_support} && [ $IP_INC_GLOBAL -gt 1 ]; then
-		ocf_log err "LVS and load sharing do not go together well"
+		ocf_exit_reason "LVS and load sharing do not go together well"
 		exit $OCF_ERR_CONFIGURED
 	fi
 
 	if ocf_is_decimal "$IP_INC_GLOBAL" && [ $IP_INC_GLOBAL -gt 0 ]; then
 		:
 	else
-		ocf_log err "Invalid OCF_RESKEY_incarnations_max_global [$IP_INC_GLOBAL], should be positive integer"
+		ocf_exit_reason "Invalid meta-attribute clone_max [$IP_INC_GLOBAL], should be positive integer"
 		exit $OCF_ERR_CONFIGURED
 	fi
 
@@ -400,20 +400,20 @@ ip_init() {
 	if [ $? -ne 0 ];then
 		FAMILY=inet
 		if ocf_is_true $OCF_RESKEY_lvs_ipv6_addrlabel ;then
-			ocf_log err "IPv4 does not support lvs_ipv6_addrlabel"
+			ocf_exit_reason "IPv4 does not support lvs_ipv6_addrlabel"
 			exit $OCF_ERR_CONFIGURED
 		fi
 	else
 		FAMILY=inet6
 		if ocf_is_true $OCF_RESKEY_lvs_support ;then
-			ocf_log err "The IPv6 does not support lvs_support"
+			ocf_exit_reason "The IPv6 does not support lvs_support"
 			exit $OCF_ERR_CONFIGURED
 		fi
 		if ocf_is_true $OCF_RESKEY_lvs_ipv6_addrlabel ;then
 			if ocf_is_decimal "$OCF_RESKEY_lvs_ipv6_addrlabel_value" && [ $OCF_RESKEY_lvs_ipv6_addrlabel_value -ge 0 ]; then
 			    :
 			else
-			    ocf_log err "Invalid lvs_ipv6_addrlabel_value [$OCF_RESKEY_lvs_ipv6_addrlabel_value], should be positive integer"
+			    ocf_exit_reason "Invalid lvs_ipv6_addrlabel_value [$OCF_RESKEY_lvs_ipv6_addrlabel_value], should be positive integer"
 			    exit $OCF_ERR_CONFIGURED
 			fi
 		fi
@@ -449,7 +449,7 @@ ip_init() {
 			ocf_log warn "[$FINDIF] failed"
 			exit $OCF_SUCCESS
 		else
-			ocf_log err "[$FINDIF] failed"
+			ocf_exit_reason "[$FINDIF] failed"
 			exit $rc
 		fi
 	fi
@@ -459,7 +459,7 @@ ip_init() {
 	if [ -n "$IFLABEL" ]; then
 		IFLABEL=${NIC}:${IFLABEL}
 		if [ ${#IFLABEL} -gt 15 ]; then
-			ocf_log err "Interface label [$IFLABEL] exceeds maximum character limit of 15"
+			ocf_exit_reason "Interface label [$IFLABEL] exceeds maximum character limit of 15"
 			exit $OCF_ERR_CONFIGURED
 		fi
 	fi
@@ -776,7 +776,8 @@ END
 }
 
 ip_start() {
-	if [ -z "$NIC" ]; then # no nic found or specified
+	if [ -z "$NIC" ]; then
+		ocf_exit_reason "No nic found or specified"
 		exit $OCF_ERR_CONFIGURED
 	fi
 
@@ -806,7 +807,7 @@ ip_start() {
 				--local-node $IP_INC_NO \
 				--hashmode $IP_CIP_HASH
 		if [ $? -ne 0 ]; then
-			ocf_log err "iptables failed"
+			ocf_exit_reason "iptables failed"
 			exit $OCF_ERR_GENERIC
 		fi
 	fi
@@ -829,7 +830,7 @@ ip_start() {
 		add_interface $OCF_RESKEY_ip $NETMASK ${BRDCAST:-none} $NIC $IFLABEL
 		
 		if [ $? -ne 0 ]; then
-			ocf_log err "$CMD failed."
+			ocf_exit_reason "$CMD failed."
 			exit $OCF_ERR_GENERIC
 		fi
 	fi
@@ -904,6 +905,7 @@ ip_stop() {
 	if [ "$ip_del_if" = "yes" ]; then
 		delete_interface $OCF_RESKEY_ip $NIC $NETMASK
 		if [ $? -ne 0 ]; then
+			ocf_exit_reason "Unable to remove IP [${OCF_RESKEY_ip} from interface [ $NIC ]"
 			exit $OCF_ERR_GENERIC
 		fi
 	
@@ -947,7 +949,7 @@ set_send_arp_program() {
 	    ARP_SEND_FUN=run_send_ib_arp
 	;;
 	*)
-	    ocf_log err "unrecognized arp_sender value: $OCF_RESKEY_arp_sender"
+	    ocf_exit_reason "unrecognized arp_sender value: $OCF_RESKEY_arp_sender"
 	    exit $OCF_ERR_CONFIGURED
 	;;
 	esac
@@ -982,21 +984,21 @@ ip_validate() {
 
     if ocf_is_true "$OCF_RESKEY_unique_clone_address" &&
 	    ! ocf_is_true "$OCF_RESKEY_CRM_meta_globally_unique"; then
-	ocf_log err "unique_clone_address makes sense only with meta globally_unique set"
+	ocf_exit_reason "unique_clone_address makes sense only with meta globally_unique set"
 	exit $OCF_ERR_CONFIGURED
     fi
 
     if ocf_is_decimal "$OCF_RESKEY_arp_interval" && [ $OCF_RESKEY_arp_interval -gt 0 ]; then
 	:
     else
-	ocf_log err "Invalid OCF_RESKEY_arp_interval [$OCF_RESKEY_arp_interval]"
+	ocf_exit_reason "Invalid OCF_RESKEY_arp_interval [$OCF_RESKEY_arp_interval]"
 	exit $OCF_ERR_CONFIGURED
     fi
 
     if ocf_is_decimal "$OCF_RESKEY_arp_count" && [ $OCF_RESKEY_arp_count -gt 0 ]; then
 	:
     else
-	ocf_log err "Invalid OCF_RESKEY_arp_count [$OCF_RESKEY_arp_count]"
+	ocf_exit_reason "Invalid OCF_RESKEY_arp_count [$OCF_RESKEY_arp_count]"
 	exit $OCF_ERR_CONFIGURED
     fi
 
@@ -1008,13 +1010,13 @@ ip_validate() {
 	sourceip|sourceip-sourceport|sourceip-sourceport-destport)
 		;;
 	*)
-		ocf_log err "Invalid OCF_RESKEY_clusterip_hash [$IP_CIP_HASH]"
+		ocf_exit_reason "Invalid OCF_RESKEY_clusterip_hash [$IP_CIP_HASH]"
 		exit $OCF_ERR_CONFIGURED
 		;;
 	esac
 
 	if ocf_is_true ${OCF_RESKEY_lvs_support}; then
-		ecf_log err "LVS and load sharing not advised to try"
+		ocf_exit_reason "LVS and load sharing not advised to try"
 		exit $OCF_ERR_CONFIGURED
 	fi
 
@@ -1027,7 +1029,7 @@ ip_validate() {
 	esac
 
 	if [ $valid -eq 0 ]; then
-	  ocf_log err "Invalid IF_MAC [$IF_MAC]"
+	  ocf_exit_reason "Invalid IF_MAC [$IF_MAC]"
 	  exit $OCF_ERR_CONFIGURED
 	fi
 
diff --git a/heartbeat/IPsrcaddr b/heartbeat/IPsrcaddr
index e8c0f77..8163c0c 100755
--- a/heartbeat/IPsrcaddr
+++ b/heartbeat/IPsrcaddr
@@ -111,7 +111,7 @@ END
 }
 
 errorexit() {
-	ocf_log err "$*"
+	ocf_exit_reason "$*"
 	exit $OCF_ERR_GENERIC
 }
 
@@ -253,7 +253,7 @@ CheckIP() {
 	    ( [ $1 -le 254 ] && [ $2 -le 254 ] && [ $3 -le 254 ] && [ $4 -le 254 ] )
 	then
 	    if [ $1 -eq 127 ]; then
-		ocf_log err "IP address [$ip] is a loopback address, thus can not be preferred source address"
+		ocf_exit_reason "IP address [$ip] is a loopback address, thus can not be preferred source address"
 		exit $OCF_ERR_CONFIGURED
 	    fi
 	else
@@ -368,7 +368,7 @@ ip_status() {
 
 	  case $IF in
 	  	lo*)  
-		    ocf_log err "IP address [$BASEIP] is served by loopback, thus can not be preferred source address"
+		    ocf_exit_reason "IP address [$BASEIP] is served by loopback, thus can not be preferred source address"
 		    exit $OCF_ERR_CONFIGURED
 		    ;;
 		*)return $OCF_SUCCESS;;
@@ -394,7 +394,7 @@ srca_validate_all() {
 	if CheckIP "$ipaddress"; then
 	  : 
 	else
-	  ocf_log err "Invalid IP address [$ipaddress]"
+	  ocf_exit_reason "Invalid IP address [$ipaddress]"
 	  exit $OCF_ERR_CONFIGURED
 	fi
 
@@ -406,7 +406,7 @@ srca_validate_all() {
 	if ip_status "$ipaddress"; then
 	  :
 	else
-	  ocf_log err "We are not serving [$ipaddress], hence can not make it a preferred source address"
+	  ocf_exit_reason "We are not serving [$ipaddress], hence can not make it a preferred source address"
 	  exit $OCF_ERR_INSTALLED
 	fi
 }
@@ -434,7 +434,7 @@ if
   [ -z "$OCF_RESKEY_ipaddress" ]
 then
 #  usage
-  ocf_log err "Please set OCF_RESKEY_ipaddress to the preferred source IP address!"
+  ocf_exit_reason "Please set OCF_RESKEY_ipaddress to the preferred source IP address!"
   exit $OCF_ERR_CONFIGURED
 fi
 
@@ -447,7 +447,7 @@ fi
 findif_out=`$FINDIF -C`
 rc=$?
 [ $rc -ne 0 ] && {
-    ocf_log err "[$FINDIF -C] failed"
+    ocf_exit_reason "[$FINDIF -C] failed"
     exit $rc
 }
 
diff --git a/heartbeat/IPv6addr.c b/heartbeat/IPv6addr.c
index 7c1d20d..255ce21 100644
--- a/heartbeat/IPv6addr.c
+++ b/heartbeat/IPv6addr.c
@@ -796,7 +796,7 @@ write_pid_file(const char *pid_file)
 			return -1;
 		}
 
-		cl_log(LOG_INFO, "Killed old send_arp process [%lu]", pid);
+		cl_log(LOG_INFO, "Killed old send_ua process [%lu]", pid);
 	}
 
 	if (snprintf(pidbuf, sizeof(pidbuf), "%u"
diff --git a/heartbeat/LVM b/heartbeat/LVM
index ec1ee7d..a32435b 100755
--- a/heartbeat/LVM
+++ b/heartbeat/LVM
@@ -68,7 +68,7 @@ as an HA resource.
 <shortdesc lang="en">Controls the availability of an LVM Volume Group</shortdesc>
 
 <parameters>
-<parameter name="volgrpname" unique="0" required="1">
+<parameter name="volgrpname" unique="1" required="1">
 <longdesc lang="en">
 The name of volume group.
 </longdesc>
@@ -175,8 +175,8 @@ verify_tags_environment()
 	# guarantee our tag will be filtered on startup
 	##
 	if ! lvm dumpconfig activation/volume_list; then
-		ocf_log err "LVM: Improper setup detected"
-		ocf_log err "The volume_list filter must be initialized in lvm.conf for exclusive activation without clvmd"
+		ocf_log err  "LVM: Improper setup detected"
+		ocf_exit_reason "The volume_list filter must be initialized in lvm.conf for exclusive activation without clvmd"
 		return $OCF_ERR_GENERIC
 	fi
 
@@ -187,7 +187,7 @@ verify_tags_environment()
 	##
 	if lvm dumpconfig activation/volume_list | grep -e "\"@$OUR_TAG\"" -e "\"${OCF_RESKEY_volgrpname}\""; then
 		ocf_log err "LVM:  Improper setup detected"
-		ocf_log err "The volume_list in lvm.conf must not contain the cluster tag, \"$OUR_TAG\", or volume group, $OCF_RESKEY_volgrpname"
+		ocf_exit_reason "The volume_list in lvm.conf must not contain the cluster tag, \"$OUR_TAG\", or volume group, $OCF_RESKEY_volgrpname"
 		return $OCF_ERR_GENERIC
 	fi
 
@@ -257,7 +257,7 @@ strip_tags()
 	done
 
 	if [ ! -z `vgs -o tags --noheadings $OCF_RESKEY_volgrpname | tr -d ' '` ]; then
-		ocf_log err "Failed to remove ownership tags from $OCF_RESKEY_volgrpname"
+		ocf_exit_reason "Failed to remove ownership tags from $OCF_RESKEY_volgrpname"
 		return $OCF_ERR_GENERIC
 	fi
 
@@ -284,7 +284,7 @@ set_tags()
 
 	vgchange --addtag $OUR_TAG $OCF_RESKEY_volgrpname
 	if [ $? -ne 0 ]; then
-		ocf_log err "Failed to add ownership tag to $OCF_RESKEY_volgrpname"
+		ocf_exit_reason "Failed to add ownership tag to $OCF_RESKEY_volgrpname"
 		return $OCF_ERR_GENERIC
 	fi
 
@@ -315,7 +315,7 @@ LVM_status() {
 		test "`cd /dev/$1 && ls`" != ""
 		rc=$?
 		if [ $rc -ne 0 ]; then
-			ocf_log err "VG $1 with no logical volumes is not supported by this RA!"
+			ocf_exit_reason "VG $1 with no logical volumes is not supported by this RA!"
 		fi
 	fi
 
@@ -328,7 +328,7 @@ LVM_status() {
 			# If vg is running, make sure the correct tag is present. Otherwise we
 			# can not guarantee exclusive activation.
 			if ! check_tags; then
-				ocf_log err "WARNING: $OCF_RESKEY_volgrpname is active without the cluster tag, \"$OUR_TAG\""
+				ocf_exit_reason "WARNING: $OCF_RESKEY_volgrpname is active without the cluster tag, \"$OUR_TAG\""
 				rc=$OCF_ERR_GENERIC
 			fi
 
@@ -397,7 +397,7 @@ retry_exclusive_start()
 			return $OCF_ERR_GENERIC;;
 		*)
 			if ! lvchange -an $OCF_RESKEY_volgrpname/$1; then
-				ocf_log err "Unable to perform required deactivation of $OCF_RESKEY_volgrpname/$1 before starting"
+				ocf_exit_reason "Unable to perform required deactivation of $OCF_RESKEY_volgrpname/$1 before starting"
 				return $OCF_ERR_GENERIC
 			fi
 			;;
@@ -456,7 +456,7 @@ LVM_start() {
 		: OK Volume $vg activated just fine!
 		return $OCF_SUCCESS 
 	else
-		ocf_log err "LVM: $vg did not activate correctly"
+		ocf_exit_reason "LVM: $vg did not activate correctly"
 		return $OCF_NOT_RUNNING
 	fi
 }
@@ -485,7 +485,7 @@ LVM_stop() {
 		ocf_run vgchange $vgchange_options $vg
 		res=$?
 		if LVM_status $vg; then
-			ocf_log err "LVM: $vg did not stop correctly"
+			ocf_exit_reason "LVM: $vg did not stop correctly"
 			res=1
 		fi
 
@@ -519,11 +519,30 @@ LVM_validate_all() {
 	check_binary $AWK
 
 	##
+	# lvmetad is a daemon that caches lvm metadata to improve the
+	# performance of LVM commands. This daemon should never be used when
+	# volume groups exist that are being managed by the cluster. The lvmetad
+	# daemon introduces a response lag, where certain LVM commands look like
+	# they have completed (like vg activation) when in fact the command
+	# is still in progress by the lvmetad.  This can cause reliability issues
+	# when managing volume groups in the cluster.  For Example, if you have a
+	# volume group that is a dependency for another application, it is possible
+	# the cluster will think the volume group is activated and attempt to start
+	# the application before volume group is really accesible... lvmetad is bad.
+	##
+	lvm dumpconfig global/use_lvmetad | grep 'use_lvmetad.*=.*1' > /dev/null 2>&1
+	if [ $? -eq 0 ]; then
+		# for now warn users that lvmetad is enabled and that they should disable it. In the
+		# future we may want to consider refusing to start, or killing the lvmetad daemon.
+		ocf_log warn "Disable lvmetad in lvm.conf. lvmetad should never be enabled in a clustered environment. Set use_lvmetad=0 and kill the lvmetad process"
+	fi
+	
+	##
 	# Off-the-shelf tests...
 	##
 	VGOUT=`vgck ${VOLUME} 2>&1`
 	if [ $? -ne 0 ]; then
-		ocf_log err "Volume group [$VOLUME] does not exist or contains error! ${VGOUT}"
+		ocf_exit_reason "Volume group [$VOLUME] does not exist or contains error! ${VGOUT}"
 		exit $OCF_ERR_GENERIC
 	fi
 
@@ -536,7 +555,7 @@ LVM_validate_all() {
 		VGOUT=`vgdisplay -v ${VOLUME} 2>&1`
 	fi
 	if [ $? -ne 0 ]; then
-		ocf_log err "Volume group [$VOLUME] does not exist or contains error! ${VGOUT}"
+		ocf_exit_reason "Volume group [$VOLUME] does not exist or contains error! ${VGOUT}"
 		exit $OCF_ERR_GENERIC
 	fi
 
@@ -552,7 +571,7 @@ LVM_validate_all() {
 	# Having cloned lvm resources with exclusive vg activation makes no sense at all.
 	##
 	if ocf_is_clone; then
-		ocf_log_err "cloned lvm resources can not be activated exclusively"
+		ocf_exit_reason "cloned lvm resources can not be activated exclusively"
 		exit $OCF_ERR_CONFIGURED
 	fi
 
@@ -571,7 +590,7 @@ LVM_validate_all() {
 		# verify is clvmd running
 		##
 		if ! ps -C clvmd > /dev/null 2>&1; then
-			ocf_log err "$OCF_RESKEY_volgrpname has the cluster attribute set, but 'clvmd' is not running"
+			ocf_exit_reason "$OCF_RESKEY_volgrpname has the cluster attribute set, but 'clvmd' is not running"
 			exit $OCF_ERR_GENERIC
 		fi
 		;;
@@ -608,7 +627,7 @@ esac
 if 
 	[ -z "$OCF_RESKEY_volgrpname" ]
 then
-	ocf_log err "You must identify the volume group name!"
+	ocf_exit_reason "You must identify the volume group name!"
 	exit $OCF_ERR_CONFIGURED 
 fi
 
@@ -635,7 +654,7 @@ rc=$?
 if
 	( [ $rc -ne 0 ] || [ -z "$LVM_VERSION" ] )
 then
-	ocf_log err "LVM: $1 could not determine LVM version. Try 'vgchange --version' manually and modify $0 ?"
+	ocf_exit_reason "LVM: $1 could not determine LVM version. Try 'vgchange --version' manually and modify $0 ?"
 	exit $OCF_ERR_INSTALLED
 fi
 LVM_MAJOR="${LVM_VERSION%%.*}"
diff --git a/heartbeat/MailTo b/heartbeat/MailTo
index acf6730..3936c39 100755
--- a/heartbeat/MailTo
+++ b/heartbeat/MailTo
@@ -131,7 +131,7 @@ MailToStatus () {
 
 MailToValidateAll () {
 	if [ -z "$MAILCMD" ]; then
-		ocf_log err "MAILCMD not set: complain to the packager"
+		ocf_exit_reason "MAILCMD not set: complain to the packager"
 		exit $OCF_ERR_INSTALLED
 	fi
 	check_binary "$MAILCMD"
@@ -169,7 +169,7 @@ esac
 if 
   [ -z "$OCF_RESKEY_email" ]
 then
-  ocf_log err "At least 1 Email address has to be given!"
+  ocf_exit_reason "At least 1 Email address has to be given!"
   exit $OCF_ERR_CONFIGURED
 fi
 
diff --git a/heartbeat/Makefile.am b/heartbeat/Makefile.am
index bc95f89..d9ce755 100644
--- a/heartbeat/Makefile.am
+++ b/heartbeat/Makefile.am
@@ -61,10 +61,13 @@ ocf_SCRIPTS	     =  ClusterMon		\
 			asterisk		\
 			nginx			\
 			AudibleAlarm		\
+			clvm		\
 			conntrackd		\
 			db2			\
 			dhcpd		\
 			Delay			\
+			dnsupdate		\
+			docker			\
 			eDir88			\
 			EvmsSCC			\
 			Evmsd			\
@@ -72,6 +75,7 @@ ocf_SCRIPTS	     =  ClusterMon		\
 			exportfs		\
 			Filesystem		\
 			fio			\
+			galera			\
 			ids			\
 			iscsi			\
 			ICP			\
@@ -81,6 +85,7 @@ ocf_SCRIPTS	     =  ClusterMon		\
 			iface-bridge		\
 			iface-vlan		\
 			jboss			\
+			kamailio		\
 			LinuxSCSI		\
 			LVM			\
 			lxc			\
@@ -90,6 +95,7 @@ ocf_SCRIPTS	     =  ClusterMon		\
 			mysql			\
 			mysql-proxy		\
 			named			\
+			nfsnotify		\
 			nfsserver		\
 			oracle			\
 			oralsnr			\
@@ -115,6 +121,7 @@ ocf_SCRIPTS	     =  ClusterMon		\
 			SysInfo			\
 			scsi2reservation	\
 			sfex			\
+                        sg_persist              \
 			symlink			\
 			syslog-ng		\
 			tomcat			\
@@ -140,6 +147,7 @@ ocfcommon_DATA		= ocf-shellfuncs 	\
 			  sapdb-nosha.sh	\
 			  sapdb.sh		\
 			  ora-common.sh		\
+			  mysql-common.sh		\
 			  findif.sh
 
 # Legacy locations
diff --git a/heartbeat/Route b/heartbeat/Route
index 9a49a26..c2314bc 100755
--- a/heartbeat/Route
+++ b/heartbeat/Route
@@ -87,7 +87,7 @@ the system default route.
 <content type="string" />
 </parameter>
 
-<parameter name="device" unique="1">
+<parameter name="device" unique="0">
 <longdesc lang="en">
 The outgoing network device to use for this route.
 </longdesc>
@@ -95,7 +95,7 @@ The outgoing network device to use for this route.
 <content type="string" default="" />
 </parameter>
 
-<parameter name="gateway" unique="1">
+<parameter name="gateway" unique="0">
 <longdesc lang="en">
 The gateway IP address to use for this route.
 </longdesc>
@@ -103,7 +103,7 @@ The gateway IP address to use for this route.
 <content type="string" default="" />
 </parameter>
 
-<parameter name="source" unique="1">
+<parameter name="source" unique="0">
 <longdesc lang="en">
 The source IP address to be configured for the route.
 </longdesc>
@@ -174,7 +174,7 @@ route_start() {
 	ocf_log info "${OCF_RESOURCE_INSTANCE} Added network route: $route_spec"
 	return $OCF_SUCCESS
     else
-	ocf_log error "${OCF_RESOURCE_INSTANCE} Failed to add network route: $route_spec"
+	ocf_exit_reason "${OCF_RESOURCE_INSTANCE} Failed to add network route: $route_spec"
     fi
     return $OCF_ERR_GENERIC
 }
@@ -189,7 +189,7 @@ route_stop() {
 		ocf_log info "${OCF_RESOURCE_INSTANCE} Removed network route: $route_spec"
 		return $OCF_SUCCESS
 	    else
-		ocf_log error "${OCF_RESOURCE_INSTANCE} Failed to remove network route: $route_spec"
+		ocf_exit_reason "${OCF_RESOURCE_INSTANCE} Failed to remove network route: $route_spec"
 	    fi
 	    ;;
 	$OCF_NOT_RUNNING)
@@ -224,24 +224,24 @@ route_validate() {
     # If we're running as a clone, are the clone meta attrs OK?
     if [ "${OCF_RESKEY_CRM_meta_clone}" ]; then
 	if [ "${OCF_RESKEY_CRM_meta_clone_node_max}" != 1 ]; then
-	    ocf_log error "Misconfigured clone parameters. Must set meta attribute \"clone_node_max\" to 1, got ${OCF_RESKEY_CRM_meta_clone_node_max}."
+	    ocf_exit_reason "Misconfigured clone parameters. Must set meta attribute \"clone_node_max\" to 1, got ${OCF_RESKEY_CRM_meta_clone_node_max}."
 	    return $OCF_ERR_ARGS
 	fi
     fi
     # Did we get a destination?
     if [ -z "${OCF_RESKEY_destination}" ]; then
-	ocf_log error "Missing required parameter \"destination\"."
+	ocf_exit_reason "Missing required parameter \"destination\"."
 	return $OCF_ERR_ARGS
     fi
     # Did we get either a device or a gateway address?
     if [ -z "${OCF_RESKEY_device}" -a -z "${OCF_RESKEY_gateway}" ]; then
-	ocf_log error "Must specifiy either \"device\", or \"gateway\", or both."
+	ocf_exit_reason "Must specifiy either \"device\", or \"gateway\", or both."
 	return $OCF_ERR_ARGS
     fi
     # If a device has been configured, is it available on this system?
     if [ -n "${OCF_RESKEY_device}" ]; then
 	if ! ip link show ${OCF_RESKEY_device} >/dev/null 2>&1; then
-	    ocf_log error "Network device ${OCF_RESKEY_device} appears not to be available on this system."
+	    ocf_exit_reason "Network device ${OCF_RESKEY_device} appears not to be available on this system."
 	    # OCF_ERR_ARGS prevents the resource from running anywhere at all,
 	    # maybe another node has the interface?
             # OCF_ERR_INSTALLED just prevents starting on this particular node.
@@ -256,7 +256,7 @@ route_validate() {
         # this system?
 	if [ -n "${OCF_RESKEY_source}" ]; then
 	    if ! ip address show | grep -w ${OCF_RESKEY_source} >/dev/null 2>&1; then
-		ocf_log error "Source address ${OCF_RESKEY_source} appears not to be available on this system."
+		ocf_exit_reason "Source address ${OCF_RESKEY_source} appears not to be available on this system."
 	    # same reason as with _device:
 		return $OCF_ERR_INSTALLED
 	    fi
@@ -264,7 +264,7 @@ route_validate() {
         # If a gateway address has been configured, is it reachable?
 	if [ -n "${OCF_RESKEY_gateway}" ]; then
 	    if ! ip route get ${OCF_RESKEY_gateway} >/dev/null 2>&1; then
-		ocf_log error "Gateway address ${OCF_RESKEY_gateway} is unreachable."
+		ocf_exit_reason "Gateway address ${OCF_RESKEY_gateway} is unreachable."
 	    # same reason as with _device:
 		return $OCF_ERR_INSTALLED
 	    fi
diff --git a/heartbeat/SendArp b/heartbeat/SendArp
index 675070c..b67404f 100755
--- a/heartbeat/SendArp
+++ b/heartbeat/SendArp
@@ -166,10 +166,10 @@ sendarp_start() {
 	# and wait-ing would be equal to not running in
 	# background
 	($SENDARP $ARGS ||
-	    ocf_log err "Could not send gratuitous arps") &
+	    ocf_exit_reason "Could not send gratuitous arps") &
     else
 	$SENDARP $ARGS || {
-	    ocf_log err "Could not send gratuitous arps"
+	    ocf_exit_reason "Could not send gratuitous arps"
 	    rc=$OCF_ERR_GENERIC
 	}
     fi
diff --git a/heartbeat/Squid b/heartbeat/Squid
index 28e2db5..70c7c3d 100755
--- a/heartbeat/Squid
+++ b/heartbeat/Squid
@@ -216,7 +216,7 @@ are_pids_sane()
 	if [[ "${SQUID_PIDS[1]}" = "${SQUID_PIDS[2]}" ]]; then
 		return $OCF_SUCCESS
 	else
-		ocf_log err "$SQUID_NAME:Pid unmatch"
+		ocf_exit_reason "$SQUID_NAME:Pid unmatch"
 		return $OCF_ERR_GENERIC
 	fi
 }
@@ -253,7 +253,7 @@ monitor_squid()
 			"${SQUID_PIDS[0]},${SQUID_PIDS[1]},${SQUID_PIDS[2]}"
 		(( trialcount = trialcount + 1 ))
 		if (( trialcount > SQUID_CONFIRM_TRIALCOUNT )); then
-			ocf_log err "$SQUID_NAME:Inconsistency of processes remains unsolved"
+			ocf_exit_reason "$SQUID_NAME:Inconsistency of processes remains unsolved"
 			return $OCF_ERR_GENERIC
 		fi
 		sleep 1
@@ -348,7 +348,7 @@ fi
 
 SQUID_CONF="${OCF_RESKEY_squid_conf}"
 if [[ -z "$SQUID_CONF" ]]; then
-	ocf_log err "SQUID_CONF is not defined"
+	ocf_exit_reason "SQUID_CONF is not defined"
 	exit $OCF_ERR_CONFIGURED
 fi
 
@@ -374,23 +374,23 @@ fi
 
 SQUID_EXE="${OCF_RESKEY_squid_exe}"
 if [[ -z "$SQUID_EXE" ]]; then
-	ocf_log err "SQUID_EXE is not defined"
+	ocf_exit_reason "SQUID_EXE is not defined"
 	exit $OCF_ERR_CONFIGURED
 fi
 if [[ ! -x "$SQUID_EXE" ]]; then
-	ocf_log err "$SQUID_EXE is not found"
+	ocf_exit_reason "$SQUID_EXE is not found"
 	exit $OCF_ERR_CONFIGURED
 fi
 
 SQUID_PIDFILE="${OCF_RESKEY_squid_pidfile}"
 if [[ -z "$SQUID_PIDFILE" ]]; then
-	ocf_log err "SQUID_PIDFILE is not defined"
+	ocf_exit_reason "SQUID_PIDFILE is not defined"
 	exit $OCF_ERR_CONFIGURED
 fi
 
 SQUID_PORT="${OCF_RESKEY_squid_port}"
 if [[ -z "$SQUID_PORT" ]]; then
-	ocf_log err "SQUID_PORT is not defined"
+	ocf_exit_reason "SQUID_PORT is not defined"
 	exit $OCF_ERR_CONFIGURED
 fi
 
diff --git a/heartbeat/Stateful b/heartbeat/Stateful
index 8ee42a0..1a90e27 100755
--- a/heartbeat/Stateful
+++ b/heartbeat/Stateful
@@ -1,7 +1,7 @@
 #!/bin/sh
 #
 #
-#	Example of a stateful OCF Resource Agent. 
+#	Example of a stateful OCF Resource Agent.
 #
 # Copyright (c) 2006 Andrew Beekhof
 #
@@ -61,6 +61,8 @@ Location to store the resource state in
 <actions>
 <action name="start"   timeout="20s" />
 <action name="stop"    timeout="20s" />
+<action name="promote"    timeout="20s" />
+<action name="demote"    timeout="20s" />
 <action name="monitor" depth="0"  timeout="20" interval="10"/>
 <action name="meta-data"  timeout="5" />
 <action name="validate-all"  timeout="20s" />
@@ -90,16 +92,16 @@ stateful_check_state() {
     if [ -f ${OCF_RESKEY_state} ]; then
 	state=`cat ${OCF_RESKEY_state}`
 	if [ "x$target" = "x$state" ]; then
-	    return 0
+	    return $OCF_SUCCESS
 	fi
 
     else
 	if [ "x$target" = "x" ]; then
-	    return 0
+	    return $OCF_SUCCESS
 	fi
     fi
 
-    return 1
+    return $OCF_ERR_GENERIC
 }
 
 stateful_start() {
@@ -110,28 +112,28 @@ stateful_start() {
     fi
     stateful_update slave
     $CRM_MASTER -v 5
-    return 0
+    return $OCF_SUCCESS
 }
 
 stateful_demote() {
-    stateful_check_state 
+    stateful_check_state
     if [ $? = 0 ]; then
        	# CRM Error - Should never happen
 	return $OCF_NOT_RUNNING
     fi
     stateful_update slave
     $CRM_MASTER -v 5
-    return 0
+    return $OCF_SUCCESS
 }
 
 stateful_promote() {
-    stateful_check_state 
+    stateful_check_state
     if [ $? = 0 ]; then
 	return $OCF_NOT_RUNNING
     fi
     stateful_update master
     $CRM_MASTER -v 10
-    return 0
+    return $OCF_SUCCESS
 }
 
 stateful_stop() {
@@ -144,7 +146,7 @@ stateful_stop() {
     if [ -f ${OCF_RESKEY_state} ]; then
 	rm ${OCF_RESKEY_state}
     fi
-    return 0
+    return $OCF_SUCCESS
 }
 
 stateful_monitor() {
@@ -163,7 +165,7 @@ stateful_monitor() {
 	cat ${OCF_RESKEY_state}
 	return $OCF_ERR_GENERIC
     fi
-    return 7
+    return $OCF_NOT_RUNNING
 }
 
 stateful_validate() {
@@ -185,4 +187,3 @@ usage|help)	stateful_usage $OCF_SUCCESS;;
 esac
 
 exit $?
-
diff --git a/heartbeat/VirtualDomain b/heartbeat/VirtualDomain
index b159c2c..0f6b0bc 100755
--- a/heartbeat/VirtualDomain
+++ b/heartbeat/VirtualDomain
@@ -21,11 +21,21 @@ OCF_RESKEY_force_stop_default=0
 OCF_RESKEY_autoset_utilization_cpu_default="true"
 OCF_RESKEY_autoset_utilization_hv_memory_default="true"
 OCF_RESKEY_migrateport_default=$(( 49152 + $(ocf_maybe_random) % 64 ))
+OCF_RESKEY_CRM_meta_timeout_default=90000
+OCF_RESKEY_save_config_on_stop_default=false
+OCF_RESKEY_sync_config_on_stop_default=false
 
 : ${OCF_RESKEY_force_stop=${OCF_RESKEY_force_stop_default}}
 : ${OCF_RESKEY_autoset_utilization_cpu=${OCF_RESKEY_autoset_utilization_cpu_default}}
 : ${OCF_RESKEY_autoset_utilization_hv_memory=${OCF_RESKEY_autoset_utilization_hv_memory_default}}
 : ${OCF_RESKEY_migrateport=${OCF_RESKEY_migrateport_default}}
+: ${OCF_RESKEY_CRM_meta_timeout=${OCF_RESKEY_CRM_meta_timeout_default}}
+: ${OCF_RESKEY_save_config_on_stop=${OCF_RESKEY_save_config_on_stop_default}}
+: ${OCF_RESKEY_sync_config_on_stop=${OCF_RESKEY_sync_config_on_stop_default}}
+
+if ocf_is_true ${OCF_RESKEY_sync_config_on_stop}; then
+	OCF_RESKEY_save_config_on_stop="true"
+fi
 #######################################################################
 
 ## I'd very much suggest to make this RA use bash,
@@ -104,11 +114,28 @@ At the moment Qemu/KVM and Xen migration via a dedicated network is supported.
 
 Note: Be sure this composed host name is locally resolveable and the
 associated IP is reachable through the favored network.
+
+See also the migrate_options parameter below.
 </longdesc>
 <shortdesc lang="en">Migration network host name suffix</shortdesc>
 <content type="string" default="" />
 </parameter>
 
+<parameter name="migrate_options" unique="0" required="0">
+<longdesc lang="en">
+Extra virsh options for the guest live migration. You can also specify
+here --migrateuri if the calculated migrate URI is unsuitable for your
+environment. If --migrateuri is set then migration_network_suffix
+and migrateport are effectively ignored. Use "%n" as the placeholder
+for the target node name.
+
+Please refer to the libvirt documentation for details on guest
+migration.
+</longdesc>
+<shortdesc lang="en">live migrate options</shortdesc>
+<content type="string" />
+</parameter>
+
 <parameter name="monitor_scripts" unique="0" required="0">
 <longdesc lang="en">
 To additionally monitor services within the virtual domain, add this
@@ -148,6 +175,26 @@ This port will be used in the qemu migrateuri. If unset, the port will be a rand
 <content type="integer" />
 </parameter>
 
+<parameter name="save_config_on_stop" unique="0" required="0">
+<longdesc lang="en">
+Changes to a running VM's config are normally lost on stop. 
+This parameter instructs the RA to save the configuration back to the xml file provided in the "config" parameter.
+</longdesc>
+<shortdesc lang="en">Save running VM's config back to its config file</shortdesc>
+<content type="boolean" />
+</parameter>
+
+<parameter name="sync_config_on_stop" unique="0" required="0">
+<longdesc lang="en">
+Setting this automatically enables save_config_on_stop. 
+When enabled this parameter instructs the RA to 
+call csync2 -x to synchronize the file to all nodes.
+csync2 must be properly set up for this to work.
+</longdesc>
+<shortdesc lang="en">Save running VM's config back to its config file</shortdesc>
+<content type="boolean" />
+</parameter>
+
 <parameter name="snapshot">
 <longdesc lang="en">
 Path to the snapshot directory where the virtual machine image will be stored.  When this 
@@ -183,9 +230,17 @@ set_util_attr() {
 	local cval outp
 
 	cval=$(crm_resource -Q -r $OCF_RESOURCE_INSTANCE -z -g $attr 2>/dev/null)
+	if [ $? -ne 0 ] && [ -z "$cval" ]; then
+		crm_resource -Q -r $OCF_RESOURCE_INSTANCE -z -g $attr 2>&1 | grep -e "not connected" > /dev/null 2>&1
+		if [ $? -eq 0 ]; then
+			ocf_log debug "Unable to set utilization attribute, cib is not available"
+			return
+		fi
+	fi
+
 	if [ "$cval" != "$val" ]; then
-	   outp=`crm_resource -r $OCF_RESOURCE_INSTANCE -z -p $attr -v $val 2>&1` ||
-	   ocf_log warn "crm_resource failed to set utilization attribute $attr: $outp"
+		outp=$(crm_resource -r $OCF_RESOURCE_INSTANCE -z -p $attr -v $val 2>&1) ||
+		ocf_log warn "crm_resource failed to set utilization attribute $attr: $outp"
 	fi
 }
 
@@ -193,41 +248,97 @@ update_utilization() {
 	local dom_cpu dom_mem
 
 	if ocf_is_true "$OCF_RESKEY_autoset_utilization_cpu"; then
-	   dom_cpu=$(LANG=C virsh $VIRSH_OPTIONS dominfo ${DOMAIN_NAME} | awk '/CPU\(s\)/{print $2}')
+	   dom_cpu=$(LANG=C virsh $VIRSH_OPTIONS dominfo ${DOMAIN_NAME} 2>/dev/null | awk '/CPU\(s\)/{print $2}')
 	   test -n "$dom_cpu" && set_util_attr cpu $dom_cpu
 	fi
 	if ocf_is_true "$OCF_RESKEY_autoset_utilization_hv_memory"; then
-	   dom_mem=$(LANG=C virsh $VIRSH_OPTIONS dominfo ${DOMAIN_NAME} | awk '/Max memory/{printf("%d", $3/1024)}')
+	   dom_mem=$(LANG=C virsh $VIRSH_OPTIONS dominfo ${DOMAIN_NAME} 2>/dev/null | awk '/Max memory/{printf("%d", $3/1024)}')
 	   test -n "$dom_mem" && set_util_attr hv_memory "$dom_mem"
 	fi
 }
 
+get_emulator()
+{
+	local emulator=""
+	# An emulator is not required, so only report message in debug mode
+	local loglevel="debug"
+
+	if ocf_is_probe; then
+		loglevel="notice"
+	fi
+
+	emulator=$(virsh $VIRSH_OPTIONS dumpxml $DOMAIN_NAME 2>/dev/null | sed -n -e 's/^.*<emulator>\(.*\)<\/emulator>.*$/\1/p')
+	if [ -z "$emulator" ] && [ -e "$EMULATOR_STATE" ]; then
+		emulator=$(cat $EMULATOR_STATE)
+	fi
+	if [ -z "$emulator" ]; then
+		emulator=$(cat ${OCF_RESKEY_config} | sed -n -e 's/^.*<emulator>\(.*\)<\/emulator>.*$/\1/p')
+	fi
+
+	if [ -n "$emulator" ]; then
+		basename $emulator
+	else 
+		ocf_log $loglevel "Unable to determine emulator for $DOMAIN_NAME"
+	fi
+}
+
+update_emulator_cache()
+{
+	local emulator
+
+	emulator=$(get_emulator)
+	if [ -n "$emulator" ]; then
+		echo $emulator > $EMULATOR_STATE
+	fi
+}
+
 # attempt to check domain status outside of libvirt using the emulator process
 pid_status()
 {
 	local rc=$OCF_ERR_GENERIC
-	local emulator
-
-	emulator=$(basename $(egrep '[[:space:]]*<emulator>.*</emulator>[[:space:]]*$' ${OCF_RESKEY_config} | sed -e 's/[[:space:]]*<emulator>\(.*\)<\/emulator>[[:space:]]*$/\1/'))
+	local emulator=$(get_emulator)
 
 	case "$emulator" in
-		qemu-kvm|qemu-system-*)
-			ps awx | grep -E "[q]emu-(kvm|system).*-name $DOMAIN_NAME " > /dev/null 2>&1
+		qemu-kvm|qemu-dm|qemu-system-*)
+			rc=$OCF_NOT_RUNNING
+			ps awx | grep -E "[q]emu-(kvm|dm|system).*-name $DOMAIN_NAME " > /dev/null 2>&1
+			if [ $? -eq 0 ]; then
+				rc=$OCF_SUCCESS
+			fi
+			;;
+		libvirt_lxc)
+			rc=$OCF_NOT_RUNNING
+			ps awx | grep -E "[l]ibvirt_lxc.*-name $DOMAIN_NAME " > /dev/null 2>&1
 			if [ $? -eq 0 ]; then
-				# domain exists and is running
-				ocf_log debug "Virtual domain $DOMAIN_NAME is currently running."
 				rc=$OCF_SUCCESS
-			else 
-				# domain pid does not exist on local machine
-				ocf_log debug "Virtual domain $DOMAIN_NAME is currently not running."
-				rc=$OCF_NOT_RUNNING
 			fi
 			;;
 		# This can be expanded to check for additional emulators
 		*)
+			# We may be running xen with PV domains, they don't
+			# have an emulator set. try xl list or xen-lists
+			if have_binary xl; then
+				rc=$OCF_NOT_RUNNING
+				xl list $DOMAIN_NAME >/dev/null 2>&1
+				if [ $? -eq 0 ]; then
+					rc=$OCF_SUCCESS
+				fi
+			elif have_binary xen-list; then
+				rc=$OCF_NOT_RUNNING
+				xen-list $DOMAIN_NAME 2>/dev/null | grep -qs "State.*[-r][-b][-p]--" 2>/dev/null
+				if [ $? -eq 0 ]; then
+					rc=$OCF_SUCCESS
+				fi
+			fi
 			;;
 	esac
 
+	if [ $rc -eq $OCF_SUCCESS ]; then
+		ocf_log debug "Virtual domain $DOMAIN_NAME is currently running."
+	elif [ $rc -eq $OCF_NOT_RUNNING ]; then
+		ocf_log debug "Virtual domain $DOMAIN_NAME is currently not running."
+	fi
+
 	return $rc
 }
 
@@ -237,13 +348,14 @@ VirtualDomain_Status() {
 	status="no state"
 	while [ "$status" = "no state" ]; do
 		try=$(($try + 1 ))
-		status=$(virsh $VIRSH_OPTIONS domstate $DOMAIN_NAME 2>&1|tr 'A-Z' 'a-z')
+		status=$(LANG=C virsh $VIRSH_OPTIONS domstate $DOMAIN_NAME 2>&1 | tr 'A-Z' 'a-z')
 		case "$status" in
-			*"error:"*"domain not found"*|"shut off")
+			*"error:"*"domain not found"|*"error:"*"failed to get domain"*|"shut off")
 				# shut off: domain is defined, but not started, will not happen if
 				#   domain is created but not defined
-				# Domain not found: domain is not defined and thus not started
-				ocf_log debug "Virtual domain $DOMAIN_NAME is currently $status."
+				# "Domain not found" or "failed to get domain": domain is not defined
+				#   and thus not started
+				ocf_log debug "Virtual domain $DOMAIN_NAME is not running: $(echo $status | sed s/error://g)"
 				rc=$OCF_NOT_RUNNING
 				;;
 			running|paused|idle|blocked|"in shutdown")
@@ -282,25 +394,45 @@ VirtualDomain_Status() {
 					# again, relying on the CRM/LRM to time us out if
 					# this takes too long.
 					ocf_log info "Virtual domain $DOMAIN_NAME currently has no state, retrying."
-					sleep 1
 				fi
+				sleep 1
 				;;
 			*)
 				# any other output is unexpected.
 				ocf_log error "Virtual domain $DOMAIN_NAME has unknown status \"$status\"!"
+				sleep 1
 				;;
 		esac
 	done
 	return $rc
 }
 
+# virsh undefine removes configuration files if they are in
+# directories which are managed by libvirt. such directories
+# include also subdirectories of /etc (for instance
+# /etc/libvirt/*) which may be surprising. VirtualDomain didn't
+# include the undefine call before, hence this wasn't an issue
+# before.
+#
+# There seems to be no way to find out which directories are
+# managed by libvirt.
+#
 verify_undefined() {
-	for dom in `virsh --connect=${OCF_RESKEY_hypervisor} list --all --name`; do
-		if [ "$dom" = "$DOMAIN_NAME" ]; then
+	local tmpf
+	if virsh --connect=${OCF_RESKEY_hypervisor} list --all --name 2>/dev/null | grep -wqs "$DOMAIN_NAME"
+	then
+		tmpf=$(mktemp -t vmcfgsave.XXXXXX)
+		if [ ! -r "$tmpf" ]; then
+			ocf_log warn "unable to create temp file, disk full?"
+			# we must undefine the domain
 			virsh $VIRSH_OPTIONS undefine $DOMAIN_NAME > /dev/null 2>&1
-			return
+		else
+			cp -p $OCF_RESKEY_config $tmpf
+			virsh $VIRSH_OPTIONS undefine $DOMAIN_NAME > /dev/null 2>&1
+			[ -f $OCF_RESKEY_config ] || cp -f $tmpf $OCF_RESKEY_config
+			rm -f $tmpf
 		fi
-	done
+	fi
 }
 
 VirtualDomain_Start() {
@@ -318,7 +450,7 @@ VirtualDomain_Start() {
 			rm -f $snapshotimage
 			return $OCF_SUCCESS
 		fi
-		ocf_log error "Failed to restore ${DOMAIN_NAME} from state file in ${OCF_RESKEY_snapshot} directory."
+		ocf_exit_reason "Failed to restore ${DOMAIN_NAME} from state file in ${OCF_RESKEY_snapshot} directory."
 		return $OCF_ERR_GENERIC
 	fi
 
@@ -333,13 +465,14 @@ VirtualDomain_Start() {
 	virsh $VIRSH_OPTIONS create ${OCF_RESKEY_config}
 	rc=$?
 	if [ $rc -ne 0 ]; then
-		ocf_log error "Failed to start virtual domain ${DOMAIN_NAME}."
+		ocf_exit_reason "Failed to start virtual domain ${DOMAIN_NAME}."
 		return $OCF_ERR_GENERIC
 	fi
 
 	while ! VirtualDomain_Monitor; do
 		sleep 1
 	done
+	
 	return $OCF_SUCCESS
 }
 
@@ -356,6 +489,7 @@ force_stop()
 		*"error:"*"domain is not running"*|*"error:"*"domain not found"*)
 			: ;; # unexpected path to the intended outcome, all is well
 		[!0]*)
+			ocf_exit_reason "forced stop failed"
 			return $OCF_ERR_GENERIC ;;
 		0*)
 			while [ $status != $OCF_NOT_RUNNING ]; do
@@ -366,6 +500,38 @@ force_stop()
 	return $OCF_SUCCESS
 }
 
+sync_config(){
+	ocf_log info "Syncing $DOMAIN_NAME config file with csync2 -x ${OCF_RESKEY_config}"
+	if ! csync2 -x ${OCF_RESKEY_config}; then
+		ocf_log warn "Syncing ${OCF_RESKEY_config} failed."; 
+	fi
+}
+
+save_config(){
+	CFGTMP=$(mktemp -t vmcfgsave.XXX)
+	virsh $VIRSH_OPTIONS dumpxml --inactive --security-info ${DOMAIN_NAME} > ${CFGTMP}
+	if [ -s ${CFGTMP} ]; then
+		if ! cmp -s ${CFGTMP} ${OCF_RESKEY_config}; then
+			if virt-xml-validate ${CFGTMP} domain 2>/dev/null ;	then
+				ocf_log info "Saving domain $DOMAIN_NAME to ${OCF_RESKEY_config}. Please make sure it's present on all nodes or sync_config_on_stop is on."
+				if cat ${CFGTMP} > ${OCF_RESKEY_config} ; then
+					ocf_log info "Saved $DOMAIN_NAME domain's configuration to ${OCF_RESKEY_config}."
+					if ocf_is_true "$OCF_RESKEY_sync_config_on_stop"; then	
+						sync_config
+					fi
+				else	
+					ocf_log warn "Moving ${CFGTMP} to ${OCF_RESKEY_config} failed."
+				fi
+			else
+				ocf_log warn "Domain $DOMAIN_NAME config failed to validate after dump. Skipping config update."
+			fi
+		fi
+	else
+		ocf_log warn "Domain $DOMAIN_NAME config has 0 size. Skipping config update."
+	fi
+	rm -f ${CFGTMP}
+}
+
 VirtualDomain_Stop() {
 	local i
 	local status
@@ -394,6 +560,11 @@ VirtualDomain_Stop() {
 				fi
 			fi
 
+			# save config if needed
+			if ocf_is_true "$OCF_RESKEY_save_config_on_stop"; then	
+				save_config
+			fi
+
 			# issue the shutdown if save state didn't shutdown for us
 			if [ $needshutdown -eq 1 ]; then
 				# Issue a graceful shutdown request
@@ -437,14 +608,45 @@ VirtualDomain_Stop() {
 	force_stop
 }
 
+mk_migrateuri() {
+	local target_node
+	local migrate_target
+	local hypervisor
+
+	target_node="$OCF_RESKEY_CRM_meta_migrate_target"
+
+	# A typical migration URI via a special  migration network looks
+	# like "tcp://bar-mig:49152". The port would be randomly chosen
+	# by libvirt from the range 49152-49215 if omitted, at least since
+	# version 0.7.4 ...
+	if [ -n "${OCF_RESKEY_migration_network_suffix}" ]; then
+		hypervisor="${OCF_RESKEY_hypervisor%%[+:]*}"
+		# Hostname might be a FQDN
+		migrate_target=$(echo ${target_node} | sed -e "s,^\([^.]\+\),\1${OCF_RESKEY_migration_network_suffix},")
+		case $hypervisor in
+			qemu)
+				# For quiet ancient libvirt versions a migration port is needed
+				# and the URI must not contain the "//". Newer versions can handle
+				# the "bad" URI.
+				echo "tcp:${migrate_target}:${OCF_RESKEY_migrateport}"
+				;;
+			xen)
+				echo "xenmigr://${migrate_target}"
+				;;
+			*)
+				ocf_log warn "$DOMAIN_NAME: Migration via dedicated network currently not supported for ${hypervisor}."
+				;;
+		esac
+	fi
+}
+
 VirtualDomain_Migrate_To() {
+	local rc
 	local target_node
 	local remoteuri
 	local transport_suffix
 	local migrateuri
-	local migrateport
-	local migrate_target
-	local hypervisor
+	local migrate_opts
 
 	target_node="$OCF_RESKEY_CRM_meta_migrate_target"
 
@@ -455,45 +657,38 @@ VirtualDomain_Migrate_To() {
 		if [ -n "${OCF_RESKEY_migration_transport}" ]; then
 			transport_suffix="+${OCF_RESKEY_migration_transport}"
 		fi
-		# A typical migration URI via a special  migration network looks
-		# like "tcp://bar-mig:49152". The port would be randomly chosen
-		# by libvirt from the range 49152-49215 if omitted, at least since
-		# version 0.7.4 ...
-		if [ -n "${OCF_RESKEY_migration_network_suffix}" ]; then
-			hypervisor="${OCF_RESKEY_hypervisor%%[+:]*}"
-			# Hostname might be a FQDN
-			migrate_target=$(echo ${target_node} | sed -e "s,^\([^.]\+\),\1${OCF_RESKEY_migration_network_suffix},")
-			case $hypervisor in
-				qemu)
-					# For quiet ancient libvirt versions a migration port is needed
-					# and the URI must not contain the "//". Newer versions can handle
-					# the "bad" URI.
-					migrateuri="tcp:${migrate_target}:${OCF_RESKEY_migrateport}"
-					;;
-				xen)
-					migrateuri="xenmigr://${migrate_target}"
-					;;
-				*)
-					ocf_log warn "$DOMAIN_NAME: Migration via dedicated network currently not supported for ${hypervisor}."
-					;;
-			esac
+
+		# User defined migrateuri or do we make one?
+		migrate_opts="$OCF_RESKEY_migrate_options"
+		if echo "$migrate_opts" | fgrep -qs -- "--migrateuri="; then
+			migrateuri=`echo "$migrate_opts" |
+				sed "s/.*--migrateuri=\([^ ]*\).*/\1/;s/%n/$target_node/g"`
+			migrate_opts=`echo "$migrate_opts" |
+				sed "s/\(.*\)--migrateuri=[^ ]*\(.*\)/\1\2/"`
+		else
+			migrateuri=`mk_migrateuri`
 		fi
 		# Scared of that sed expression? So am I. :-)
 		remoteuri=$(echo ${OCF_RESKEY_hypervisor} | sed -e "s,\(.*\)://[^/:]*\(:\?[0-9]*\)/\(.*\),\1${transport_suffix}://${target_node}\2/\3,")
 
+		# save config if needed
+		if ocf_is_true "$OCF_RESKEY_save_config_on_stop"; then	
+			save_config
+		fi
+
 		# OK, we know where to connect to. Now do the actual migration.
-		ocf_log info "$DOMAIN_NAME: Starting live migration to ${target_node} (using remote hypervisor URI ${remoteuri} ${migrateuri})."
-		virsh ${VIRSH_OPTIONS} migrate --live $DOMAIN_NAME ${remoteuri} ${migrateuri}
+		ocf_log info "$DOMAIN_NAME: Starting live migration to ${target_node} (using virsh ${VIRSH_OPTIONS} migrate --live $migrate_opts $DOMAIN_NAME $remoteuri $migrateuri)."
+		virsh ${VIRSH_OPTIONS} migrate --live $migrate_opts $DOMAIN_NAME $remoteuri $migrateuri
 		rc=$?
 		if [ $rc -ne 0 ]; then
-			ocf_log err "$DOMAIN_NAME: live migration to ${remoteuri} ${migrateuri} failed: $rc"
+			ocf_exit_reason "$DOMAIN_NAME: live migration to ${target_node} failed: $rc"
 			return $OCF_ERR_GENERIC
 		else
 			ocf_log info "$DOMAIN_NAME: live migration to ${target_node} succeeded."
 			return $OCF_SUCCESS
 		fi
 	else
-		ocf_log err "$DOMAIN_NAME: migrate_to: Not active locally!"
+		ocf_exit_reason "$DOMAIN_NAME: migrate_to: Not active locally!"
 		return $OCF_ERR_GENERIC
 	fi
 }
@@ -503,6 +698,10 @@ VirtualDomain_Migrate_From() {
 		sleep 1
 	done
 	ocf_log info "$DOMAIN_NAME: live migration from ${OCF_RESKEY_CRM_meta_migrate_source} succeeded."
+	# save config if needed
+	if ocf_is_true "$OCF_RESKEY_save_config_on_stop"; then	
+		save_config
+	fi
 	return $OCF_SUCCESS
 }
 
@@ -521,7 +720,7 @@ VirtualDomain_Monitor() {
 				# A monitor script returned a non-success exit
 				# code. Stop iterating over the list of scripts, log a
 				# warning message, and propagate $OCF_ERR_GENERIC.
-				ocf_log warn "Monitor command \"${script}\" for domain ${DOMAIN_NAME} returned ${script_rc} with output: ${script_output}"
+				ocf_exit_reason "Monitor command \"${script}\" for domain ${DOMAIN_NAME} returned ${script_rc} with output: ${script_output}"
 				rc=$OCF_ERR_GENERIC
 				break
 			else
@@ -530,7 +729,13 @@ VirtualDomain_Monitor() {
 		done
 	fi
 
+	update_emulator_cache
 	update_utilization
+   # Save configuration on monitor as well, so we will have a better chance of 
+	# having fresh and up to date config files on all nodes.
+	if ocf_is_true "$OCF_RESKEY_save_config_on_stop"; then	
+		save_config
+	fi
 
 	return ${rc}
 }
@@ -542,13 +747,13 @@ VirtualDomain_Validate_All() {
 	done
 
 	if [ -z $OCF_RESKEY_config ]; then
-		ocf_log error "Missing configuration parameter \"config\"."
+		ocf_exit_reason "Missing configuration parameter \"config\"."
 		return $OCF_ERR_CONFIGURED
 	fi
 
 	if ocf_is_true $OCF_RESKEY_force_stop; then
 		if [ -n "$OCF_RESKEY_snapshot" ]; then
-			ocf_log error "The 'force_stop' and 'snapshot' options can not be used together."
+			ocf_exit_reason "The 'force_stop' and 'snapshot' options can not be used together."
 			return $OCF_ERR_CONFIGURED
 		fi
 	fi
@@ -561,10 +766,15 @@ VirtualDomain_Validate_All() {
 		elif [ "$__OCF_ACTION" = "stop" ]; then
 			ocf_log info "Configuration file $OCF_RESKEY_config not readable, resource considered stopped."
 		else
-			ocf_log error "Configuration file $OCF_RESKEY_config does not exist or is not readable."
+			ocf_exit_reason "Configuration file $OCF_RESKEY_config does not exist or is not readable."
 			return $OCF_ERR_INSTALLED
 		fi
 	fi
+
+	# Check if csync2 is available when config tells us we might need it.
+	if ocf_is_true $OCF_RESKEY_sync_config_on_stop; then
+		check_binary csync2; 
+	fi
 }
 
 if [ $# -ne 1 ]; then
@@ -582,7 +792,7 @@ case $1 in
 esac
 
 # Grab the virsh uri default, but only if hypervisor isn't set
-: ${OCF_RESKEY_hypervisor=$(virsh --quiet uri)}
+: ${OCF_RESKEY_hypervisor=$(virsh --quiet uri 2>/dev/null)}
 
 # Set options to be passed to virsh:
 VIRSH_OPTIONS="--connect=${OCF_RESKEY_hypervisor} --quiet"
@@ -604,10 +814,12 @@ fi
 # Retrieve the domain name from the xml file.
 DOMAIN_NAME=`egrep '[[:space:]]*<name>.*</name>[[:space:]]*$' ${OCF_RESKEY_config} | sed -e 's/[[:space:]]*<name>\(.*\)<\/name>[[:space:]]*$/\1/' 2>/dev/null`
 if [ -z $DOMAIN_NAME ]; then
-	ocf_log err "This is unexpected. Cannot determine domain name."
+	ocf_exit_reason "Unable to determine domain name."
 	exit $OCF_ERR_GENERIC
 fi
 
+EMULATOR_STATE="${HA_RSCTMP}/VirtualDomain-${DOMAIN_NAME}-emu.state"
+
 case $1 in
 	start)
 		VirtualDomain_Start
diff --git a/heartbeat/Xen b/heartbeat/Xen
index cbc7c37..e273f6e 100755
--- a/heartbeat/Xen
+++ b/heartbeat/Xen
@@ -6,7 +6,7 @@
 #
 #	Resource Agent for the Xen Hypervisor.
 #	Manages Xen virtual machine instances by
-#	mapping cluster resource start and stop,  
+#	mapping cluster resource start and stop,
 #	to Xen create and shutdown, respectively.
 #
 #	usage: $0  {start|stop|status|monitor|meta-data}
@@ -32,11 +32,10 @@
 
 #######################################################################
 
-
 usage() {
-  cat <<-!
+	cat <<-END
 	usage: $0 {start|stop|status|monitor|meta-data|validate-all}
-	!
+END
 }
 
 
@@ -45,6 +44,9 @@ usage() {
 : ${OCF_RESKEY_allow_mem_management=0}
 : ${OCF_RESKEY_reserved_Dom0_memory=512}
 
+# prefer xl
+xentool=$(which xl 2> /dev/null || which xm)
+
 meta_data() {
 	cat <<END
 <?xml version="1.0"?>
@@ -55,7 +57,7 @@ meta_data() {
 <longdesc lang="en">
 Resource Agent for the Xen Hypervisor.
 Manages Xen virtual machine instances by mapping cluster resource
-start and stop,  to Xen create and shutdown, respectively.
+start and stop, to Xen create and shutdown, respectively.
 
 A note on names
 
@@ -95,9 +97,9 @@ Name of the virtual machine.
 </parameter>
 <parameter name="shutdown_timeout">
 <longdesc lang="en">
-The Xen agent will first try an orderly shutdown using xm shutdown.
+The Xen agent will first try an orderly shutdown using xl shutdown.
 Should this not succeed within this timeout, the agent will escalate to
-xm destroy, forcibly killing the node.
+xl destroy, forcibly killing the node.
 
 If this is not set, it will default to two-third of the stop action
 timeout.
@@ -119,7 +121,7 @@ without installed PV drivers.
 </parameter>
 <parameter name="allow_mem_management" unique="0" required="0">
 <longdesc lang="en">
-This parameter enables dynamic adjustment of memory for start 
+This parameter enables dynamic adjustment of memory for start
 and stop actions used for Dom0 and the DomUs. The default is
 to not adjust memory dynamically.
 </longdesc>
@@ -179,88 +181,95 @@ END
 }
 
 Xen_Status() {
-  if have_binary xen-list; then
-     xen-list $1 2>/dev/null | grep -qs "State.*[-r][-b][-p]--" 2>/dev/null
-     if [ $? -ne 0 ]; then
-     	return $OCF_NOT_RUNNING
-     else
-     	return $OCF_SUCCESS
-     fi
-  fi
-  STATUS=`xm list --long $1 2>/dev/null | grep status 2>/dev/null`
-  if [ "X${STATUS}" != "X" ]; then
-    # we have Xen 3.0.4 or higher
-    STATUS_NOSPACES=`echo "$STATUS" | awk '{ print $1,$2}'`
-    if [ "$STATUS_NOSPACES" = "(status 2)" -o "$STATUS_NOSPACES" = "(status 1)" ]; then
-      return $OCF_SUCCESS
-    else 
-      return $OCF_NOT_RUNNING
-    fi
-  else
-    # we have Xen 3.0.3 or lower
-    STATUS=`xm list --long $1 2>/dev/null | grep state 2>/dev/null`
-    echo "${STATUS}" | grep -qs "[-r][-b][-p]---"
-    if [ $? -ne 0 ]; then
-      return $OCF_NOT_RUNNING
-    else
-      return $OCF_SUCCESS
-    fi
-    
-  fi
+	if expr "x$xentool" : "x.*xl" >/dev/null; then
+		$xentool list $1 >/dev/null 2>&1
+		if [ $? -ne 0 ]; then
+			return $OCF_NOT_RUNNING
+		else
+			return $OCF_SUCCESS
+		fi
+	fi
+	if have_binary xen-list; then
+		xen-list $1 2>/dev/null | grep -qs "State.*[-r][-b][-p]--" 2>/dev/null
+		if [ $? -ne 0 ]; then
+			return $OCF_NOT_RUNNING
+		else
+			return $OCF_SUCCESS
+		fi
+	fi
+	STATUS=`$xentool list --long $1 2>/dev/null | grep status 2>/dev/null`
+	if [ "X${STATUS}" != "X" ]; then
+		# we have Xen 3.0.4 or higher
+		STATUS_NOSPACES=`echo "$STATUS" | awk '{ print $1,$2}'`
+		if [ "$STATUS_NOSPACES" = "(status 2)" -o "$STATUS_NOSPACES" = "(status 1)" ]; then
+			return $OCF_SUCCESS
+		else
+			return $OCF_NOT_RUNNING
+		fi
+	else
+		# we have Xen 3.0.3 or lower
+		STATUS=`$xentool list --long $1 2>/dev/null | grep state 2>/dev/null`
+		echo "${STATUS}" | grep -qs "[-r][-b][-p]---"
+		if [ $? -ne 0 ]; then
+			return $OCF_NOT_RUNNING
+		else
+			return $OCF_SUCCESS
+		fi
+	fi
 }
 
 # If the guest is rebooting, it may completely disappear from the
-# list of defined guests, thus xm/xen-list would return with not
+# list of defined guests, thus xl/xen-list would return with not
 # running; apparently, this period lasts only for a second or
 # two
 # If a status returns not running, then test status
 # again for 5 times (perhaps it'll show up)
 Xen_Status_with_Retry() {
-  local rc cnt=5
+	local rc cnt=5
 
-  Xen_Status $1
-  rc=$?
-  while [ $rc -eq $OCF_NOT_RUNNING -a $cnt -gt 0 ]; do
-	case "$__OCF_ACTION" in
-	stop)
-	  ocf_log debug "domain $1 reported as not running, waiting $cnt seconds ..."
-	  ;;
-	monitor)
-	  ocf_log warn "domain $1 reported as not running, but it is expected to be running! Retrying for $cnt seconds ..."
-	  ;;
-	*) : not reachable
-		;;
-	esac
-	sleep 1
 	Xen_Status $1
 	rc=$?
-	let cnt=$((cnt-1))
-  done
-  return $rc
+	while [ $rc -eq $OCF_NOT_RUNNING -a $cnt -gt 0 ]; do
+		case "$__OCF_ACTION" in
+			stop)
+				ocf_log debug "domain $1 reported as not running, waiting $cnt seconds ..."
+				;;
+			monitor)
+				ocf_log warn "domain $1 reported as not running, but it is expected to be running! Retrying for $cnt seconds ..."
+				;;
+			*) : not reachable
+				;;
+		esac
+		sleep 1
+		Xen_Status $1
+		rc=$?
+		cnt=$((cnt-1))
+	done
+	return $rc
 }
 
 Xen_Adjust_Memory() {
-    if ocf_is_true "${OCF_RESKEY_allow_mem_management}"; then
-      CNTNEW=$1
-      RUNNING=`Xen_List_running`
-      RUNCNT=`Xen_Count_running`
-      MAXMEM=`Xen_Total_Memory`
-      if [ ${RUNCNT} -eq 0 -a ${CNTNEW} -eq 0 ]; then
-	RUNCNT=1
-      fi
-      #NEWMEM=`echo "(${MAXMEM}-${OCF_RESKEY_reserved_Dom0_memory})/(${RUNCNT}+${CNTNEW})"|bc`
-      NEWMEM=$(( (${MAXMEM} - ${OCF_RESKEY_reserved_Dom0_memory}) / (${RUNCNT} + ${CNTNEW} ) ))
-      # do not rely on ballooning add dom0_mem=512 instead to force memory for dom0
-      #xm mem-set Domain-0 ${OCF_RESKEY_reserved_Dom0_memory}
-      for DOM in ${RUNNING}; do
-        xm mem-set ${DOM} ${NEWMEM} 
-      done
-      ocf_log info "Adjusted memory to: $NEWMEM, for the following $RUNCNT domains: $RUNNING"
-    fi
+	if ocf_is_true "${OCF_RESKEY_allow_mem_management}"; then
+		CNTNEW=$1
+		RUNNING=`Xen_List_running`
+		RUNCNT=`Xen_Count_running`
+		MAXMEM=`Xen_Total_Memory`
+		if [ ${RUNCNT} -eq 0 -a ${CNTNEW} -eq 0 ]; then
+			RUNCNT=1
+		fi
+		#NEWMEM=`echo "(${MAXMEM}-${OCF_RESKEY_reserved_Dom0_memory})/(${RUNCNT}+${CNTNEW})"|bc`
+		NEWMEM=$(( (${MAXMEM} - ${OCF_RESKEY_reserved_Dom0_memory}) / (${RUNCNT} + ${CNTNEW} ) ))
+		# do not rely on ballooning add dom0_mem=512 instead to force memory for dom0
+		#$xentool mem-set Domain-0 ${OCF_RESKEY_reserved_Dom0_memory}
+		for DOM in ${RUNNING}; do
+			$xentool mem-set ${DOM} ${NEWMEM}
+		done
+		ocf_log info "Adjusted memory to: $NEWMEM, for the following $RUNCNT domains: $RUNNING"
+	fi
 }
 
 Xen_List_all() {
-	xm list | grep -v -e "Name" -e "Domain-0" | awk '{print $1}'
+	$xentool list | grep -v -e "Name" -e "Domain-0" | awk '{print $1}'
 }
 Xen_List_running() {
 	ALL_DOMS=`Xen_List_all`
@@ -275,205 +284,226 @@ Xen_Count_running() {
 }
 
 Xen_Monitor() {
-  if ocf_is_probe; then
-	Xen_Status ${DOMAIN_NAME}
-  else
-	Xen_Status_with_Retry ${DOMAIN_NAME}
-  fi
-  if [ $? -eq ${OCF_NOT_RUNNING} ]; then
-	ocf_is_probe ||
-	  ocf_log err "Xen domain $DOMAIN_NAME stopped"
-	return ${OCF_NOT_RUNNING}
-  fi
-  if [ "X${OCF_RESKEY_monitor_scripts}" = "X" ]; then
-	return ${OCF_SUCCESS}
-  fi
-  for SCRIPT in ${OCF_RESKEY_monitor_scripts}; do
-	$SCRIPT
-	if [ $? -ne 0 ]; then
-		return ${OCF_ERR_GENERIC}
+	if ocf_is_probe; then
+		Xen_Status ${DOMAIN_NAME}
+	else
+		Xen_Status_with_Retry ${DOMAIN_NAME}
 	fi
-  done
-  return ${OCF_SUCCESS}
+	if [ $? -eq ${OCF_NOT_RUNNING} ]; then
+		ocf_is_probe ||
+		ocf_log err "Xen domain $DOMAIN_NAME stopped"
+		return ${OCF_NOT_RUNNING}
+	fi
+	if [ "X${OCF_RESKEY_monitor_scripts}" = "X" ]; then
+		return ${OCF_SUCCESS}
+	fi
+	for SCRIPT in ${OCF_RESKEY_monitor_scripts}; do
+		$SCRIPT
+		if [ $? -ne 0 ]; then
+			return ${OCF_ERR_GENERIC}
+		fi
+	done
+	return ${OCF_SUCCESS}
 }
 
 Xen_Total_Memory() {
-	xm info | grep "^total_memory" | awk '{print $3}'
+	$xentool info | grep "^total_memory" | awk '{print $3}'
 }
 
 Xen_Start() {
-  if Xen_Status ${DOMAIN_NAME}; then
-    ocf_log info "Xen domain $DOMAIN_NAME already running."
-    return $OCF_SUCCESS
-  fi
-
-  if [ ! -f "${OCF_RESKEY_xmfile}" ]; then
-    ocf_log err "Config file ${OCF_RESKEY_xmfile} for $DOMAIN_NAME does not exist."
-    return $OCF_ERR_INSTALLED
-  fi
-
-  if ocf_is_true "${OCF_RESKEY_allow_mem_management}"; then
-    Xen_Adjust_Memory 1
-    ocf_log info "New memory for virtual domains: ${NEWMEM}"
-    sed -i -e "/^memory=/ s/^memory=.*/memory=${NEWMEM}/" ${OCF_RESKEY_xmfile}
-    xm mem-set ${DOMAIN_NAME} ${NEWMEM} 
-  fi
-
-  xm create ${OCF_RESKEY_xmfile} name=$DOMAIN_NAME
-  rc=$?
-  if [ $rc -ne 0 ]; then
-    return $OCF_ERR_GENERIC
-  else 
-    if ocf_is_true "${OCF_RESKEY_allow_mem_management}"; then
-      xm mem-set ${DOMAIN_NAME} ${NEWMEM}
-    fi
-  fi
-  while sleep 1; do
-    Xen_Monitor && return $OCF_SUCCESS
-  done
+	if Xen_Status ${DOMAIN_NAME}; then
+		ocf_log info "Xen domain $DOMAIN_NAME already running."
+		return $OCF_SUCCESS
+	fi
+
+	if [ ! -f "${OCF_RESKEY_xmfile}" ]; then
+		ocf_log err "Config file ${OCF_RESKEY_xmfile} for $DOMAIN_NAME does not exist."
+		return $OCF_ERR_INSTALLED
+	fi
+
+	if ocf_is_true "${OCF_RESKEY_allow_mem_management}"; then
+		Xen_Adjust_Memory 1
+		ocf_log info "New memory for virtual domains: ${NEWMEM}"
+		sed -i -e "/^memory=/ s/^memory=.*/memory=${NEWMEM}/" ${OCF_RESKEY_xmfile}
+		$xentool mem-set ${DOMAIN_NAME} ${NEWMEM}
+	fi
+
+	# the latest xl management tool is squeamish about some
+	# characters in a name (the vm name is xen-f):
+	# /etc/xen/vm/xen-f:15: config parsing error near `xen':
+	# syntax error, unexpected IDENT, expecting STRING or NUMBER
+	# or '['
+	# /etc/xen/vm/xen-f:15: config parsing error near `-f': lexical error
+	#
+	# the older xm management tool cannot digest quotes (see
+	# https://developerbugs.linuxfoundation.org/show_bug.cgi?id=2671)
+	#
+	# hence the following
+	if expr "x$xentool" : "x.*xl" >/dev/null; then
+		$xentool create ${OCF_RESKEY_xmfile} name=\"$DOMAIN_NAME\"
+	else
+		$xentool create ${OCF_RESKEY_xmfile} name="$DOMAIN_NAME"
+	fi
+	rc=$?
+
+	if [ $rc -ne 0 ]; then
+		return $OCF_ERR_GENERIC
+	else
+		if ocf_is_true "${OCF_RESKEY_allow_mem_management}"; then
+			$xentool mem-set ${DOMAIN_NAME} ${NEWMEM}
+		fi
+	fi
+	while sleep 1; do
+		Xen_Monitor && return $OCF_SUCCESS
+	done
 }
 
 xen_domain_stop() {
-    local dom=$1
-    local timeout
-
-    if [ -n "$OCF_RESKEY_shutdown_timeout" ]; then
-      timeout=$OCF_RESKEY_shutdown_timeout
-    elif [ -n "$OCF_RESKEY_CRM_meta_timeout" ]; then
-      # Allow 2/3 of the action timeout for the orderly shutdown
-      # (The origin unit is ms, hence the conversion)
-      timeout=$((OCF_RESKEY_CRM_meta_timeout/1500))
-    else
-      timeout=60
-    fi
-
-    if [ "$timeout" -gt 0 ]; then
-      ocf_log info "Xen domain $dom will be stopped (timeout: ${timeout}s)"
-      if ocf_is_true "${OCF_RESKEY_shutdown_acpi}"; then
-        xm trigger $dom power
-      else
-        xm shutdown $dom
-      fi
-          
-      while Xen_Status $dom && [ "$timeout" -gt 0 ]; do
-        ocf_log debug "$dom still not stopped. Waiting..."
-        timeout=$((timeout-1))
-        sleep 1
-      done
-    fi
-    
-    if [ "$timeout" -eq 0 ]; then
-      while Xen_Status $dom; do
-        ocf_log warn "Xen domain $dom will be destroyed!"
-	$xenkill $dom
-        sleep 1
-      done
-      # Note: This does not give up. stop isn't allowed to to fail.
-      # If xm destroy fails, stop will eventually timeout.
-      # This is the correct behaviour.
-    fi
-
-    ocf_log info "Xen domain $dom stopped."
+	local dom=$1
+	local timeout
+
+	if [ -n "$OCF_RESKEY_shutdown_timeout" ]; then
+		timeout=$OCF_RESKEY_shutdown_timeout
+	elif [ -n "$OCF_RESKEY_CRM_meta_timeout" ]; then
+		# Allow 2/3 of the action timeout for the orderly shutdown
+		# (The origin unit is ms, hence the conversion)
+		timeout=$((OCF_RESKEY_CRM_meta_timeout/1500))
+	else
+		timeout=60
+	fi
+
+	if [ "$timeout" -gt 0 ]; then
+		ocf_log info "Xen domain $dom will be stopped (timeout: ${timeout}s)"
+		if ocf_is_true "${OCF_RESKEY_shutdown_acpi}"; then
+			$xentool trigger $dom power
+		else
+			$xentool shutdown $dom
+		fi
+
+		while Xen_Status $dom && [ "$timeout" -gt 0 ]; do
+			ocf_log debug "$dom still not stopped. Waiting..."
+			timeout=$((timeout-1))
+			sleep 1
+		done
+	fi
+
+	if [ "$timeout" -eq 0 ]; then
+		while Xen_Status $dom; do
+			ocf_log warn "Xen domain $dom will be destroyed!"
+			$xenkill $dom
+			sleep 1
+		done
+		# Note: This does not give up. stop isn't allowed to to fail.
+		# If $xentool destroy fails, stop will eventually timeout.
+		# This is the correct behaviour.
+	fi
+
+	ocf_log info "Xen domain $dom stopped."
 }
 
 Xen_Stop() {
-  local vm
-  if Xen_Status_with_Retry ${DOMAIN_NAME}; then
-    vm=${DOMAIN_NAME}
-  elif Xen_Status migrating-${DOMAIN_NAME}; then
-    ocf_log info "Xen domain $DOMAIN_NAME is migrating" 
-    vm="migrating-${DOMAIN_NAME}"
-  else
-    ocf_log info "Xen domain $DOMAIN_NAME already stopped." 
-  fi
-
-  if [ "$vm" ]; then
-    xen_domain_stop $vm
-  else
-  # It is supposed to be gone, but there have been situations where xm
-  # list / xen-list showed it as stopped but it was still instantiated.
-  # Nuke it once more to make sure:
-    $xenkill ${DOMAIN_NAME}
-  fi
-
-  Xen_Adjust_Memory 0
-  return $OCF_SUCCESS
+	local vm
+	if Xen_Status_with_Retry ${DOMAIN_NAME}; then
+		vm=${DOMAIN_NAME}
+	elif Xen_Status migrating-${DOMAIN_NAME}; then
+		ocf_log info "Xen domain $DOMAIN_NAME is migrating"
+		vm="migrating-${DOMAIN_NAME}"
+	else
+		ocf_log info "Xen domain $DOMAIN_NAME already stopped."
+	fi
+
+	if [ "$vm" ]; then
+		xen_domain_stop $vm
+	else
+		# It is supposed to be gone, but there have been situations where
+		# $xentool list / xen-list showed it as stopped but it was still
+		# instantiated. Nuke it once more to make sure:
+		$xenkill ${DOMAIN_NAME}
+	fi
+
+	Xen_Adjust_Memory 0
+	return $OCF_SUCCESS
 }
 
 Xen_Migrate_To() {
-  target_node="$OCF_RESKEY_CRM_meta_migrate_target"
-  target_attr="$OCF_RESKEY_node_ip_attribute"
-  target_addr="$target_node"
- 
-  if Xen_Status ${DOMAIN_NAME}; then
-    ocf_log info "$DOMAIN_NAME: Starting xm migrate to $target_node"
-    
-    if [ -n "$target_attr" ]; then
-	  nodevalue=`crm_attribute --type nodes --node-uname $target_node --attr-name $target_attr --get-value -q`
-	  if [ -n "${nodevalue}" -a "${nodevalue}" != "(null)" ]; then
-	  	target_addr="$nodevalue"
-		ocf_log info "$DOMAIN_NAME: $target_node is using address $target_addr"
-	  fi
-    fi
-
-    xm migrate --live $DOMAIN_NAME $target_addr
-
-    rc=$?
-    if [ $rc -ne 0 ]; then
-      ocf_log err "$DOMAIN_NAME: xm migrate to $target_node failed: $rc"
-      return $OCF_ERR_GENERIC
-    else 
-      Xen_Adjust_Memory 0
-      ocf_log info "$DOMAIN_NAME: xm migrate to $target_node succeeded."
-      return $OCF_SUCCESS
-    fi
-  else
-    ocf_log err "$DOMAIN_NAME: migrate_to: Not active locally!"
-    return $OCF_ERR_GENERIC
-  fi
+	target_node="$OCF_RESKEY_CRM_meta_migrate_target"
+	target_attr="$OCF_RESKEY_node_ip_attribute"
+	target_addr="$target_node"
+
+	if Xen_Status ${DOMAIN_NAME}; then
+		ocf_log info "$DOMAIN_NAME: Starting $xentool migrate to $target_node"
+
+		if [ -n "$target_attr" ]; then
+			nodevalue=`crm_attribute --type nodes --node-uname $target_node --attr-name $target_attr --get-value -q`
+			if [ -n "${nodevalue}" -a "${nodevalue}" != "(null)" ]; then
+				target_addr="$nodevalue"
+				ocf_log info "$DOMAIN_NAME: $target_node is using address $target_addr"
+			fi
+		fi
+
+		if expr "x$xentool" : "x.*xm" >/dev/null; then
+			$xentool migrate --live $DOMAIN_NAME $target_addr
+		else
+			$xentool migrate $DOMAIN_NAME $target_addr
+		fi
+		rc=$?
+		if [ $rc -ne 0 ]; then
+			ocf_log err "$DOMAIN_NAME: $xentool migrate to $target_node failed: $rc"
+			return $OCF_ERR_GENERIC
+		else
+			Xen_Adjust_Memory 0
+			ocf_log info "$DOMAIN_NAME: $xentool migrate to $target_node succeeded."
+			return $OCF_SUCCESS
+		fi
+	else
+		ocf_log err "$DOMAIN_NAME: migrate_to: Not active locally!"
+		return $OCF_ERR_GENERIC
+	fi
 }
 
 Xen_Migrate_From() {
-  if [ -n "$OCF_RESKEY_CRM_meta_timeout" ]; then
-      # Allow 2/3 of the action timeout for status to stabilize
-      # (The origin unit is ms, hence the conversion)
-      timeout=$((OCF_RESKEY_CRM_meta_timeout/1500))
-  else
-      timeout=10		# should be plenty
-  fi
-
-  while ! Xen_Status ${DOMAIN_NAME} && [ $timeout -gt 0 ]; do
-    ocf_log debug "$DOMAIN_NAME: Not yet active locally, waiting (timeout: ${timeout}s)"
-    timeout=$((timeout-1))
-    sleep 1
-  done
-
-  if Xen_Status ${DOMAIN_NAME}; then
-    Xen_Adjust_Memory 0
-    ocf_log info "$DOMAIN_NAME: Active locally, migration successful"
-    return $OCF_SUCCESS
-  else
-    ocf_log err "$DOMAIN_NAME: Not active locally, migration failed!"
-    return $OCF_ERR_GENERIC
-  fi
+	if [ -n "$OCF_RESKEY_CRM_meta_timeout" ]; then
+		# Allow 2/3 of the action timeout for status to stabilize
+		# (The origin unit is ms, hence the conversion)
+		timeout=$((OCF_RESKEY_CRM_meta_timeout/1500))
+	else
+		timeout=10		# should be plenty
+	fi
+
+	while ! Xen_Status ${DOMAIN_NAME} && [ $timeout -gt 0 ]; do
+		ocf_log debug "$DOMAIN_NAME: Not yet active locally, waiting (timeout: ${timeout}s)"
+		timeout=$((timeout-1))
+		sleep 1
+	done
+
+	if Xen_Status ${DOMAIN_NAME}; then
+		Xen_Adjust_Memory 0
+		ocf_log info "$DOMAIN_NAME: Active locally, migration successful"
+		return $OCF_SUCCESS
+	else
+		ocf_log err "$DOMAIN_NAME: Not active locally, migration failed!"
+		return $OCF_ERR_GENERIC
+	fi
 }
 
 Xen_Validate_All() {
-  return $OCF_SUCCESS
+	return $OCF_SUCCESS
 }
 
 if [ $# -ne 1 ]; then
-  usage
-  exit $OCF_ERR_ARGS
+	usage
+	exit $OCF_ERR_ARGS
 fi
 
 case $1 in
-  meta-data)		meta_data
-			exit $OCF_SUCCESS
-			;;
-  usage)		usage
-			exit $OCF_SUCCESS
-			;;
+	meta-data)
+		meta_data
+		exit $OCF_SUCCESS
+		;;
+	usage)
+		usage
+		exit $OCF_SUCCESS
+		;;
 esac
 
 # the name business:
@@ -495,41 +525,48 @@ else
 	DOMAIN_NAME=${DOMAIN_NAME:-${OCF_RESOURCE_INSTANCE}}
 fi
 
-for binary in xm sed awk; do
-    check_binary $binary
+for binary in sed awk; do
+	check_binary $binary
 done
 
 if have_binary xen-destroy ; then
-  xenkill="xen-destroy"
+	xenkill="xen-destroy"
 else
-  xenkill="xm destroy"
+	xenkill="$xentool destroy"
 fi
 
 if [ -n "$OCF_RESKEY_shutdown_timeout" ]; then
 	ocf_is_decimal "$OCF_RESKEY_shutdown_timeout" || {
-		ocf_log err "shutdown_timeout must be a number" 
+		ocf_log err "shutdown_timeout must be a number"
 		exit $OCF_ERR_CONFIGURED
 	}
 fi
 
 case $1 in
-  start)		Xen_Start
-			;;
-  stop)			Xen_Stop
-			;;
-  migrate_to)		Xen_Migrate_To
-			;;
-  migrate_from)		Xen_Migrate_From
-			;;
-  monitor)		Xen_Monitor
-			;;
-  status)		Xen_Status ${DOMAIN_NAME}
-			;;
-  validate-all)		Xen_Validate_All
-			;;
-  *)			usage
-			exit $OCF_ERR_UNIMPLEMENTED
-			;;
+	start)
+		Xen_Start
+		;;
+	stop)
+		Xen_Stop
+		;;
+	migrate_to)
+		Xen_Migrate_To
+		;;
+	migrate_from)
+		Xen_Migrate_From
+		;;
+	monitor)
+		Xen_Monitor
+		;;
+	status)
+		Xen_Status ${DOMAIN_NAME}
+		;;
+	validate-all)
+		Xen_Validate_All
+		;;
+	*)
+		usage
+		exit $OCF_ERR_UNIMPLEMENTED
+		;;
 esac
 exit $?
-# vim:sw=2:ts=4:
diff --git a/heartbeat/Xinetd b/heartbeat/Xinetd
index ee2c4fa..b6a7b56 100755
--- a/heartbeat/Xinetd
+++ b/heartbeat/Xinetd
@@ -89,11 +89,11 @@ hup_inetd () {
 	if kill -s HUP $pid; then
             ocf_log info "asked xinetd to reload by sending SIGHUP to process $pid!"
 	else
-            ocf_log err "could not send SIGHUP to process $pid!"
+            ocf_exit_reason "could not send SIGHUP to process $pid!"
             exit $OCF_ERR_GENERIC
 	fi
     else
-	ocf_log err "xinetd process not found!"
+	ocf_exit_reason "xinetd process not found!"
 	exit $OCF_ERR_GENERIC
     fi
 }
@@ -161,7 +161,7 @@ xup_usage () {
 
 xup_validate_all () {
     if [ ! -f "$SVCDEF" ]; then
-        ocf_log err "service $service missing $SVCDEF"
+        ocf_exit_reason "service $service missing $SVCDEF"
 	return $OCF_ERR_INSTALLED
     fi
     return $OCF_SUCCESS
@@ -185,7 +185,7 @@ case "$1" in
 esac
 
 if [ -z "$OCF_RESKEY_service" ]; then
-    ocf_log err "please define \"service\" parameter"
+    ocf_exit_reason "please define \"service\" parameter"
     if [ "$1" = "start" ]; then 
 	exit $OCF_ERR_CONFIGURED
     else
@@ -195,16 +195,26 @@ fi
 
 # Is xinetd running at all
 if [ -z "`get_xinetd_pid`" ]; then
-    ocf_log err "xinetd not running, we manage just xinetd services, not the daemon itself"
     case "$1" in
     stop) exit $OCF_SUCCESS;;
-    start|monitor|status) exit $OCF_ERR_INSTALLED;;
+    start)
+	ocf_exit_reason "xinetd not running, we manage just xinetd services, not the daemon itself"
+	exit $OCF_ERR_INSTALLED
+	;;
+    status|monitor)
+	if ocf_is_probe; then
+	    exit $OCF_NOT_RUNNING
+	else
+	    ocf_exit_reason "xinetd stopped"
+	    exit $OCF_ERR_GENERIC
+	fi
+	;;
     esac
 fi
 
 # Make sure the OCF_RESKEY_service is a valid xinetd service name
 if [ ! -f $SVCDEF ]; then
-    ocf_log err "service definition $SVCDEF not found!"
+    ocf_exit_reason "service definition $SVCDEF not found!"
     if [ "$1" = "start" ]; then 
 	exit $OCF_ERR_INSTALLED
     else
diff --git a/heartbeat/anything b/heartbeat/anything
index f7debf9..96dc14e 100755
--- a/heartbeat/anything
+++ b/heartbeat/anything
@@ -76,14 +76,9 @@ anything_start() {
 		then
 			# We have logfile and errlogfile, so redirect STDOUT und STDERR to different files
 			cmd="su - $user -c \"cd $workdir; nohup $binfile $cmdline_options >> $logfile 2>> $errlogfile & \"'echo \$!' "
-		else if [ -n "$logfile" ]
-			then
-				# We only have logfile so redirect STDOUT and STDERR to the same file
-				cmd="su - $user -c \"cd $workdir; nohup $binfile $cmdline_options >> $logfile 2>&1 & \"'echo \$!' "
-			else
-				# We have neither logfile nor errlogfile, so we're not going to redirect anything
-				cmd="su - $user -c \"cd $workdir; nohup $binfile $cmdline_options & \"'echo \$!'"
-			fi
+		else
+			# We only have logfile so redirect STDOUT and STDERR to the same file
+			cmd="su - $user -c \"cd $workdir; nohup $binfile $cmdline_options >> $logfile 2>&1 & \"'echo \$!' "
 		fi
 		ocf_log debug "Starting $process: $cmd"
 		# Execute the command as created above
@@ -133,14 +128,17 @@ anything_stop() {
                 done
                 ocf_log warn "Stop with SIGTERM failed/timed out, now sending SIGKILL."
                 kill -s 9 $pid
-                if ! anything_status
-                then
-                        ocf_log warn "SIGKILL did the job."
-                        rc=$OCF_SUCCESS
-                else
-                        ocf_log err "Failed to stop - even with SIGKILL."
-                        rc=$OCF_ERR_GENERIC
-                fi
+                while :
+                do
+                        if ! anything_status
+                        then
+                                ocf_log warn "SIGKILL did the job."
+                                rc=$OCF_SUCCESS
+                                break
+                        fi
+                        ocf_log info "The job still hasn't stopped yet. Waiting..."
+                        sleep 1
+                done
 	fi
 	rm -f $pidfile 
 	return $rc
@@ -172,7 +170,7 @@ cmdline_options="$OCF_RESKEY_cmdline_options"
 workdir="$OCF_RESKEY_workdir"
 pidfile="$OCF_RESKEY_pidfile"
 [ -z "$pidfile" ] && pidfile=${HA_VARRUN}/anything_${process}.pid
-logfile="$OCF_RESKEY_logfile"
+logfile="${OCF_RESKEY_logfile:-/dev/null}"
 errlogfile="$OCF_RESKEY_errlogfile"
 user="$OCF_RESKEY_user"
 [ -z "$user" ] && user=root
@@ -249,7 +247,7 @@ File to read/write the PID from/to.
 File to write STDOUT to
 </longdesc>
 <shortdesc lang="en">File to write STDOUT to</shortdesc>
-<content type="string" />
+<content type="string" default="/dev/null" />
 </parameter>
 <parameter name="errlogfile" required="0">
 <longdesc lang="en">
diff --git a/heartbeat/apache b/heartbeat/apache
index d72c534..ab7c43f 100755
--- a/heartbeat/apache
+++ b/heartbeat/apache
@@ -86,7 +86,7 @@ CMD=`basename $0`
 #	will be either $DEFAULT_RHELCONFIG or $DEFAULT_SUSECONFIG depending
 #	on which is detected.
 usage() {
-cat <<-!
+cat <<-END
 usage: $0 action
 
 action:
@@ -104,7 +104,7 @@ action:
 	meta-data	show meta data message
 
 	validate-all	validate the instance parameters
-	!
+END
 }
 
 get_pid() {
@@ -162,11 +162,15 @@ validate_default_suse_config() {
 		grep -Eq '^Include[[:space:]]+/etc/apache2/sysconfig.d/include.conf' "$CONFIGFILE"
 	then
 		[ -x "/usr/sbin/a2enmod" ] && ocf_run -q /usr/sbin/a2enmod status
-		ocf_run -q /etc/init.d/apache2 configtest
-		return
-	else
-		return 0
+		# init script style, for crusty old SUSE
+		if [ -e "/etc/init.d/apache2" ]; then
+			ocf_run -q /etc/init.d/apache2 configtest || return 1
+		# systemd style, for shiny new SUSE
+		elif [ -e "/usr/sbin/start_apache2" ]; then
+			ocf_run -q /usr/sbin/start_apache2 -t || return 1
+		fi
 	fi
+	return 0
 }
 
 apache_start() {
@@ -284,7 +288,7 @@ apache_stop() {
 		kill_stop $pid
 
 		if ProcessRunning $pid; then
-			ocf_log info "$CMD still running ($pid). Killing pid failed."
+			ocf_exit_reason "$CMD still running ($pid). Killing pid failed."
 			ret=$OCF_ERR_GENERIC
 		fi
 	fi
@@ -315,7 +319,7 @@ apache_monitor_10() {
 		return $OCF_SUCCESS
 	else
 		if ! ocf_is_probe; then
-			ocf_log err "Failed to access httpd status page."
+			ocf_exit_reason "Failed to access httpd status page."
 		fi
 		return $OCF_ERR_GENERIC
 	fi
@@ -366,7 +370,7 @@ apache_monitor_basic() {
 	fi
 
 	if ! ocf_is_probe; then
-		ocf_log err "Failed to access httpd status page."
+		ocf_exit_reason "Failed to access httpd status page."
 	fi
 	return $OCF_ERR_GENERIC
 }
@@ -379,7 +383,7 @@ apache_monitor() {
 
 	ourhttpclient=`findhttpclient`  # we'll need one
 	if [ -z "$ourhttpclient" ]; then
-		ocf_log err "could not find a http client; make sure that either wget or curl is available"
+		ocf_exit_reason "could not find a http client; make sure that either wget or curl is available"
 		return $OCF_ERR_INSTALLED
 	fi
 
@@ -578,15 +582,15 @@ END
 
 apache_validate_all() {
 	if [ -z "$HTTPD" ]; then
-		ocf_log err "apache httpd program not found"
+		ocf_exit_reason "apache httpd program not found"
 		return $OCF_ERR_INSTALLED
 	fi
 	if [ ! -x "$HTTPD" ]; then
-		ocf_log err "HTTPD $HTTPD not found or is not an executable!"
+		ocf_exit_reason "HTTPD $HTTPD not found or is not an executable!"
 		return $OCF_ERR_INSTALLED
 	fi
 	if [ ! -f $CONFIGFILE ]; then
-		ocf_log err "Configuration file $CONFIGFILE not found!"
+		ocf_exit_reason "Configuration file $CONFIGFILE not found!"
 		return $OCF_ERR_INSTALLED
 	fi
 	return $OCF_SUCCESS
diff --git a/heartbeat/asterisk b/heartbeat/asterisk
index cc78ba1..2d9f076 100755
--- a/heartbeat/asterisk
+++ b/heartbeat/asterisk
@@ -220,7 +220,7 @@ asterisk_validate() {
             ocf_log err "Config $OCF_RESKEY_config doesn't exist"
             return $OCF_ERR_INSTALLED
         fi
-        ocf_log_warn "Config $OCF_RESKEY_config not available during a probe"
+        ocf_log warn "Config $OCF_RESKEY_config not available during a probe"
     fi
 
     getent passwd $OCF_RESKEY_user >/dev/null 2>&1
diff --git a/heartbeat/clvm b/heartbeat/clvm
new file mode 100755
index 0000000..9d312cc
--- /dev/null
+++ b/heartbeat/clvm
@@ -0,0 +1,410 @@
+#!/bin/bash
+#
+# Copyright (c) 2014 David Vossel <dvossel at redhat.com>
+#                    All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of version 2 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# Further, this software is distributed without any warranty that it is
+# free of the rightful claim of any third person regarding infringement
+# or the like.  Any license provided herein, whether implied or
+# otherwise, applies only to this software file.  Patent licenses, if
+# any, provided herein do not apply to combinations of this program with
+# other software, or any other product whatsoever.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
+#
+
+#######################################################################
+# Initialization:
+
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+. ${OCF_FUNCTIONS_DIR}/ocf-directories
+
+#######################################################################
+
+meta_data() {
+	cat <<END
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="clvm" version="0.9">
+<version>1.0</version>
+
+<longdesc lang="en">
+This agent manages the clvmd daemon.
+</longdesc>
+<shortdesc lang="en">clvmd</shortdesc>
+
+<parameters>
+<parameter name="with_cmirrord" unique="0" required="0">
+<longdesc lang="en">
+Start with cmirrord (cluster mirror log daemon).
+</longdesc>
+<shortdesc lang="en">activate cmirrord</shortdesc>
+<content type="boolean" default="false" />
+</parameter>
+
+<parameter name="daemon_options" unique="0">
+<longdesc lang="en">
+Options to clvmd. Refer to clvmd.8 for detailed descriptions.
+</longdesc>
+<shortdesc lang="en">Daemon Options</shortdesc>
+<content type="string" default="-d0"/>
+</parameter>
+</parameters>
+
+<actions>
+<action name="start"        timeout="90" />
+<action name="stop"         timeout="90" />
+<action name="monitor"      timeout="90" interval="30" depth="0" />
+<action name="reload"       timeout="90" />
+<action name="meta-data"    timeout="10" />
+<action name="validate-all"   timeout="20" />
+</actions>
+</resource-agent>
+END
+}
+
+#######################################################################
+
+: ${OCF_RESKEY_daemon_options:="-d0"}
+
+sbindir=$HA_SBIN_DIR
+if [ -z $sbindir ]; then
+	sbindir=/usr/sbin
+fi
+DAEMON="clvmd"
+CMIRROR="cmirrord"
+DAEMON_PATH="${sbindir}/clvmd"
+CMIRROR_PATH="${sbindir}/cmirrord"
+LVMCONF="${sbindir}/lvmconf"
+LOCK_FILE="/var/lock/subsys/$DAEMON"
+
+# attempt to detect where the vg tools are located
+# for some reason this isn't consistent with sbindir
+# in some distros.
+vgtoolsdir=$(dirname $(which vgchange 2> /dev/null) 2> /dev/null)
+if [ -z "$vgtoolsdir" ]; then
+       vgtoolsdir="$sbindir"
+fi
+
+LVM_VGCHANGE=${vgtoolsdir}/vgchange
+LVM_VGDISPLAY=${vgtoolsdir}/vgdisplay
+LVM_VGSCAN=${vgtoolsdir}/vgscan
+
+# Leaving this in for legacy. We do not want to advertize
+# the abilty to set options in the systconfig exists, we want
+# to expand the OCF style options as necessary instead.
+[ -f /etc/sysconfig/cluster ] && . /etc/sysconfig/cluster
+[ -f /etc/sysconfig/$DAEMON ] && . /etc/sysconfig/$DAEMON
+
+CLVMD_TIMEOUT="90"
+if [ -n "$OCF_RESKEY_CRM_meta_timeout" ]; then
+	CLVMD_TIMEOUT=$(($OCF_RESKEY_CRM_meta_timeout/1000))
+fi
+
+clvmd_usage()
+{
+	cat <<END
+usage: $0 {start|stop|monitor|validate-all|meta-data}
+
+Expects to have a fully populated OCF RA-compliant environment set.
+END
+}
+
+clvmd_validate()
+{
+	# check_binary will exit with OCF_ERR_INSTALLED
+	# when binary is missing
+	check_binary "pgrep"
+	check_binary $DAEMON_PATH
+	if ocf_is_true $OCF_RESKEY_with_cmirrord; then
+		check_binary $CMIRROR_PATH
+	fi
+
+	if [ "$__OCF_ACTION" != "monitor" ]; then
+		check_binary "killall"
+		check_binary $LVM_VGCHANGE
+		check_binary $LVM_VGDISPLAY
+		check_binary $LVM_VGSCAN
+	fi
+
+	# Future validation checks here.
+	return $OCF_SUCCESS
+}
+
+check_process()
+{
+	local binary=$1
+	local pidfile="${HA_RSCTMP}/${binary}-${OCF_RESOURCE_INSTANCE}.pid"
+	local pid
+
+	ocf_log debug "Checking status for ${binary}."
+	if [ -e "$pidfile" ]; then
+		cat /proc/$(cat $pidfile)/cmdline 2>/dev/null | grep -a "${binary}" > /dev/null 2>&1
+		if [ $? -eq 0 ];then
+			# shortcut without requiring pgrep to search through all procs
+			return $OCF_SUCCESS
+		fi
+	fi
+
+	pid=$(pgrep ${binary})
+	case $? in
+		0)
+			ocf_log info "PID file (pid:${pid} at $pidfile) created for ${binary}."
+			echo "$pid" > $pidfile
+			return $OCF_SUCCESS;;
+		1)
+			rm -f "$pidfile" > /dev/null 2>&1 
+			ocf_log info "$binary is not running"
+			return $OCF_NOT_RUNNING;;
+		*)
+			rm -f "$pidfile" > /dev/null 2>&1 
+			ocf_exit_reason "Error encountered detecting pid status of $binary"
+			return $OCF_ERR_GENERIC;;
+	esac
+}
+
+clvmd_status()
+{
+	local rc
+	local mirror_rc
+	clvmd_validate
+	if [ $? -ne $OCF_SUCCESS ]; then
+		ocf_exit_reason "Unable to monitor, Environment validation failed."
+		return $?
+	fi
+
+	check_process $DAEMON
+	rc=$?
+	mirror_rc=$rc
+
+	if ocf_is_true $OCF_RESKEY_with_cmirrord; then
+		check_process $CMIRROR
+		mirror_rc=$?
+	fi
+
+	# If these ever don't match, return error to force recovery
+	if [ $mirror_rc -ne $rc ]; then
+		return $OCF_ERR_GENERIC
+	fi
+
+	return $rc
+}
+
+# NOTE: replace this with vgs, once display filter per attr is implemented.
+clustered_vgs() {
+	${LVM_VGDISPLAY} 2>/dev/null | awk 'BEGIN {RS="VG Name"} {if (/Clustered/) print $1;}'
+}
+
+wait_for_process()
+{
+	local binary=$1
+	local timeout=$2
+	local count=0
+
+	ocf_log info "Waiting for $binary to exit"
+	while [ $count -le $timeout ]; do
+		check_process $binary
+		if [ $? -eq $OCF_NOT_RUNNING ]; then
+			ocf_log info "$binary terminated"
+			return $OCF_SUCCESS
+		fi
+		sleep 1
+		count=$((count+1))
+	done
+
+	return $OCF_ERR_GENERIC
+}
+
+time_left()
+{
+	local end=$1
+	local default=$2
+	local now=$SECONDS
+	local result=0
+
+	result=$(( $end - $now ))
+	if [ $result -lt $default ]; then
+		return $default
+	fi
+	return $result
+}
+
+clvmd_stop()
+{
+	local LVM_VGS
+	local rc=$OCF_SUCCESS
+	local end=$(( $SECONDS + $CLVMD_TIMEOUT ))
+
+	clvmd_status
+	if [ $? -eq $OCF_NOT_RUNNING ]; then
+		return $OCF_SUCCESS
+	fi
+
+	check_process $DAEMON
+	if [ $? -ne $OCF_NOT_RUNNING ]; then
+		LVM_VGS="$(clustered_vgs)"
+
+		if [ -n "$LVM_VGS" ]; then
+			ocf_log info "Deactivating clustered VG(s):" 
+			ocf_run ${LVM_VGCHANGE} -anl $LVM_VGS
+			if [ $? -ne 0 ]; then
+				ocf_exit_reason "Failed to deactivate volume groups, cluster vglist = $LVM_VGS" 
+				return $OCF_ERR_GENERIC
+			fi
+		fi
+
+		ocf_log info "Signaling $DAEMON to exit"
+		killall -TERM $DAEMON
+		if [ $? != 0 ]; then
+			ocf_exit_reason "Failed to signal -TERM to $DAEMON"
+			return $OCF_ERR_GENERIC
+		fi
+
+		wait_for_process $DAEMON $CLVMD_TIMEOUT
+		rc=$?
+		if [ $rc -ne $OCF_SUCCESS ]; then
+			ocf_exit_reason "$DAEMON failed to exit"
+			return $rc
+		fi
+
+		rm -f $LOCK_FILE
+	fi
+
+	check_process $CMIRROR
+	if [ $? -ne $OCF_NOT_RUNNING ] && ocf_is_true $OCF_RESKEY_with_cmirrord; then
+		local timeout
+		ocf_log info "Signaling $CMIRROR to exit"
+		killall -INT $CMIRROR
+
+		time_left $end 10; timeout=$?
+		wait_for_process $CMIRROR $timeout
+		rc=$?
+		if [ $rc -ne $OCF_SUCCESS ]; then
+			killall -KILL $CMIRROR
+			time_left $end 10; timeout=$?
+			wait_for_process $CMIRROR $(time_left $end 10)
+			rc=$?
+		fi
+	fi
+
+	return $rc
+}
+
+start_process()
+{
+	local binary_path=$1
+	local opts=$2
+
+	check_process "$(basename $binary_path)"
+	if [ $? -ne $OCF_SUCCESS ]; then
+		ocf_log info "Starting $binary_path: "
+		ocf_run $binary_path $opts
+		rc=$?
+		if [ $rc -ne 0 ]; then
+			ocf_exit_reason "Failed to launch $binary_path, exit code $rc"
+			exit $OCF_ERR_GENERIC
+		fi
+	fi
+
+	return $OCF_SUCCESS
+}
+
+clvmd_activate_all()
+{
+	# Activate all volume groups by leaving the
+	# "volume group name" parameter empty
+	ocf_run ${LVM_VGCHANGE} -aay
+	if [ $? -ne 0 ]; then
+		ocf_log info "Failed to activate VG(s):"
+		clvmd_stop
+		return $OCF_ERR_GENERIC
+	fi
+	return $OCF_SUCCESS
+}
+
+clvmd_start()
+{
+	local rc=0
+	local CLVMDOPTS="-T${CLVMD_TIMEOUT} $OCF_RESKEY_daemon_options"
+
+	clvmd_validate
+	if [ $? -ne $OCF_SUCCESS ]; then
+		ocf_exit_reason "Unable to start, Environment validation failed."
+		return $?
+	fi
+
+	clvmd_status
+	if [ $? -eq $OCF_SUCCESS ]; then
+		ocf_log debug "$DAEMON already started"
+		clvmd_activate_all
+		return $?;
+	fi
+
+	# autoset locking type to clusted when lvmconf tool is available
+	if [ -x "$LVMCONF"  ]; then
+		$LVMCONF --enable-cluster > /dev/null 2>&1
+	fi
+
+	# if either of these fail, script will exit OCF_ERR_GENERIC
+	if ocf_is_true $OCF_RESKEY_with_cmirrord; then
+		start_process $CMIRROR_PATH
+	fi
+	start_process $DAEMON_PATH $CLVMDOPTS
+
+	# Refresh local cache.
+	#
+	# It's possible that new PVs were added to this, or other VGs
+	# while this node was down. So we run vgscan here to avoid
+	# any potential "Missing UUID" messages with subsequent
+	# LVM commands.
+
+	# The following step would be better and more informative to the user:
+	# 'action "Refreshing VG(s) local cache:" ${LVM_VGSCAN}'
+	# but it could show warnings such as:
+	# 'clvmd not running on node x-y-z  Unable to obtain global lock.'
+	# and the action would be shown as FAILED when in reality it didn't.
+	# Ideally vgscan should have a startup mode that would not print
+	# unnecessary warnings.
+
+	${LVM_VGSCAN} > /dev/null 2>&1
+	touch $LOCK_FILE
+
+	clvmd_activate_all
+
+	clvmd_status
+	return $?
+}
+
+case $__OCF_ACTION in
+	meta-data)   meta_data
+		exit $OCF_SUCCESS;;
+
+	start)         clvmd_start;;
+
+	stop)          clvmd_stop;;
+
+	monitor)       clvmd_status;;
+
+	validate-all)  clvmd_validate;;
+
+	usage|help)    clvmd_usage;;
+
+	*)             clvmd_usage
+	               exit $OCF_ERR_UNIMPLEMENTED;;
+esac
+
+rc=$?
+ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION : $rc"
+exit $rc
+
diff --git a/heartbeat/conntrackd b/heartbeat/conntrackd
index 32eab6b..e81cda3 100755
--- a/heartbeat/conntrackd
+++ b/heartbeat/conntrackd
@@ -98,7 +98,7 @@ meta_expect()
                 # [, not [[, or it won't work ;)
                 [ $val $op $expect ] && return
         fi
-        ocf_log err "meta parameter misconfigured, expected $what $op $expect, but found ${val:-unset}."
+        ocf_exit_reason "meta parameter misconfigured, expected $what $op $expect, but found ${val:-unset}."
         exit $OCF_ERR_CONFIGURED
 }
 
@@ -123,7 +123,7 @@ conntrackd_monitor() {
 		# now see if it acceppts queries
 		if ! $OCF_RESKEY_binary -C $OCF_RESKEY_config -s > /dev/null 2>&1; then
 			rc=$OCF_ERR_GENERIC
-			ocf_log err "conntrackd is running but not responding to queries"
+			ocf_exit_reason "conntrackd is running but not responding to queries"
 		fi
 		if conntrackd_is_master; then
 			rc=$OCF_RUNNING_MASTER
@@ -154,7 +154,7 @@ conntrackd_start() {
 			conntrackd_set_master_score $slave_score
 			# -n = request resync from the others
 			if ! $OCF_RESKEY_binary -C $OCF_RESKEY_config -n; then
-				ocf_log err "$OCF_RESKEY_binary -C $OCF_RESKEY_config -n failed during start."
+				ocf_exit_reason "$OCF_RESKEY_binary -C $OCF_RESKEY_config -n failed during start."
 				rc=$OCF_ERR_GENERIC
 			else
 				rc=$OCF_SUCCESS
@@ -170,7 +170,7 @@ conntrackd_start() {
 			ha_pseudo_resource $statefile stop
 			;;
 		$OCF_ERR_GENERIC)
-			ocf_log err "conntrackd start failed"
+			ocf_exit_reason "conntrackd start failed"
 			rc=$OCF_ERR_GENERIC
 			break
 			;;
@@ -208,7 +208,7 @@ conntrackd_stop() {
 conntrackd_validate_all() {
 	check_binary "$OCF_RESKEY_binary"
 	if ! [ -e "$OCF_RESKEY_config" ]; then
-		ocf_log err "Config FILE $OCF_RESKEY_config does not exist"
+		ocf_exit_reason "Config FILE $OCF_RESKEY_config does not exist"
 		return $OCF_ERR_INSTALLED
 	fi
         meta_expect master-node-max = 1
@@ -227,7 +227,7 @@ conntrackd_promote() {
 		# -B = send a bulk update on the line
 		for parm in c f R B; do
 			if ! $OCF_RESKEY_binary -C $OCF_RESKEY_config -$parm; then
-				ocf_log err "$OCF_RESKEY_binary -C $OCF_RESKEY_config -$parm failed during promote."
+				ocf_exit_reason "$OCF_RESKEY_binary -C $OCF_RESKEY_config -$parm failed during promote."
 				rc=$OCF_ERR_GENERIC
 				break
 			fi
@@ -245,7 +245,7 @@ conntrackd_demote() {
 		# -n = request a resync from the others
 		for parm in t n; do
 			if ! $OCF_RESKEY_binary -C $OCF_RESKEY_config -$parm; then
-                        	ocf_log err "$OCF_RESKEY_binary -C $OCF_RESKEY_config -$parm failed during demote."
+                        	ocf_exit_reason "$OCF_RESKEY_binary -C $OCF_RESKEY_config -$parm failed during demote."
                         	rc=$OCF_ERR_GENERIC
                         	break
                 	fi
@@ -284,7 +284,7 @@ Expects to have a fully populated OCF RA-compliant environment set.
 EOF
 }
 
-statefile=conntrackd.${OCF_RESOURCE_INSTANCE}.master
+statefile=conntrackd.${OCF_RESOURCE_INSTANCE//:[0-9]*}.master
 
 master_score=1000
 slave_score=100
diff --git a/heartbeat/db2 b/heartbeat/db2
index f9db2f8..fed2d86 100755
--- a/heartbeat/db2
+++ b/heartbeat/db2
@@ -132,6 +132,9 @@ END
 db2_validate() {
     local db2home db2sql db2instance
 
+    # db2 uses korn shell
+    check_binary "ksh"
+
     # check required instance vars
     if [ -z "$OCF_RESKEY_instance" ]
     then
@@ -208,6 +211,14 @@ db2_validate() {
     return $OCF_SUCCESS
 }
 
+master_score()
+{
+    if ! have_binary "crm_master"; then
+        return
+    fi
+
+    crm_master $*
+}
 
 #
 # Run the given command as db2 instance user
@@ -380,8 +391,17 @@ db2_check_config_compatibility() {
 #
 db2_start() {
     local output start_cmd db
+    local start_opts="dbpartitionnum $db2node"
+
+    # If we detect that db partitions are not in use, and no
+    # partition is explicitly specified, activate without
+    # partition information. This allows db2 instances without
+    # partition support to be managed. 
+    if [ -z "$OCF_RESKEY_dbpartitionnum" ] && ! [ -a "$db2sql/db2nodes.cfg" ]; then
+        start_opts=""
+    fi
 
-    if output=$(runasdb2 db2start dbpartitionnum $db2node)
+    if output=$(runasdb2 db2start $start_opts)
     then
         ocf_log info "DB2 instance $instance($db2node) started: $output"
     else
@@ -473,10 +493,15 @@ db2_start() {
 #
 db2_stop_bg() {
     local rc output
+    local stop_opts="dbpartitionnum $db2node"
 
     rc=$OCF_SUCCESS
 
-    if output=$(runasdb2 db2stop force dbpartitionnum $db2node)
+    if [ -z "$OCF_RESKEY_dbpartitionnum" ] && ! [ -a "$db2sql/db2nodes.cfg" ]; then
+        stop_opts=""
+    fi
+
+    if output=$(runasdb2 db2stop force $stop_opts)
     then
         ocf_log info "DB2 instance $instance($db2node) stopped: $output"
     else
@@ -502,13 +527,13 @@ db2_stop() {
     local stop_timeout grace_timeout stop_bg_pid i must_kill
 
     # remove master score
-    crm_master -D -l reboot
+    master_score -D -l reboot
 
     # be very early here in order to avoid stale data
     rm -f $STATE_FILE
 
-    if ! db2_instance_status
-    then
+    db2_instance_status
+    if [ $? -eq $OCF_NOT_RUNNING ]; then
         ocf_log info "DB2 instance $instance already stopped"
         return $OCF_SUCCESS
     fi
@@ -585,7 +610,12 @@ db2_instance_status() {
     local pscount
 
     pscount=$(runasdb2 $db2bin/db2nps $db2node | cut -c9- |  grep ' db2[^ ]' | wc -l)
-    test $pscount -ge 4
+    if [ $pscount -ge 4 ]; then
+        return $OCF_SUCCESS;
+    elif [ $pscount -ge 1 ]; then
+        return $OCF_GENERIC_ERR
+    fi
+    return $OCF_NOT_RUNNING
 }
 
 #
@@ -626,12 +656,14 @@ db2_hadr_status() {
 #
 db2_monitor() {
     local CMD output hadr db
+    local rc
 
-    if ! db2_instance_status
-    then
+    db2_instance_status
+    rc=$?
+    if [ $rc -ne $OCF_SUCCESS ]; then
         # instance is dead remove master score
-        crm_master -D -l reboot
-        exit $OCF_NOT_RUNNING
+        master_score -D -l reboot
+        exit $rc
     fi
 
     [ $db2node = 0 ] || return 0
@@ -667,22 +699,22 @@ db2_monitor() {
                     ocf_log err "DB2 message: $output"
 
                     # dead primary, remove master score
-                    crm_master -D -l reboot
+                    master_score -D -l reboot
                     return $OCF_ERR_GENERIC
                 esac
             fi
 
             ocf_log debug "DB2 database $instance($db2node)/$db appears to be working"
-            ocf_is_ms && crm_master -v 10000 -l reboot
+            ocf_is_ms && master_score -v 10000 -l reboot
             ;;
 
             Standby/*Peer)
-            crm_master -v 8000 -l reboot
+            master_score -v 8000 -l reboot
             ;;
 
             Standby/*)
             ocf_log warn "DB2 database $instance($db2node)/$db in status $hadr can never be promoted"
-            crm_master -D -l reboot
+            master_score -D -l reboot
             ;;
 
             *)
diff --git a/heartbeat/dhcpd b/heartbeat/dhcpd
index 835a788..861386d 100755
--- a/heartbeat/dhcpd
+++ b/heartbeat/dhcpd
@@ -64,6 +64,7 @@ usage() {
 
         The 'start' operation starts the dhcpd server.
         The 'stop' operation stops the dhcpd server.
+        The 'restart' operation restarts the dhcpd server.
         The 'monitor' operation reports whether the dhcpd service is running.
         The 'validate-all' operation reports whether the parameters are valid.
 EOF
@@ -171,6 +172,7 @@ Manage an ISC DHCP server service in a chroot environment.
   <actions>
     <action name="start"        timeout="20" />
     <action name="stop"         timeout="20" />
+    <action name="restart"         timeout="20" />
     <action name="monitor"      timeout="20" interval="10" depth="0" />
     <action name="meta-data"    timeout="5" />
     <action name="validate-all"   timeout="20" />
@@ -189,17 +191,17 @@ dhcpd_validate_all() {
 	# chroot mode is enabled.
 	if ocf_is_true $OCF_RESKEY_chrooted ; then
 	    if ! test -e "$OCF_RESKEY_chrooted_path"; then
-		ocf_log err "Path $OCF_RESKEY_chrooted_path does not exist."
+		ocf_exit_reason "Path $OCF_RESKEY_chrooted_path does not exist."
 		return $OCF_ERR_INSTALLED
 	    fi
 
 	    if test -n "$OCF_RESKEY_chrooted_path/$OCF_RESKEY_config" -a ! -r "$OCF_RESKEY_chrooted_path/$OCF_RESKEY_config"; then
-		ocf_log err "Configuration file $OCF_RESKEY_chrooted_path/$OCF_RESKEY_config doesn't exist"
+		ocf_exit_reason "Configuration file $OCF_RESKEY_chrooted_path/$OCF_RESKEY_config doesn't exist"
 		return $OCF_ERR_INSTALLED
 	    fi
 	else
 	    if test -n "$OCF_RESKEY_config" -a ! -r "$OCF_RESKEY_config"; then
-		ocf_log err "Configuration file $OCF_RESKEY_config doesn't exist"
+		ocf_exit_reason "Configuration file $OCF_RESKEY_config doesn't exist"
 		return $OCF_ERR_INSTALLED
 	    fi
 	fi
@@ -207,7 +209,7 @@ dhcpd_validate_all() {
     fi 
 
     if ! getent passwd $OCF_RESKEY_user >/dev/null 2>&1; then
-        ocf_log err "User $OCF_RESKEY_user doesn't exist"
+        ocf_exit_reason "User $OCF_RESKEY_user doesn't exist"
         return $OCF_ERR_INSTALLED
     fi
 
@@ -264,7 +266,7 @@ dhcpd_initialize_chroot() {
     ## If there is no conf file, we can't initialize the chrooted 
     ## environment, return with "program not configured"
     if ! [ -f $OCF_RESKEY_config ] ; then
-	ocf_log err "dhcpd has not been configured."
+	ocf_exit_reason "dhcpd has not been configured."
 	return $OCF_ERR_CONFIGURED
     fi
 
@@ -283,7 +285,7 @@ dhcpd_initialize_chroot() {
 	if [ -e $i ] ; then
 	    DEFAULT_FILE_LIST="$DEFAULT_FILE_LIST $i"
 	else
-	    ocf_log err "include file $i does not exist"
+	    ocf_exit_reason "include file $i does not exist"
 	    return $OCF_ERR_INSTALLED
 	fi
     done
@@ -299,19 +301,19 @@ dhcpd_initialize_chroot() {
 
 	# Next, we copy the configuration file into place.
 	cp -aL "$i" "$OCF_RESKEY_chrooted_path/${i%/*}/" > /dev/null 2>&1 ||
-	    { ocf_log err "could not copy $i to chroot jail"; return $OCF_ERR_GENERIC; }
+	    { ocf_exit_reason "could not copy $i to chroot jail"; return $OCF_ERR_GENERIC; }
     done
 
     libdir=$(basename $(echo /var/lib/dhcp/lib*))
     if test -x /usr/bin/ldd ; then
 	get_ldd_deps()
 	{
-	    ldd_wl="\/$libdir\/lib"
-	    ldd_bl="\/$libdir\/libc\."
+	    ldd_wl="/$libdir/lib"
+	    ldd_bl="/$libdir/libc\."
 	    /usr/bin/ldd "$1" | while read a b c d ; do
 		[ -n "$c" ] || continue
-		[[ $c =~ $ldd_wl ]] || continue
-		[[ $c =~ $ldd_bl ]] && continue
+		echo "$c" | grep -q "$ldd_wl" || continue
+		echo "$c" | grep -q "$ldd_bl" && continue
 		echo $c
 	    done
 	}
@@ -328,7 +330,7 @@ dhcpd_initialize_chroot() {
     for i in $cplibs ; do
 	if [ -s "$i" ]; then
 	    cp -pL "$i" "/var/lib/dhcp/$libdir/" ||
-		{ ocf_log err "could not copy $i to chroot jail"; return $OCF_ERR_GENERIC; }
+		{ ocf_exit_reason "could not copy $i to chroot jail"; return $OCF_ERR_GENERIC; }
 	fi
     done
 
@@ -339,7 +341,7 @@ dhcpd_initialize_chroot() {
 dhcpd_initialize() {
     ## If there is no conf file, we can't start a dhcp service.
     if ! [ -f $OCF_RESKEY_config ] ; then
-	ocf_log err "dhcpd has not been configured."
+	ocf_exit_reason "dhcpd has not been configured."
 	return $OCF_ERR_CONFIGURED
     fi
 
@@ -392,10 +394,10 @@ dhcpd_start() {
     # Only initialize the chrooted path(s) if chroot mode is enabled.
     if ocf_is_true $OCF_RESKEY_chrooted ; then
 	dhcpd_initialize_chroot ||
-	    { ocf_log err "Could not fully initialize the chroot environment." ; return $OCF_ERR_INSTALLED; }
+	    { ocf_exit_reason "Could not fully initialize the chroot environment." ; return $OCF_ERR_INSTALLED; }
     else
 	dhcpd_initialize ||
-	    { ocf_log err "Could not fully initialize the runtime environment." ; return $OCF_ERR_INSTALLED; }
+	    { ocf_exit_reason "Could not fully initialize the runtime environment." ; return $OCF_ERR_INSTALLED; }
     fi
 
     dhcpd_validate_all || exit
@@ -501,7 +503,7 @@ dhcpd_stop () {
 
     #If still up
     if dhcpd_monitor 2>&1; then
-        ocf_log err "dhcpd is still up! Trying kill -s KILL"
+        ocf_log notice "dhcpd is still up! Trying kill -s KILL"
 
 	kill -s SIGKILL `cat $PIDF`
     fi
@@ -538,6 +540,9 @@ esac
 case $__OCF_ACTION in
 start)          dhcpd_start;;
 stop)           dhcpd_stop;;
+restart)        dhcpd_stop
+                dhcpd_start
+                ;;
 monitor)        dhcpd_monitor;;
 *)              dhcpd_usage
                 exit $OCF_ERR_UNIMPLEMENTED
diff --git a/heartbeat/dnsupdate b/heartbeat/dnsupdate
new file mode 100755
index 0000000..50ed8ec
--- /dev/null
+++ b/heartbeat/dnsupdate
@@ -0,0 +1,272 @@
+#!/bin/bash
+#
+#
+# Support:      linux-ha at lists.linux-ha.org
+# License:      GNU General Public License v2
+#
+# Copyright (c) 2014 SUSE Linux Products GmbH, Lars Marowsky-Brée
+#                    All Rights Reserved.
+#
+#######################################################################
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+#######################################################################
+
+# TODO:
+# - Should setting CNAMEs be supported?
+# - Should multiple A records be supported?
+
+usage() {
+  cat <<-!
+	usage: $0 {start|stop|status|monitor|meta-data|validate-all}
+	!
+}
+
+meta_data() {
+	cat <<END
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="dnsupdate">
+<version>1.0</version>
+
+<longdesc lang="en">
+This resource agent manages IP take-over via dynamic DNS updates.
+</longdesc>
+<shortdesc lang="en">IP take-over via dynamic DNS update</shortdesc>
+
+<parameters>
+
+<parameter name="hostname" unique="1" required="1">
+<longdesc lang="en">
+The hostname whose IP address will need to be updated.
+</longdesc>
+<shortdesc lang="en">Hostname to update</shortdesc>
+<content type="string" default="" />
+</parameter>
+
+<parameter name="ip" unique="0" required="1">
+<longdesc lang="en">
+IP address to set.
+</longdesc>
+<shortdesc lang="en">IP address to set</shortdesc>
+<content type="string" default="" />
+</parameter>
+
+<parameter name="ttl" unique="0" required="0">
+<longdesc lang="en">
+Time to live, in seconds, for the DNS record. This
+affects how soon DNS updates propagate. It should be
+a reasonable compromise between update speed and DNS
+server load.
+
+If using booth, the ticket timeout is a good start.
+</longdesc>
+<shortdesc lang="en">TTL for the DNS record</shortdesc>
+<content type="integer" default="300" />
+</parameter>
+
+<parameter name="keyfile" unique="0" required="0">
+<longdesc lang="en">
+The file containing the shared secret needed to update
+the DNS record. Please see the nsupdate man page for
+the exact syntax.
+</longdesc>
+<shortdesc lang="en">nsupdate key file</shortdesc>
+<content type="string" default="" />
+</parameter>
+
+<parameter name="server" unique="0" required="0">
+<longdesc lang="en">
+Which DNS server to send these updates for. When no
+server is provided, this defaults to the master server
+for the correct zone.
+</longdesc>
+<shortdesc lang="en">DNS server to contact</shortdesc>
+<content type="string" default="" />
+</parameter>
+
+<parameter name="serverport" unique="0" required="0">
+<longdesc lang="en">
+Port number on the DNS server.
+
+Note: due to a limitation in the nsupdate command, this option will only
+take effect if you also specify the DNS server!
+</longdesc>
+<shortdesc lang="en">Port number on the DNS server</shortdesc>
+<content type="integer" default="53" />
+</parameter>
+
+<parameter name="nsupdate_opts" unique="0" required="0">
+<longdesc lang="en">
+Additional options to be passed to nsupdate.
+</longdesc>
+<shortdesc lang="en">Additional nsupdate options</shortdesc>
+<content type="string" default="" />
+</parameter>
+
+<parameter name="unregister_on_stop" unique="0" required="0">
+<longdesc lang="en">
+Whether or not to actively remove records on stop. This is not needed
+for normal operation, since the site taking over the IP address will
+delete all previous records.
+</longdesc>
+<shortdesc lang="en">Remove A record on stop</shortdesc>
+<content type="boolean" default="false" />
+</parameter>
+
+</parameters>
+
+<actions>
+<action name="start" timeout="30" />
+<action name="stop" timeout="30" />
+<action name="status" depth="0" timeout="30" interval="10" />
+<action name="monitor" depth="0" timeout="30" interval="10" />
+<action name="meta-data" timeout="5" />
+<action name="validate-all" timeout="5" />
+</actions>
+</resource-agent>
+END
+}
+
+dnsupdate_status() {
+	# The resource is considered active if the current IP
+	# address is returned as the only response.
+	local record=$(dig ${dig_opts} ${hostname}. A +short 2>/dev/null)
+	if [ "$record" = "$ip" ]; then
+		return $OCF_SUCCESS
+	fi
+	return $OCF_NOT_RUNNING
+}
+
+dnsupdate_monitor() {
+	if ocf_is_probe ; then
+		# 
+		return $OCF_NOT_RUNNING
+	fi
+	dnsupdate_status
+}
+
+dnsupdate_start() {
+	if dnsupdate_status ; then
+		ocf_log info "$hostname already resolves to $ip"
+		return $OCF_SUCCESS
+	fi
+
+	ocf_log info "Updating DNS records for $hostname"
+
+	(
+	if [ -n "$dns_server" ]; then
+		echo "server ${dns_server} ${dns_serverport}"
+	fi
+	echo "update delete $hostname A"
+	echo "update add $hostname ${OCF_RESKEY_ttl} A $ip"
+	echo "send"
+	) | nsupdate ${nsupdate_opts}
+
+	dnsupdate_monitor
+
+	return $?
+}
+
+dnsupdate_stop() {
+	if ocf_is_true "${OCF_RESKEY_unregister_on_stop}" && dnsupdate_status ; then
+		ocf_log info "Unregistering $hostname with $ip from DNS server"
+		(
+		if [ -n "$dns_server" ]; then
+			echo "server ${dns_server} ${dns_serverport}"
+		fi
+		echo "update delete $hostname A $ip"
+		echo "send"
+		) | nsupdate ${nsupdate_opts}
+
+		dnsupdate_monitor
+		if [ $? -ne $OCF_NOT_RUNNING ]; then
+			ocf_log warn "Unregistering failed!"
+			# There's no point in invoking a stop failure
+			# here. If another site takes over the record,
+			# it'll delete all previous entries anyway.
+		fi
+	fi
+	return $OCF_SUCCESS
+}
+
+dnsupdate_validate() {
+	hostname=${OCF_RESKEY_hostname}
+	ip=${OCF_RESKEY_ip}
+	dig_opts=""
+	dns_server=${OCF_RESKEY_server}
+	: ${OCF_RESKEY_serverport:="53"}
+	dns_serverport=${OCF_RESKEY_serverport}
+	: ${OCF_RESKEY_ttl:="300"}
+	nsupdate_opts=${OCF_RESKEY_opts}
+
+	if [ -z "$hostname" ]; then
+		ocf_log err "No hostname specified."
+		exit $OCF_ERR_CONFIGURED
+	fi
+	if [ -z "$ip" ]; then
+		ocf_log err "No IP specified."
+		exit $OCF_ERR_CONFIGURED
+	fi
+	if ! ocf_is_decimal $OCF_RESKEY_ttl ; then
+		ocf_log err "ttl $OCF_RESKEY_ttl is not valid"
+		exit $OCF_ERR_CONFIGURED
+	fi
+
+	if ! ocf_is_decimal $dns_serverport ; then
+		ocf_log err "serverport $dns_serverport is not valid"
+		exit $OCF_ERR_CONFIGURED
+	fi
+	dig_opts+=" -p ${dns_serverport}"
+
+	if [ -n "$dns_server" ]; then
+		dig_opts+=" @${dns_server}"
+	fi
+
+	if [ -n "$OCF_RESKEY_keyfile" ]; then
+		if [ ! -f ${OCF_RESKEY_keyfile} ]; then
+			ocf_log err "keyfile $OCF_RESKEY_keyfile does not exist"
+			exit $OCF_ERR_CONFIGURED
+		fi
+		nsupdate_opts+=" -k $OCF_RESKEY_keyfile"
+	fi
+}
+
+if [ $# -ne 1 ]; then
+  usage
+  exit $OCF_ERR_ARGS
+fi
+
+case $1 in
+meta-data)	meta_data
+		exit $OCF_SUCCESS
+		;;
+usage)		usage
+		exit $OCF_SUCCESS
+		;;
+esac
+
+check_binary dig
+check_binary nsupdate
+
+dnsupdate_validate
+
+case $1 in
+start)		dnsupdate_start
+		;;
+stop)		dnsupdate_stop
+		;;
+monitor)	dnsupdate_monitor
+		;;
+status)		dnsupdate_status
+		;;
+validate-all)	# We've already run this
+		exit $OCF_SUCCESS
+		;;
+*)		usage
+		exit $OCF_ERR_UNIMPLEMENTED
+		;;
+esac
+exit $?
+
+
diff --git a/heartbeat/docker b/heartbeat/docker
new file mode 100755
index 0000000..a0dcee4
--- /dev/null
+++ b/heartbeat/docker
@@ -0,0 +1,425 @@
+#!/bin/sh
+#
+# The docker HA resource agent creates and launches a docker container
+# based off a supplied docker image. Containers managed by this agent
+# are both created and removed upon the agent's start and stop actions.
+#
+# Copyright (c) 2014 David Vossel <dvossel at redhat.com>
+#                    All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of version 2 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# Further, this software is distributed without any warranty that it is
+# free of the rightful claim of any third person regarding infringement
+# or the like.  Any license provided herein, whether implied or
+# otherwise, applies only to this software file.  Patent licenses, if
+# any, provided herein do not apply to combinations of this program with
+# other software, or any other product whatsoever.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
+#
+
+#######################################################################
+# Initialization:
+
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+
+#######################################################################
+
+meta_data()
+{
+	cat <<END
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="docker" version="0.9">
+<version>1.0</version>
+
+<longdesc lang="en">
+The docker HA resource agent creates and launches a docker container
+based off a supplied docker image. Containers managed by this agent
+are both created and removed upon the agent's start and stop actions.
+</longdesc>
+<shortdesc lang="en">Docker container resource agent.</shortdesc>
+
+<parameters>
+<parameter name="image" required="1" unique="0">
+<longdesc lang="en">
+The docker image to base this container off of.
+</longdesc>
+<shortdesc lang="en">docker image</shortdesc>
+<content type="string"/>
+</parameter>
+
+<parameter name="name" required="0" unique="0">
+<longdesc lang="en">
+The name to give the created container. By default this will 
+be that resource's instance name.
+</longdesc>
+<shortdesc lang="en">docker container name</shortdesc>
+<content type="string"/>
+</parameter>
+
+<parameter name="allow_pull" unique="0">
+<longdesc lang="en">
+Allow the image to be pulled from the configured docker registry when
+the image does not exist locally. NOTE, this can drastically increase
+the time required to start the container if the image repository is 
+pulled over the network.
+</longdesc>
+<shortdesc lang="en">Allow pulling non-local images</shortdesc>
+<content type="boolean"/>
+</parameter>
+
+<parameter name="run_opts" required="0" unique="0">
+<longdesc lang="en">
+Add options to be appended to the 'docker run' command which is used
+when creating the container during the start action. This option allows
+users to do things such as setting a custom entry point and injecting
+environment variables into the newly created container. Note the '-d' 
+option is supplied regardless of this value to force containers to run 
+in the background.
+
+NOTE: Do not explicitly specify the --name argument in the run_opts. This
+agent will set --name using either the resource's instance or the name
+provided in the 'name' argument of this agent.
+
+</longdesc>
+<shortdesc lang="en">run options</shortdesc>
+<content type="string"/>
+</parameter>
+
+<parameter name="run_cmd" required="0" unique="0">
+<longdesc lang="en">
+Specifiy a command to launch within the container once 
+it has initialized.
+</longdesc>
+<shortdesc lang="en">run command</shortdesc>
+<content type="string"/>
+</parameter>
+
+<parameter name="monitor_cmd" required="0" unique="0">
+<longdesc lang="en">
+Specifiy the full path of a command to launch within the container to check
+the health of the container. This command must return 0 to indicate that
+the container is healthy. A non-zero return code will indicate that the
+container has failed and should be recovered.
+
+The command is executed using nsenter. In the future 'docker exec' will
+be used once it is more widely supported.
+</longdesc>
+<shortdesc lang="en">monitor command</shortdesc>
+<content type="string"/>
+</parameter>
+
+<parameter name="force_kill" required="0" unique="0">
+<longdesc lang="en">
+Kill a container immediately rather than waiting for it to gracefully
+shutdown
+</longdesc>
+<shortdesc lang="en">force kill</shortdesc>
+<content type="boolean"/>
+</parameter>
+
+<parameter name="reuse" required="0" unique="0">
+<longdesc lang="en">
+Allow the container to be reused after stopping the container. By default
+containers are removed after stop. With the reuse option containers
+will persist after the container stops.
+</longdesc>
+<shortdesc lang="en">reuse container</shortdesc>
+<content type="boolean"/>
+</parameter>
+
+</parameters>
+
+<actions>
+<action name="start"        timeout="90" />
+<action name="stop"         timeout="90" />
+<action name="monitor"      timeout="30" interval="30" depth="0" />
+<action name="meta-data"    timeout="5" />
+<action name="validate-all"   timeout="30" />
+</actions>
+</resource-agent>
+END
+}
+
+#######################################################################
+REQUIRE_IMAGE_PULL=0
+
+docker_usage()
+{
+	cat <<END
+usage: $0 {start|stop|monitor|validate-all|meta-data}
+
+Expects to have a fully populated OCF RA-compliant environment set.
+END
+}
+
+
+monitor_cmd_exec()
+{
+	local rc=$OCF_SUCCESS
+	local out
+
+	if [ -z "$OCF_RESKEY_monitor_cmd" ]; then
+		return $rc
+	fi
+
+	out=$(echo "$OCF_RESKEY_monitor_cmd" | nsenter --target $(docker inspect --format {{.State.Pid}} ${CONTAINER}) --mount --uts --ipc --net --pid 2>&1)
+	rc=$?
+	if [ $rc -ne 0 ]; then
+		ocf_log info "monitor cmd exit code = $rc"
+		ocf_log info "stdout/stderr: $out"
+
+		if [ $rc -eq 127 ]; then
+			ocf_exit_reason "monitor_cmd, ${OCF_RESKEY_monitor_cmd} , not found within container."
+			# there is no recovering from this, exit immediately
+			exit $OCF_ERR_ARGS
+		fi
+		rc=$OCF_ERR_GENERIC
+	else 
+		ocf_log info "monitor cmd passed: exit code = $rc"
+	fi
+
+	return $rc
+}
+
+container_exists()
+{
+	docker inspect $CONTAINER > /dev/null 2>&1
+}
+
+remove_container()
+{
+	if ocf_is_true "$OCF_RESKEY_reuse"; then
+		# never remove the container if we have reuse enabled.
+		return 0
+	fi
+
+	container_exists
+	if [ $? -ne 0 ]; then
+		# don't attempt to remove a container that doesn't exist
+		return 0
+	fi
+	ocf_log notice "Cleaning up inactive container, ${CONTAINER}."
+	ocf_run docker rm $CONTAINER
+}
+
+docker_simple_status()
+{
+	local val
+
+	container_exists
+	if [ $? -ne 0 ]; then
+		return $OCF_NOT_RUNNING
+	fi
+
+	# retrieve the 'Running' attribute for the container
+	val=$(docker inspect --format {{.State.Running}} $CONTAINER 2>/dev/null)
+	if [ $? -ne 0 ]; then
+		#not running as a result of container not being found
+		return $OCF_NOT_RUNNING
+	fi
+
+	if ocf_is_true "$val"; then
+		# container exists and is running
+		return $OCF_SUCCESS
+	fi
+
+	return $OCF_NOT_RUNNING
+}
+
+docker_monitor()
+{
+	local rc=0
+
+	docker_simple_status
+	rc=$?
+
+	if [ $rc -ne 0 ]; then
+		return $rc
+	fi
+
+	monitor_cmd_exec
+}
+
+docker_start()
+{
+	local run_opts="-d --name=${CONTAINER}"
+	# check to see if the container has already started
+	docker_simple_status
+	if [ $? -eq $OCF_SUCCESS ]; then
+		return $OCF_SUCCESS
+	fi
+
+	if [ -n "$OCF_RESKEY_run_opts" ]; then
+		run_opts="$run_opts $OCF_RESKEY_run_opts"
+	fi
+
+	if [ $REQUIRE_IMAGE_PULL -eq 1 ]; then
+		ocf_log notice "Beginning pull of image, ${OCF_RESKEY_image}"
+		docker pull "${OCF_RESKEY_image}"
+		if [ $? -ne 0 ]; then
+			ocf_exit_reason "failed to pull image ${OCF_RESKEY_image}"
+			return $OCF_ERR_GENERIC
+		fi
+	fi
+
+	if ocf_is_true "$OCF_RESKEY_reuse" && container_exists; then
+		ocf_log info "starting existing container $CONTAINER."
+		ocf_run docker start $CONTAINER
+	else
+		# make sure any previous container matching our container name is cleaned up first.
+		# we already know at this point it wouldn't be running 
+		remove_container
+		ocf_log info "running container $CONTAINER for the first time"
+		ocf_run docker run $run_opts $OCF_RESKEY_image $OCF_RESKEY_run_cmd
+	fi
+
+	if [ $? -ne 0 ]; then
+		ocf_exit_reason "docker failed to launch container"
+		return $OCF_ERR_GENERIC
+	fi
+
+
+	# wait for monitor to pass before declaring that the container is started
+	while true; do
+		docker_simple_status
+		if [ $? -ne $OCF_SUCCESS ]; then
+			ocf_exit_reason "Newly created docker container exited after start"
+			return $OCF_ERR_GENERIC
+		fi
+
+		monitor_cmd_exec
+		if [ $? -eq $OCF_SUCCESS ]; then
+			ocf_log notice "Container $CONTAINER  started successfully"
+			return $OCF_SUCCESS
+		fi
+
+		ocf_exit_reason "waiting on monitor_cmd to pass after start"
+		sleep 1
+	done
+}
+
+docker_stop()
+{
+	local timeout=60
+	docker_simple_status
+	if [ $? -eq  $OCF_NOT_RUNNING ]; then
+		remove_container
+		return $OCF_SUCCESS
+	fi
+
+	if [ -n "$OCF_RESKEY_CRM_meta_timeout" ]; then
+		timeout=$((($OCF_RESKEY_CRM_meta_timeout/1000) -10 ))
+		if [ $timeout -lt 10 ]; then
+			timeout=10
+		fi
+	fi
+
+	if ocf_is_true "$OCF_RESKEY_force_kill"; then
+		ocf_run docker kill $CONTAINER
+	else 	
+		ocf_log debug "waiting $timeout second[s] before killing container"
+		ocf_run docker stop -t=$timeout $CONTAINER
+	fi
+
+	if [ $? -ne 0 ]; then
+		ocf_exit_reason "Failed to stop container, ${CONTAINER}, based on image, ${OCF_RESKEY_image}."
+		return $OCF_ERR_GENERIC
+	fi
+
+	remove_container
+	if [ $? -ne 0 ]; then
+		ocf_exit_reason "Failed to remove stopped container, ${CONTAINER}, based on image, ${OCF_RESKEY_image}."
+		return $OCF_ERR_GENERIC
+	fi
+
+	return $OCF_SUCCESS
+}
+
+image_exists()
+{
+	local res=1
+
+
+	echo "${OCF_RESKEY_image}" | grep -q ":"
+	if [ $? -eq 0 ]; then
+		docker images | awk '{print $1 ":" $2}' |  grep "^${OCF_RESKEY_image}\$" > /dev/null 2>&1
+	else
+		docker images | awk '{print $1}' | grep "^${OCF_RESKEY_image}\$" > /dev/null 2>&1
+	fi
+	if [ $? -eq 0 ]; then
+		return 0
+	fi
+	if ocf_is_true "$OCF_RESKEY_allow_pull"; then
+		REQUIRE_IMAGE_PULL=1
+		ocf_log notice "Image (${OCF_RESKEY_image}) does not exist locally but will be pulled during start"
+		return 0
+	fi 
+	# image not found.
+	return 1
+}
+
+docker_validate()
+{
+	check_binary docker
+	if [ -z "$OCF_RESKEY_image" ]; then
+		ocf_exit_reason "'image' option is required"
+		exit $OCF_ERR_CONFIGURED
+	fi 
+
+	if [ -n "$OCF_RESKEY_monitor_cmd" ]; then
+		ocf_log info "checking for nsenter, which is required when 'monitor_cmd' is specified"
+		check_binary nsenter
+	fi
+
+	image_exists
+	if [ $? -ne 0 ]; then
+		ocf_exit_reason "base image, ${OCF_RESKEY_image}, could not be found."
+		exit $OCF_ERR_CONFIGURED
+	fi
+
+	return $OCF_SUCCESS
+}
+
+: ${OCF_RESKEY_name=${OCF_RESOURCE_INSTANCE}}
+
+if [ -n "$OCF_RESKEY_container" ]; then
+	# we'll keep the container attribute around for a bit in order not to break
+	# any existing deployments. The 'name' attribute is prefered now though.
+	CONTAINER=$OCF_RESKEY_container
+	ocf_log warn "The 'container' attribute is depreciated"
+else
+	CONTAINER=$OCF_RESKEY_name
+fi
+
+case $__OCF_ACTION in
+meta-data) meta_data
+		exit $OCF_SUCCESS;;
+start)
+	docker_validate
+	docker_start;;
+stop)		docker_stop;;
+monitor)	docker_monitor;;
+validate-all)	docker_validate;;
+usage|help)	docker_usage
+		exit $OCF_SUCCESS
+		;;
+*)		docker_usage
+		exit $OCF_ERR_UNIMPLEMENTED
+		;;
+esac
+rc=$?
+ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION : $rc"
+exit $rc
+
diff --git a/heartbeat/ethmonitor b/heartbeat/ethmonitor
index b85d7fc..a447391 100755
--- a/heartbeat/ethmonitor
+++ b/heartbeat/ethmonitor
@@ -1,14 +1,14 @@
 #!/bin/sh
 #
-#       OCF Resource Agent compliant script.
-#       Monitor the vitality of a local network interface.
+#	   OCF Resource Agent compliant script.
+#	   Monitor the vitality of a local network interface.
 #
 # 	Based on the work by Robert Euhus and Lars Marowsky-Br�e.
 #
 #	Transfered from Ipaddr2 into ethmonitor by Alexander Krauth
 #
 # Copyright (c) 2011 Robert Euhus, Alexander Krauth, Lars Marowsky-Br�e
-#                    All Rights Reserved.
+#					All Rights Reserved.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of version 2 of the GNU General Public License as
@@ -29,12 +29,12 @@
 # along with this program; if not, write the Free Software Foundation,
 # Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
 #
-#     OCF parameters are as below
+#	 OCF parameters are as below
 #
 #	OCF_RESKEY_interface
 #	OCF_RESKEY_multiplicator
 #	OCF_RESKEY_name
-#       OCF_RESKEY_repeat_count
+#	   OCF_RESKEY_repeat_count
 #	OCF_RESKEY_repeat_interval
 #	OCF_RESKEY_pktcnt_timeout
 #	OCF_RESKEY_arping_count
@@ -70,10 +70,13 @@ The resource configuration requires a monitor operation, because the monitor doe
 In addition to the resource configuration, you need to configure some location constraints, based on a CIB attribute value.
 The name of the attribute value is configured in the 'name' option of this RA.
 
-Example constraint configuration:
+Example constraint configuration using crmsh
 location loc_connected_node my_resource_grp \
         rule $id="rule_loc_connected_node" -INF: ethmonitor eq 0
 
+Example constraint configuration using pcs. Only allow 'my_resource' to run on nodes where eth0 ethernet device is available.
+pcs constraint location my_resource rule score=-INFINITY ethmonitor-eth0 ne 1
+
 The ethmonitor works in 3 different modes to test the interface vitality.
 1. call ip to see if the link status is up (if link is down -> error)
 2. call ip and watch the RX counter (if packages come around in a certain time -> success)
@@ -157,14 +160,30 @@ Maximum number of IPs from ARP cache list to check for ARP REQUEST (arping) answ
 <content type="integer" default="5"/>
 </parameter>
 
+<parameter name="infiniband_device">
+<longdesc lang="en">
+For interfaces that are infiniband devices.
+</longdesc>
+<shortdesc lang="en">infiniband device</shortdesc>
+<content type="string" />
+</parameter>
+
+<parameter name="infiniband_port">
+<longdesc lang="en">
+For infiniband devices, this is the port to monitor.
+</longdesc>
+<shortdesc lang="en">infiniband port</shortdesc>
+<content type="integer" />
+</parameter>
+
 </parameters>
 <actions>
-<action name="start"   timeout="20s" />
-<action name="stop"    timeout="20s" />
-<action name="status" depth="0"  timeout="20s" interval="10s" />
-<action name="monitor" depth="0"  timeout="20s" interval="10s" />
-<action name="meta-data"  timeout="5s" />
-<action name="validate-all"  timeout="20s" />
+<action name="start" timeout="60s" />
+<action name="stop" timeout="20s" />
+<action name="status" depth="0" timeout="60s" interval="10s" />
+<action name="monitor" depth="0" timeout="60s" interval="10s" />
+<action name="meta-data" timeout="5s" />
+<action name="validate-all" timeout="20s" />
 </actions>
 </resource-agent>
 END
@@ -173,7 +192,7 @@ END
 }
 
 #
-#	Return true, if the interface exists
+# Return true, if the interface exists
 #
 is_interface() {
 	#
@@ -181,14 +200,25 @@ is_interface() {
 	#
 	local iface=`$IP2UTIL -o -f inet addr show | grep " $1 " \
 		| cut -d ' ' -f2 | sort -u | grep -v '^ipsec[0-9][0-9]*$'`
-        [ "$iface" != "" ]
+		[ "$iface" != "" ]
+}
+
+infiniband_status()
+{
+	local device="$OCF_RESKEY_infiniband_device"
+
+	if [ -n "$OCF_RESKEY_infiniband_port" ]; then
+		device="${OCF_RESKEY_infiniband_device}:${OCF_RESKEY_infiniband_port}"
+	fi
+	
+	ibstatus ${device} | grep -q ACTIVE 
 }
 
 if_init() {
 	local rc
 
 	if [ X"$OCF_RESKEY_interface" = "X" ]; then
-		ocf_log err "Interface name (the interface parameter) is mandatory"
+		ocf_exit_reason "Interface name (the interface parameter) is mandatory"
 		exit $OCF_ERR_CONFIGURED
 	fi
 
@@ -196,60 +226,67 @@ if_init() {
 
 	if is_interface $NIC
 	then
-	  case "$NIC" in
-	    *:*) ocf_log err "Do not specify a virtual interface : $OCF_RESKEY_interface"
-	         exit $OCF_ERR_CONFIGURED;;
-	    *)  ;;
-	  esac
+		case "$NIC" in
+			*:*) ocf_exit_reason "Do not specify a virtual interface : $OCF_RESKEY_interface"
+				 exit $OCF_ERR_CONFIGURED;;
+			*)   ;;
+		esac
 	else
-	  case $__OCF_ACTION in
-	    validate-all) ocf_log err "Interface $NIC does not exist"
-                            exit $OCF_ERR_CONFIGURED;;
-	    *)   	    ocf_log warn "Interface $NIC does not exist"
-                            ## It might be a bond interface which is temporarily not available, therefore we want to continue here
-	                    ;;
-	  esac
+		case $__OCF_ACTION in
+			validate-all)
+				ocf_exit_reason "Interface $NIC does not exist"
+				exit $OCF_ERR_CONFIGURED;;
+			*)	
+				## It might be a bond interface which is temporarily not available, therefore we want to continue here
+				ocf_log warn "Interface $NIC does not exist"
+				;;
+		esac
 	fi
 
 	: ${OCF_RESKEY_multiplier:="1"}
 	if ! ocf_is_decimal "$OCF_RESKEY_multiplier"; then
-		ocf_log err "Invalid OCF_RESKEY_multiplier [$OCF_RESKEY_multiplier]"
+		ocf_exit_reason "Invalid OCF_RESKEY_multiplier [$OCF_RESKEY_multiplier]"
 		exit $OCF_ERR_CONFIGURED
 	fi
 	
 	ATTRNAME=${OCF_RESKEY_name:-"ethmonitor-$NIC"}
 	
-        REP_COUNT=${OCF_RESKEY_repeat_count:-5}
+	REP_COUNT=${OCF_RESKEY_repeat_count:-5}
 	if ! ocf_is_decimal "$REP_COUNT" -o [ $REP_COUNT -lt 1 ]; then
-		ocf_log err "Invalid OCF_RESKEY_repeat_count [$REP_COUNT]"
+		ocf_exit_reason "Invalid OCF_RESKEY_repeat_count [$REP_COUNT]"
 		exit $OCF_ERR_CONFIGURED
-        fi
+	fi
 	REP_INTERVAL_S=${OCF_RESKEY_repeat_interval:-10}
 	if ! ocf_is_decimal "$REP_INTERVAL_S"; then
-		ocf_log err "Invalid OCF_RESKEY_repeat_interval [$REP_INTERVAL_S]"
+		ocf_exit_reason "Invalid OCF_RESKEY_repeat_interval [$REP_INTERVAL_S]"
 		exit $OCF_ERR_CONFIGURED
 	fi
 	: ${OCF_RESKEY_pktcnt_timeout:="5"}
 	if ! ocf_is_decimal "$OCF_RESKEY_pktcnt_timeout"; then
-		ocf_log err "Invalid OCF_RESKEY_pktcnt_timeout [$OCF_RESKEY_pktcnt_timeout]"
+		ocf_exit_reason "Invalid OCF_RESKEY_pktcnt_timeout [$OCF_RESKEY_pktcnt_timeout]"
 		exit $OCF_ERR_CONFIGURED
 	fi
 	: ${OCF_RESKEY_arping_count:="1"}
 	if ! ocf_is_decimal "$OCF_RESKEY_arping_count"; then
-		ocf_log err "Invalid OCF_RESKEY_arping_count [$OCF_RESKEY_arping_count]"
+		ocf_exit_reason "Invalid OCF_RESKEY_arping_count [$OCF_RESKEY_arping_count]"
 		exit $OCF_ERR_CONFIGURED
 	fi
 	: ${OCF_RESKEY_arping_timeout:="1"}
 	if ! ocf_is_decimal "$OCF_RESKEY_arping_timeout"; then
-		ocf_log err "Invalid OCF_RESKEY_arping_timeout [$OCF_RESKEY_arping_count]"
+		ocf_exit_reason "Invalid OCF_RESKEY_arping_timeout [$OCF_RESKEY_arping_count]"
 		exit $OCF_ERR_CONFIGURED
 	fi
 	: ${OCF_RESKEY_arping_cache_entries:="5"}
 	if ! ocf_is_decimal "$OCF_RESKEY_arping_cache_entries"; then
-		ocf_log err "Invalid OCF_RESKEY_arping_cache_entries [$OCF_RESKEY_arping_cache_entries]"
+		ocf_exit_reason "Invalid OCF_RESKEY_arping_cache_entries [$OCF_RESKEY_arping_cache_entries]"
 		exit $OCF_ERR_CONFIGURED
 	fi
-  return $OCF_SUCCESS
+
+	if [ -n "$OCF_RESKEY_infiniband_device" ]; then
+		#ibstatus is required if an infiniband_device is provided
+		check_binary ibstatus
+	fi
+	return $OCF_SUCCESS
 }
 
 # get the link status on $NIC
@@ -277,7 +314,7 @@ watch_pkt_counter () {
 	for n in `seq $(( $OCF_RESKEY_pktcnt_timeout * 10 ))`; do
 		sleep 0.1
 		RX_PACKETS_NEW="`get_rx_packets`"
-		ocf_log debug "RX_PACKETS_OLD: $RX_PACKETS_OLD    RX_PACKETS_NEW: $RX_PACKETS_NEW"
+		ocf_log debug "RX_PACKETS_OLD: $RX_PACKETS_OLD	RX_PACKETS_NEW: $RX_PACKETS_NEW"
 		if [ "$RX_PACKETS_OLD" -ne "$RX_PACKETS_NEW" ]; then
 			ocf_log debug "we received some packets."
 			return 0
@@ -308,7 +345,7 @@ do_arping () {
 }
 
 #
-# 	Check the interface depending on the level given as parameter: $OCF_RESKEY_check_level
+# Check the interface depending on the level given as parameter: $OCF_RESKEY_check_level
 #
 # 09: check for nonempty ARP cache
 # 10: watch for packet counter changes
@@ -322,21 +359,47 @@ do_arping () {
 # the tests for higher check levels are run.
 #
 if_check () {
+	local arp_list
 	# always check link status first
 	link_status="`get_link_status`"
 	ocf_log debug "link_status: $link_status (1=up, 0=down)"
-        [ $link_status -eq 0 ] && return $OCF_NOT_RUNNING
+
+	if [ $link_status -eq 0 ]; then
+		ocf_log notice "link_status: DOWN"
+		return $OCF_NOT_RUNNING
+	fi
+
+	# if this is an infiniband device, try ibstatus script
+	if [ -n "$OCF_RESKEY_infiniband_device" ]; then
+		if infiniband_status; then
+			return $OCF_SUCCESS
+		fi
+		ocf_log info "Infiniband device $OCF_RESKEY_infiniband_device is not available, check ibstatus for more information"
+		return $OCF_NOT_RUNNING	
+	fi
 
 	# watch for packet counter changes
-	ocf_log debug "watch for packet counter changes" 
-	watch_pkt_counter && return $OCF_SUCCESS
+	ocf_log debug "watch for packet counter changes"
+	watch_pkt_counter
+	if [ $? -eq 0 ]; then
+		return $OCF_SUCCESS
+	else 
+		ocf_log debug "No packets received during packet watch timeout"
+	fi
 
 	# check arping ARP cache entries
-	ocf_log debug "check arping ARP cache entries" 
-	for ip in `get_arp_list`; do
+	ocf_log debug "check arping ARP cache entries"
+	arp_list=`get_arp_list`
+	for ip in `echo $arp_list`; do
 		do_arping $ip && return $OCF_SUCCESS
 	done
 
+	# if we get here, the ethernet device is considered not running.
+	# provide some logging information
+	if [ -z "$arp_list" ]; then
+		ocf_log info "No ARP cache entries found to arping" 
+	fi
+
 	# watch for packet counter changes in promiscios mode
 #	ocf_log debug "watch for packet counter changes in promiscios mode" 
 	# be sure switch off promiscios mode in any case
@@ -362,67 +425,89 @@ END
 }
 
 set_cib_value() {
-    local score=`expr $1 \* $OCF_RESKEY_multiplier`
-    attrd_updater -n $ATTRNAME -v $score -q
-    local rc=$?
-    case $rc in
-        0) ocf_log debug "attrd_updater: Updated $ATTRNAME = $score" ;;
-        *) ocf_log warn "attrd_updater: Could not update $ATTRNAME = $score: rc=$rc";;
-    esac
-    return $rc
+	local score=`expr $1 \* $OCF_RESKEY_multiplier`
+	attrd_updater -n $ATTRNAME -v $score -q
+	local rc=$?
+	case $rc in
+		0) ocf_log debug "attrd_updater: Updated $ATTRNAME = $score" ;;
+		*) ocf_log warn "attrd_updater: Could not update $ATTRNAME = $score: rc=$rc";;
+	esac
+	return $rc
 }
 
 if_monitor() {
-    ha_pseudo_resource $OCF_RESOURCE_INSTANCE monitor
-    local pseudo_status=$?
-    if [ $pseudo_status -ne $OCF_SUCCESS ]; then
-      exit $pseudo_status
-    fi
-    
-    local mon_rc=$OCF_NOT_RUNNING
-    local attr_rc=$OCF_NOT_RUNNING
-    local runs=0
-    local start_time
-    local end_time
-    local sleep_time
-    while [ $mon_rc -ne $OCF_SUCCESS -a $REP_COUNT -gt 0 ]
-    do
-      start_time=`date +%s%N`
-      if_check
-      mon_rc=$?
-      REP_COUNT=$(( $REP_COUNT - 1 ))
-      if [ $mon_rc -ne $OCF_SUCCESS -a $REP_COUNT -gt 0 ]; then
-        ocf_log warn "Monitoring of $OCF_RESOURCE_INSTANCE failed, $REP_COUNT retries left."
-	end_time=`date +%s%N`
-	sleep_time=`echo "scale=9; ( $start_time + ( $REP_INTERVAL_S * 1000000000 ) - $end_time ) / 1000000000" | bc -q 2> /dev/null`
-        sleep $sleep_time 2> /dev/null
-        runs=$(($runs + 1))
-      fi
-
-      if [ $mon_rc -eq $OCF_SUCCESS -a $runs -ne 0 ]; then
-        ocf_log info "Monitoring of $OCF_RESOURCE_INSTANCE recovered from error"
-      fi
-    done
-    
-    ocf_log debug "Monitoring return code: $mon_rc"
-    if [ $mon_rc -eq $OCF_SUCCESS ]; then
-      set_cib_value 1
-      attr_rc=$?
-    else
-      ocf_log err "Monitoring of $OCF_RESOURCE_INSTANCE failed."
-      set_cib_value 0
-      attr_rc=$?
-    fi
-
-    ## The resource should not fail, if the interface is down. It should fail, if the update of the CIB variable has errors.
-    ## To react on the interface failure you must use constraints based on the CIB variable value, not on the resource itself.
-    exit $attr_rc
+	ha_pseudo_resource $OCF_RESOURCE_INSTANCE monitor
+	local pseudo_status=$?
+	if [ $pseudo_status -ne $OCF_SUCCESS ]; then
+		exit $pseudo_status
+	fi
+	
+	local mon_rc=$OCF_NOT_RUNNING
+	local attr_rc=$OCF_NOT_RUNNING
+	local runs=0
+	local start_time
+	local end_time
+	local sleep_time
+	while [ $mon_rc -ne $OCF_SUCCESS -a $REP_COUNT -gt 0 ]
+	do
+		start_time=`date +%s%N`
+		if_check
+		mon_rc=$?
+		REP_COUNT=$(( $REP_COUNT - 1 ))
+		if [ $mon_rc -ne $OCF_SUCCESS -a $REP_COUNT -gt 0 ]; then
+			ocf_log warn "Monitoring of $OCF_RESOURCE_INSTANCE failed, $REP_COUNT retries left."
+			end_time=`date +%s%N`
+			sleep_time=`echo "scale=9; ( $start_time + ( $REP_INTERVAL_S * 1000000000 ) - $end_time ) / 1000000000" | bc -q 2> /dev/null`
+			sleep $sleep_time 2> /dev/null
+			runs=$(($runs + 1))
+		fi
+
+		if [ $mon_rc -eq $OCF_SUCCESS -a $runs -ne 0 ]; then
+			ocf_log info "Monitoring of $OCF_RESOURCE_INSTANCE recovered from error"
+		fi
+	done
+	
+	ocf_log debug "Monitoring return code: $mon_rc"
+	if [ $mon_rc -eq $OCF_SUCCESS ]; then
+		set_cib_value 1
+		attr_rc=$?
+	else
+		ocf_log err "Monitoring of $OCF_RESOURCE_INSTANCE failed."
+		set_cib_value 0
+		attr_rc=$?
+	fi
+
+	## The resource should not fail, if the interface is down. It should fail, if the update of the CIB variable has errors.
+	## To react on the interface failure you must use constraints based on the CIB variable value, not on the resource itself.
+	exit $attr_rc
+}
+
+if_stop()
+{
+	attrd_updater -D -n $ATTRNAME
+	ha_pseudo_resource $OCF_RESOURCE_INSTANCE stop
 }
 
+if_start()
+{
+	local rc
+	ha_pseudo_resource $OCF_RESOURCE_INSTANCE start
+	rc=$?
+	if [ $rc -ne $OCF_SUCCESS ]; then
+		ocf_exit_reason "Failure to create ethmonitor state file"
+		return $rc
+	fi
+
+	# perform the first monitor during the start operation
+	if_monitor
+	return $?
+}
+
+
 if_validate() {
-    check_binary $IP2UTIL
-    check_binary arping
-    if_init
+	check_binary $IP2UTIL
+	check_binary arping
+	if_init
 }
 
 case $__OCF_ACTION in
@@ -436,18 +521,17 @@ esac
 if_validate
 
 case $__OCF_ACTION in
-start)		ha_pseudo_resource $OCF_RESOURCE_INSTANCE start
+start)		if_start
 		exit $?
 		;;
-stop)		attrd_updater -D -n $ATTRNAME
-                ha_pseudo_resource $OCF_RESOURCE_INSTANCE stop
+stop)		if_stop
 		exit $?
 		;;
 monitor|status)	if_monitor
 		exit $?
 		;;
 validate-all)	exit $?
-                ;;
+		;;
 *)		if_usage
 		exit $OCF_ERR_UNIMPLEMENTED
 		;;
diff --git a/heartbeat/exportfs b/heartbeat/exportfs
index 0c6d092..c4ec653 100755
--- a/heartbeat/exportfs
+++ b/heartbeat/exportfs
@@ -14,7 +14,7 @@
 . ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
 
 # Defaults
-OCF_RESKEY_unlock_on_stop_default=0
+OCF_RESKEY_unlock_on_stop_default=1
 OCF_RESKEY_wait_for_leasetime_on_stop_default=0
 OCF_RESKEY_rmtab_backup_default=".rmtab"
 
@@ -46,7 +46,7 @@ Manages NFS exports
 <parameter name="clientspec" unique="0" required="1">
 <longdesc lang="en">
 The client specification allowing remote machines to mount the directory
-over NFS.
+(or directories) over NFS.
 </longdesc>
 <shortdesc lang="en">
 Client ACL.
@@ -56,7 +56,8 @@ Client ACL.
 
 <parameter name="options" unique="0" required="0">
 <longdesc lang="en">
-The options to pass to exportfs for the exported directory.
+The options to pass to exportfs for the exported directory
+or directories.
 </longdesc>
 <shortdesc lang="en">
 Export options.
@@ -66,10 +67,11 @@ Export options.
 
 <parameter name="directory" unique="0" required="1">
 <longdesc lang="en">
-The directory which you wish to export using NFS.
+The directory or directories to be exported using NFS. Multiple
+directories are separated by white space.
 </longdesc>
 <shortdesc lang="en">
-The directory to export.
+The directory or directories to export.
 </shortdesc>
 <content type="string" />
 </parameter>
@@ -79,13 +81,17 @@ The directory to export.
 The fsid option to pass to exportfs. This can be a unique positive
 integer, a UUID, or the special string "root" which is functionally
 identical to numeric fsid of 0.
+If multiple directories are being exported, then they are
+assigned ids sequentially starting with this fsid (fsid, fsid+1,
+fsid+2, ...). Obviously, in that case the fsid must be an
+integer.
 0 (root) identifies the export as the root of an NFSv4
 pseudofilesystem -- avoid this setting unless you understand its
 special status.
 This value will override any fsid provided via the options parameter.
 </longdesc>
 <shortdesc lang="en">
-Unique fsid within cluster.
+Unique fsid within cluster or starting fsid for multiple exports.
 </shortdesc>
 <content type="string" />
 </parameter>
@@ -95,6 +101,12 @@ Unique fsid within cluster.
 Relinquish NFS locks associated with this filesystem when the resource
 stops. Enabling this parameter is highly recommended unless the path exported
 by this ${__SCRIPT_NAME} resource is also exported by a different resource.
+
+Note: Unlocking is only possible on Linux systems where
+/proc/fs/nfsd/unlock_filesystem exists and is writable. If your system does
+not fulfill this requirement (on account of having an nonrecent kernel,
+for example), you may set this parameter to 0 to silence the associated
+warning.
 </longdesc>
 <shortdesc lang="en">
 Unlock filesystem on stop?
@@ -141,7 +153,7 @@ Location of the rmtab backup, relative to directory.
 
 <actions>
 <action name="start"   timeout="40" />
-<action name="stop"    timeout="10" />
+<action name="stop"    timeout="120" />
 <action name="monitor" depth="0"  timeout="20" interval="10" />
 <action name="meta-data"  timeout="5" />
 <action name="validate-all"  timeout="30" />
@@ -165,18 +177,48 @@ exportfs_methods() {
 	!
 }
 
+reset_fsid() {
+	CURRENT_FSID=$OCF_RESKEY_fsid
+}
+bump_fsid() {
+	CURRENT_FSID=$((CURRENT_FSID+1))
+}
+get_fsid() {
+	echo $CURRENT_FSID
+}
+
+# run a function on all directories
+forall() {
+	local func=$1
+	shift 1
+	local fast_exit=""
+	local dir rc=0
+	if [ "$2" = fast_exit ]; then
+		fast_exit=1
+		shift 1
+	fi
+	reset_fsid
+	for dir in $OCF_RESKEY_directory; do
+		$func $dir "$@"
+		rc=$(($rc | $?))
+		bump_fsid
+		[ "$fast_exit" ] && continue
+		[ $rc -ne 0 ] && return $rc
+	done
+	return $rc
+}
+
 backup_rmtab() {
+	local dir=$1
 	local rmtab_backup
-	if [ ${OCF_RESKEY_rmtab_backup} != "none" ]; then
-		rmtab_backup="${OCF_RESKEY_directory}/${OCF_RESKEY_rmtab_backup}"
-		grep ":${OCF_RESKEY_directory}:" /var/lib/nfs/rmtab > ${rmtab_backup}
-	fi
+	rmtab_backup="$dir/${OCF_RESKEY_rmtab_backup}"
+	grep ":$dir:" /var/lib/nfs/rmtab > ${rmtab_backup}
 }
 
 restore_rmtab() {
+	local dir=$1
 	local rmtab_backup
-	if [ ${OCF_RESKEY_rmtab_backup} != "none" ]; then
-	rmtab_backup="${OCF_RESKEY_directory}/${OCF_RESKEY_rmtab_backup}"
+	rmtab_backup="$dir/${OCF_RESKEY_rmtab_backup}"
 	if [ -r ${rmtab_backup} ]; then
 		local tmpf=`mktemp`
 		sort -u ${rmtab_backup} /var/lib/nfs/rmtab > $tmpf &&
@@ -186,7 +228,6 @@ restore_rmtab() {
 	else
 		ocf_log warn "rmtab backup ${rmtab_backup} not found or not readable."
 	fi
-	fi
 }
 
 exportfs_usage() {
@@ -195,86 +236,98 @@ exportfs_usage() {
 END
 }
 
+format_exports() {
+	# exportfs output wraps lines for long export directory names.
+	# We unwrap here with sed.
+	# We then do a literal match on the full line (grep -x -F)
+	exportfs |
+		sed -e '$! N; s/\n[[:space:]]\+/ /; t; s/[[:space:]]\+\([^[:space:]]\+\)\(\n\|$\)/ \1\2/g; P;D;'
+}
 is_exported() {
 	local dir=$1
 	local spec=$2
-	exportfs |
-		sed -e '$! N; s/\n[[:space:]]\+/ /; t; s/[[:space:]]\+\([^[:space:]]\+\)\(\n\|$\)/ \1\2/g; P;D;' |
-		grep -q -x -F "$dir $spec"
-}
-
-exportfs_monitor ()
-{
 	local rc
-	# exportfs output wraps lines for long export directory names.
-	# We unwrap here with sed.
-	# We then do a literal match on the full line (grep -x -F)
-	is_exported "${OCF_RESKEY_directory}" "${OCF_RESKEY_clientspec}"
+	format_exports | grep -q -x -F "$dir $spec"
 	rc=$?
-
-	# on some platforms, exportfs may print "<world>" instead of
-	# "*"
-	if [ $rc -eq 1 -a "${OCF_RESKEY_clientspec}" = "*" ]; then
-		is_exported "${OCF_RESKEY_directory}" "<world>"
+	if [ $rc -ne 0 -a "$spec" = "*" ]; then
+		# on some platforms, exportfs may print
+		# "<world>" instead of "*"
+		format_exports | grep -q -x -F "$dir <world>"
 		rc=$?
 	fi
-
-#Adapt grep status code to OCF return code
-	case $rc in
-	0)
-		ocf_log info "Directory ${OCF_RESKEY_directory} is exported to ${OCF_RESKEY_clientspec} (started)."
-		# Backup the rmtab to ensure smooth NFS-over-TCP failover
-		backup_rmtab
-		return $OCF_SUCCESS
-		;;
-	1)
-		ocf_log info "Directory ${OCF_RESKEY_directory} is not exported to ${OCF_RESKEY_clientspec} (stopped)."
-		return $OCF_NOT_RUNNING;;
-	*)
-		ocf_log err "Unable to determine export status for ${OCF_RESKEY_directory}."
-		return $OCF_ERR_GENERIC;;
-	esac
+	# log something only for monitors
+	if [ $rc -ne 0 -a "$__OCF_ACTION" = "monitor" ]; then
+		local sev="info"
+		ocf_is_probe || sev="err"
+		ocf_log $sev "$dir not exported to $spec (stopped)."
+	fi
+	return $rc
 }
 
-exportfs_start ()
+exportfs_monitor ()
 {
-	if exportfs_monitor; then
-		ocf_log debug "${OCF_RESKEY_directory} already exported"
+	if forall is_exported "${OCF_RESKEY_clientspec}"; then
+		if [ ${OCF_RESKEY_rmtab_backup} != "none" ]; then
+			forall backup_rmtab
+		fi
 		return $OCF_SUCCESS
+	else
+		return $OCF_NOT_RUNNING
 	fi
-		
-	ocf_log info "Exporting file system ..."
+}
 
-	if [ ${OCF_RESKEY_options} ]; then
-		OPTIONS="${OCF_RESKEY_options}"
-		OPTPREFIX=','
+export_one() {
+	local dir=$1
+	local opts sep
+	sep=""
+	if [ -n "$OCF_RESKEY_options" ]; then
+		opts="$OCF_RESKEY_options"
+		sep=","
 	fi
-	if [ `echo ${OPTIONS} | grep fsid` ]; then
-		#replace fsid provided in options list with one provided in fsid param.
-		OPTIONS=`echo ${OPTIONS} | sed "s/fsid=[0-9]\+/fsid=${OCF_RESKEY_fsid}/g"`
+	if echo "$opts" | grep fsid >/dev/null; then
+		#replace fsid in options list
+		opts=`echo "$opts" | sed "s/fsid=[0-9]\+/fsid=$(get_fsid)/g"`
 	else
 		#tack the fsid option onto our options list.
-		OPTIONS="${OPTIONS}${OPTPREFIX}fsid=${OCF_RESKEY_fsid}"
+		opts="${opts}${sep}fsid=$(get_fsid)"
 	fi
-	OPTIONS="-o ${OPTIONS}"
+	opts="-o $opts"
 
-	ocf_run exportfs -v ${OPTIONS} ${OCF_RESKEY_clientspec}:${OCF_RESKEY_directory} || exit $OCF_ERR_GENERIC
-
-	# Restore the rmtab to ensure smooth NFS-over-TCP failover
-	restore_rmtab
+	# if any of directories fails to export we can exit
+	# immediately
+	ocf_run exportfs -v $opts "${OCF_RESKEY_clientspec}:$dir"
+	if [ $? -ne 0 ]; then
+		ocf_exit_reason "exportfs failed - exportfs -v $opts ${OCF_RESKEY_clientspec}:$dir"
+		exit $OCF_ERR_GENERIC
+	fi
 
-	ocf_log info "File system exported"
+	ocf_log info "directory $dir exported"
 	return $OCF_SUCCESS
 }
+exportfs_start ()
+{
+	if exportfs_monitor; then
+		ocf_log debug "already exported"
+		return $OCF_SUCCESS
+	fi
+	ocf_log info "Exporting file system(s) ..."
+	forall export_one
+
+	# Restore the rmtab to ensure smooth NFS-over-TCP failover
+	if [ ${OCF_RESKEY_rmtab_backup} != "none" ]; then
+		forall restore_rmtab
+	fi
+}
 
 unlock_fs() {
+	local dir=$1
 	local unlockfile
 	unlockfile=/proc/fs/nfsd/unlock_filesystem
 	if [ -w ${unlockfile} ]; then
-		echo "${OCF_RESKEY_directory}" > ${unlockfile}
-		ocf_log info "Unlocked NFS export ${OCF_RESKEY_directory}"
+		echo "$dir" > ${unlockfile}
+		ocf_log info "Unlocked NFS export $dir"
 	else
-		ocf_log warn "Unable to unlock NFS export ${OCF_RESKEY_directory}, ${unlockfile} not found or not writable"
+		ocf_log warn "Unable to unlock NFS export $dir, ${unlockfile} not found or not writable"
 	fi
 }
 wait_for_leasetime() {
@@ -292,35 +345,44 @@ wait_for_leasetime() {
 cleanup_export_cache() {
 	# see if the cache is blocking unexport
 	local contentfile=/proc/net/rpc/nfsd.export/content
-	local fsid_re="fsid=$OCF_RESKEY_fsid,"
+	local fsid_re
 	local i=1
+	fsid_re="fsid=(echo `forall get_fsid`|sed 's/ /|/g'),"
 	while :; do
-		fgrep -q "$fsid_re" $contentfile ||
+		grep -E -q "$fsid_re" $contentfile ||
 			break
 		ocf_log info "Cleanup export cache ... (try $i)"
 		ocf_run exportfs -f
 		sleep 0.5
-		let i=$i+1
+		i=$((i + 1))
 	done
 }
+unexport_one() {
+	local dir=$1
+	ocf_run exportfs -v -u ${OCF_RESKEY_clientspec}:$dir
+}
 exportfs_stop ()
 {
+	local rc
+
 	exportfs_monitor
 	if [ $? -eq $OCF_NOT_RUNNING ]; then
-		ocf_log debug "${OCF_RESKEY_directory} not exported"
+		ocf_log debug "not exported"
 		return $OCF_SUCCESS
 	fi
 
 	ocf_log info "Un-exporting file system ..."
 
 	# Backup the rmtab to ensure smooth NFS-over-TCP failover
-	backup_rmtab
+	if [ ${OCF_RESKEY_rmtab_backup} != "none" ]; then
+		forall backup_rmtab
+	fi
 
-	ocf_run exportfs -v -u ${OCF_RESKEY_clientspec}:${OCF_RESKEY_directory}
+	forall unexport_one
 	rc=$?
 
 	if ocf_is_true ${OCF_RESKEY_unlock_on_stop}; then
-		unlock_fs
+		forall unlock_fs
 	fi
 
 	if ocf_is_true ${OCF_RESKEY_wait_for_leasetime_on_stop}; then
@@ -329,22 +391,43 @@ exportfs_stop ()
 
 	if [ $rc -eq 0 ]; then
 		cleanup_export_cache
-		ocf_log info "Un-exported file system"
+		ocf_log info "Un-exported file system(s)"
 		return $OCF_SUCCESS
 	else
-		ocf_log err "Failed to un-export file system"
-		exit $OCF_ERR_GENERIC
+		ocf_exit_reason "Failed to un-export file system(s)"
+		return $OCF_ERR_GENERIC
 	fi
 }
 
+testdir() {
+	if [ ! -d $1 ]; then
+		ocf_is_probe ||
+			ocf_log err "$1 does not exist or is not a directory"
+		return 1
+	fi
+	return 0
+}
 exportfs_validate_all ()
 {
-	if [ ! -d $OCF_RESKEY_directory ]; then
-		ocf_log err "$OCF_RESKEY_directory does not exist or is not a directory"
+	if [ `echo "$OCF_RESKEY_directory" | wc -w` -gt 1 ] &&
+			! ocf_is_decimal "$OCF_RESKEY_fsid"; then
+		ocf_exit_reason "use integer fsid when exporting multiple directories"
+		return $OCF_ERR_CONFIGURED
+	fi
+	if ! forall testdir; then
 		return $OCF_ERR_INSTALLED
 	fi
 }
 
+# If someone puts a trailing slash at the end of the export directory,
+# this agent is going to fail in some unexpected ways due to how
+# export strings are matched.  The simplest solution here is to strip off
+# a trailing '/' in the directory before processing anything.
+newdir=$(echo "$OCF_RESKEY_directory" | sed -n -e 's/^\(.*\)\/$/\1/p')
+if [ -n "$newdir" ]; then
+	OCF_RESKEY_directory=$newdir
+fi
+
 OCF_REQUIRED_PARAMS="directory fsid clientspec"
 OCF_REQUIRED_BINARIES="exportfs"
 ocf_rarun $*
diff --git a/heartbeat/findif.sh b/heartbeat/findif.sh
index fd92d9b..2092d7a 100644
--- a/heartbeat/findif.sh
+++ b/heartbeat/findif.sh
@@ -45,15 +45,91 @@ getnetworkinfo()
   echo $line)
 }
 
+# previous versions of the IPaddr2 resource agent used to accept a netmask
+# in dotted quad notation (and convert to cidr notation implicitly; possibly
+# with warnings nobody ever noticed)
+# We can do so here as well.
+maybe_convert_dotted_quad_to_cidr()
+{
+	# does this even look like a dotted quad notation?
+	case $netmask in
+	# invalid if it contains other than digits and dots
+	# invalid if it contains adjacent dots,
+	#   or starts or ends with a dot
+	#   or more than three dots
+	#   or more than three digits in a row
+	*[!0-9.]* | *..* | *.*.*.*.* | .* | *. | *[0-9][0-9][0-9][0-9]* )
+		return ;;
+
+	# do we have three dots?
+	# component range check on <= 255 is done below
+	*.*.*.*) : ;;
+
+	*)	return ;;
+	esac
+
+	local IFS=.
+	set -- $netmask
+	[ $# = 4 ] || return;
+
+	local b m=0 mask;
+	for b ; do
+		[ $b -le 255 ] || return;
+		m=$(( (m << 8) + b ));
+	done;
+	case $m in
+	# for i in `seq 32 -1 0`; do printf "%10u) netmask=$i ;;\n" $(( ((1 << i)-1) << (32 - i) )); done
+        4294967295) mask=32 ;;
+        4294967294) mask=31 ;;
+        4294967292) mask=30 ;;
+        4294967288) mask=29 ;;
+        4294967280) mask=28 ;;
+        4294967264) mask=27 ;;
+        4294967232) mask=26 ;;
+        4294967168) mask=25 ;;
+        4294967040) mask=24 ;;
+        4294966784) mask=23 ;;
+        4294966272) mask=22 ;;
+        4294965248) mask=21 ;;
+        4294963200) mask=20 ;;
+        4294959104) mask=19 ;;
+        4294950912) mask=18 ;;
+        4294934528) mask=17 ;;
+        4294901760) mask=16 ;;
+        4294836224) mask=15 ;;
+        4294705152) mask=14 ;;
+        4294443008) mask=13 ;;
+        4293918720) mask=12 ;;
+        4292870144) mask=11 ;;
+        4290772992) mask=10 ;;
+        4286578688) mask=9 ;;
+        4278190080) mask=8 ;;
+        4261412864) mask=7 ;;
+        4227858432) mask=6 ;;
+        4160749568) mask=5 ;;
+        4026531840) mask=4 ;;
+        3758096384) mask=3 ;;
+        3221225472) mask=2 ;;
+        2147483648) mask=1 ;;
+                 0) mask=0 ;;
+		 *) ocf_log err "Bogus netmask: $netmask" ; return ;;
+	esac
+	ocf_log warn "Please convert dotted quad netmask $netmask to CIDR notation $mask!"
+	netmask=$mask
+}
+
 findif_check_params()
 {
   local family="$1"
   local match="$OCF_RESKEY_ip"
   local nic="$OCF_RESKEY_nic"
-  local netmask="$OCF_RESKEY_cidr_netmask"
+  # netmask NOT local, see maybe_convert_dotted_quad_to_cidr
+        netmask="$OCF_RESKEY_cidr_netmask"
   local brdcast="$OCF_RESKEY_broadcast"
   local errmsg
 
+  maybe_convert_dotted_quad_to_cidr
+
   # Do a sanity check only on start and validate-all
   # to avoid returning OCF_ERR_CONFIGURED from the monitor operation.
   case $__OCF_ACTION in
@@ -137,9 +213,9 @@ findif()
   fi
   if [ -n "$nic" ] ; then
     # NIC supports more than two.
-    set -- `ip -o -f $family route list match $match $scope | grep "dev $nic"`
+    set -- $(ip -o -f $family route list match $match $scope | grep "dev $nic " | awk 'BEGIN{best=0} { mask=$1; sub(".*/", "", mask); if( int(mask)>=best ) { best=int(mask); best_ln=$0; } } END{print best_ln}')
   else
-    set -- `ip -o -f $family route list match $match $scope`
+    set -- $(ip -o -f $family route list match $match $scope | awk 'BEGIN{best=0} { mask=$1; sub(".*/", "", mask); if( int(mask)>=best ) { best=int(mask); best_ln=$0; } } END{print best_ln}')
   fi
   if [ $# = 0 ] ; then
     case $OCF_RESKEY_ip in
diff --git a/heartbeat/galera b/heartbeat/galera
new file mode 100755
index 0000000..994aad0
--- /dev/null
+++ b/heartbeat/galera
@@ -0,0 +1,691 @@
+#!/bin/sh
+#
+# Copyright (c) 2014 David Vossel <dvossel at redhat.com>
+#                    All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of version 2 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# Further, this software is distributed without any warranty that it is
+# free of the rightful claim of any third person regarding infringement
+# or the like.  Any license provided herein, whether implied or
+# otherwise, applies only to this software file.  Patent licenses, if
+# any, provided herein do not apply to combinations of this program with
+# other software, or any other product whatsoever.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
+#
+
+##
+# README.
+# 
+# This agent only supports being configured as a multistate Master
+# resource.
+#
+# Slave vs Master role:
+#
+# During the 'Slave' role, galera instances are in read-only mode and
+# will not attempt to connect to the cluster. This role exists only as
+# a means to determine which galera instance is the most up-to-date. The
+# most up-to-date node will be used to bootstrap a galera cluster that
+# has no current members.
+#
+# The galera instances will only begin to be promoted to the Master role
+# once all the nodes in the 'wsrep_cluster_address' connection address
+# have entered read-only mode. At that point the node containing the
+# database that is most current will be promoted to Master. Once the first
+# Master instance bootstraps the galera cluster, the other nodes will be
+# promoted to Master as well.
+#
+# Example: Create a galera cluster using nodes rhel7-node1 rhel7-node2 rhel7-node3
+#
+# pcs resource create db galera enable_creation=true \
+# wsrep_cluster_address="gcomm://rhel7-auto1,rhel7-auto2,rhel7-auto3" meta master-max=3 --master
+#
+# By setting the 'enable_creation' option, the database will be automatically 
+# generated at startup. The meta attribute 'master-max=3' means that all 3
+# nodes listed in the wsrep_cluster_address list will be allowed to connect
+# to the galera cluster and perform replication.
+#
+# NOTE: If you have more nodes in the pacemaker cluster then you wish
+# to have in the galera cluster, make sure to use location contraints to prevent
+# pacemaker from attempting to place a galera instance on a node that is
+# not in the 'wsrep_cluster_address" list. 
+#
+##
+
+#######################################################################
+# Initialization:
+
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+. ${OCF_FUNCTIONS_DIR}/mysql-common.sh
+
+# It is common for some galera instances to store
+# check user that can be used to query status
+# in this file
+if [ -f "/etc/sysconfig/clustercheck" ]; then
+    . /etc/sysconfig/clustercheck
+fi
+
+#######################################################################
+
+usage() {
+  cat <<UEND
+usage: $0 (start|stop|validate-all|meta-data|monitor|promote|demote)
+
+$0 manages a galera Database as an HA resource.
+
+The 'start' operation starts the database.
+The 'stop' operation stops the database.
+The 'status' operation reports whether the database is running
+The 'monitor' operation reports whether the database seems to be working
+The 'promote' operation makes this mysql server run as master
+The 'demote' operation makes this mysql server run as slave
+The 'validate-all' operation reports whether the parameters are valid
+
+UEND
+}
+
+meta_data() {
+   cat <<END
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="galera">
+<version>1.0</version>
+
+<longdesc lang="en">
+Resource script for managing galara database.
+</longdesc>
+<shortdesc lang="en">Manages a galara instance</shortdesc>
+<parameters>
+
+<parameter name="binary" unique="0" required="0">
+<longdesc lang="en">
+Location of the MySQL server binary
+</longdesc>
+<shortdesc lang="en">MySQL server binary</shortdesc>
+<content type="string" default="${OCF_RESKEY_binary_default}" />
+</parameter>
+
+<parameter name="client_binary" unique="0" required="0">
+<longdesc lang="en">
+Location of the MySQL client binary
+</longdesc>
+<shortdesc lang="en">MySQL client binary</shortdesc>
+<content type="string" default="${OCF_RESKEY_client_binary_default}" />
+</parameter>
+
+<parameter name="config" unique="0" required="0">
+<longdesc lang="en">
+Configuration file
+</longdesc>
+<shortdesc lang="en">MySQL config</shortdesc>
+<content type="string" default="${OCF_RESKEY_config_default}" />
+</parameter>
+
+<parameter name="datadir" unique="0" required="0">
+<longdesc lang="en">
+Directory containing databases
+</longdesc>
+<shortdesc lang="en">MySQL datadir</shortdesc>
+<content type="string" default="${OCF_RESKEY_datadir_default}" />
+</parameter>
+
+<parameter name="user" unique="0" required="0">
+<longdesc lang="en">
+User running MySQL daemon
+</longdesc>
+<shortdesc lang="en">MySQL user</shortdesc>
+<content type="string" default="${OCF_RESKEY_user_default}" />
+</parameter>
+
+<parameter name="group" unique="0" required="0">
+<longdesc lang="en">
+Group running MySQL daemon (for logfile and directory permissions)
+</longdesc>
+<shortdesc lang="en">MySQL group</shortdesc>
+<content type="string" default="${OCF_RESKEY_group_default}"/>
+</parameter>
+
+<parameter name="log" unique="0" required="0">
+<longdesc lang="en">
+The logfile to be used for mysqld.
+</longdesc>
+<shortdesc lang="en">MySQL log file</shortdesc>
+<content type="string" default="${OCF_RESKEY_log_default}"/>
+</parameter>
+
+<parameter name="pid" unique="0" required="0">
+<longdesc lang="en">
+The pidfile to be used for mysqld.
+</longdesc>
+<shortdesc lang="en">MySQL pid file</shortdesc>
+<content type="string" default="${OCF_RESKEY_pid_default}"/>
+</parameter>
+
+<parameter name="socket" unique="0" required="0">
+<longdesc lang="en">
+The socket to be used for mysqld.
+</longdesc>
+<shortdesc lang="en">MySQL socket</shortdesc>
+<content type="string" default="${OCF_RESKEY_socket_default}"/>
+</parameter>
+
+<parameter name="enable_creation" unique="0" required="0">
+<longdesc lang="en">
+If the MySQL database does not exist, it will be created
+</longdesc>
+<shortdesc lang="en">Create the database if it does not exist</shortdesc>
+<content type="boolean" default="${OCF_RESKEY_enable_creation_default}"/>
+</parameter>
+
+<parameter name="additional_parameters" unique="0" required="0">
+<longdesc lang="en">
+Additional parameters which are passed to the mysqld on startup.
+(e.g. --skip-external-locking or --skip-grant-tables)
+</longdesc>
+<shortdesc lang="en">Additional parameters to pass to mysqld</shortdesc>
+<content type="string" default="${OCF_RESKEY_additional_parameters_default}"/>
+</parameter>
+
+
+<parameter name="wsrep_cluster_address" unique="0" required="1">
+<longdesc lang="en">
+The galera cluster address. This takes the form of:
+gcomm://node,node,node
+
+Only nodes present in this node list will be allowed to start a galera instance.
+It is expected that the galera node names listed in this address match valid
+pacemaker node names.
+</longdesc>
+<shortdesc lang="en">Galera cluster address</shortdesc>
+<content type="string" default=""/>
+</parameter>
+
+<parameter name="check_user" unique="0" required="0">
+<longdesc lang="en">
+Cluster check user.
+</longdesc>
+<shortdesc lang="en">MySQL test user</shortdesc>
+<content type="string" default="root" />
+</parameter>
+
+<parameter name="check_passwd" unique="0" required="0">
+<longdesc lang="en">
+Cluster check user password
+</longdesc>
+<shortdesc lang="en">check password</shortdesc>
+<content type="string" default="" />
+</parameter>
+
+</parameters>
+
+<actions>
+<action name="start" timeout="120" />
+<action name="stop" timeout="120" />
+<action name="status" timeout="60" />
+<action name="monitor" depth="0" timeout="30" interval="20" />
+<action name="monitor" role="Master" depth="0" timeout="30" interval="10" />
+<action name="monitor" role="Slave" depth="0" timeout="30" interval="30" />
+<action name="promote" timeout="300" />
+<action name="demote" timeout="120" />
+<action name="validate-all" timeout="5" />
+<action name="meta-data" timeout="5" />
+</actions>
+</resource-agent>
+END
+}
+
+get_option_variable()
+{
+    local key=$1
+
+    $MYSQL $MYSQL_OPTIONS_CHECK  -e "SHOW VARIABLES like '$key';" | tail -1
+}
+
+get_status_variable()
+{
+    local key=$1
+
+    $MYSQL $MYSQL_OPTIONS_CHECK -e "show status like '$key';" | tail -1
+}
+
+set_bootstrap_node()
+{
+    local node=$1
+
+    ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-bootstrap" -v "true"
+}
+
+clear_bootstrap_node()
+{
+    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-bootstrap" -D
+}
+
+is_bootstrap()
+{
+    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-bootstrap" -Q 2>/dev/null
+
+}
+
+clear_last_commit()
+{
+    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" -D
+}
+
+set_last_commit()
+{
+    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" -v $1
+}
+
+get_last_commit()
+{
+    local node=$1
+
+    if [ -z "$node" ]; then
+       ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" -Q 2>/dev/null
+    else 
+       ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" -Q 2>/dev/null
+    fi
+}
+
+wait_for_sync()
+{
+    local state=$(get_status_variable "wsrep_local_state")
+
+    ocf_log info "Waiting for database to sync with the cluster. "
+    while [ "$state" != "4" ]; do
+        sleep 1
+        state=$(get_status_variable "wsrep_local_state")
+    done
+    ocf_log info "Database synced."
+}
+
+is_primary()
+{
+    cluster_status=$(get_status_variable "wsrep_cluster_status")
+    if [ "$cluster_status" = "Primary" ]; then
+        return 0
+    fi
+
+    if [ -z "$cluster_status" ]; then
+        ocf_exit_reason "Unable to retrieve wsrep_cluster_status, verify check_user '$OCF_RESKEY_check_user' has permissions to view status"
+    else
+        ocf_log info "Galera instance wsrep_cluster_status=${cluster_status}"
+    fi
+    return 1
+}
+
+is_readonly()
+{
+    local res=$(get_option_variable "read_only")
+
+    if ! ocf_is_true "$res"; then
+        return 1
+    fi
+
+    cluster_status=$(get_status_variable "wsrep_cluster_status")
+    if ! [ "$cluster_status" = "Disconnected" ]; then
+        return 1
+    fi
+
+    return 0
+}
+
+master_exists()
+{
+    # determine if a master instance is already up and is healthy
+    crm_mon --as-xml | grep "resource.*id=\"${OCF_RESOURCE_INSTANCE}\".*role=\"Master\".*active=\"true\".*orphaned=\"false\".*failed=\"false\"" > /dev/null 2>&1
+    return $?
+}
+
+clear_master_score()
+{
+    local node=$1
+    if [ -z "$node" ]; then
+        $CRM_MASTER -D
+    else 
+        $CRM_MASTER -D -N $node
+    fi
+}
+
+set_master_score()
+{
+    local node=$1
+
+    if [ -z "$node" ]; then
+        $CRM_MASTER -v 100
+    else 
+        $CRM_MASTER -N $node -v 100
+    fi
+}
+
+promote_everyone()
+{
+
+    for node in $(echo "$OCF_RESKEY_wsrep_cluster_address" | sed 's/gcomm:\/\///g' | tr -d ' ' | tr -s ',' ' '); do
+
+        set_master_score $node
+    done
+}
+
+greater_than_equal_long()
+{
+    # there are values we need to compare in this script
+    # that are too large for shell -gt to process
+    echo | awk -v n1="$1" -v n2="$2"  '{if (n1>=n2) printf ("true"); else printf ("false");}' |  grep -q "true"
+}
+
+detect_first_master()
+{
+    local best_commit=0
+    local best_node="$NODENAME"
+    local last_commit=0
+    local missing_nodes=0
+
+    for node in $(echo "$OCF_RESKEY_wsrep_cluster_address" | sed 's/gcomm:\/\///g' | tr -d ' ' | tr -s ',' ' '); do
+        last_commit=$(get_last_commit $node)
+
+        if [ -z "$last_commit" ]; then
+            ocf_log info "Waiting on node <${node}> to report database status before Master instances can start."
+            missing_nodes=1
+            continue
+        fi
+
+        # this means -1, or that no commit has occured yet.
+        if [ "$last_commit" = "18446744073709551615" ]; then
+            last_commit="0"
+        fi
+
+        greater_than_equal_long "$last_commit" "$best_commit"
+        if [ $? -eq 0 ]; then
+            best_node=$node
+            best_commit=$last_commit
+        fi
+
+    done
+
+    if [ $missing_nodes -eq 1 ]; then
+        return
+    fi
+
+    ocf_log info "Promoting $best_node to be our bootstrap node"
+    set_master_score $best_node
+    set_bootstrap_node $best_node
+}
+
+# For galera, promote is really start
+galera_promote()
+{
+    local rc
+    local extra_opts
+    local bootstrap
+    
+    master_exists
+    if [ $? -eq 0 ]; then
+        # join without bootstrapping
+        extra_opts="--wsrep-cluster-address=${OCF_RESKEY_wsrep_cluster_address}"
+    else
+        bootstrap=$(is_bootstrap)
+
+        if ocf_is_true $bootstrap; then
+            ocf_log info "Node <${NODENAME}> is bootstrapping the cluster"
+            extra_opts="--wsrep-cluster-address=gcomm://"
+        else
+            ocf_exit_reason "Failure, Attempted to promote Master instance of $OCF_RESOURCE_INSTANCE before bootstrap node has been detected."
+            return $OCF_ERR_GENERIC
+        fi
+
+    fi
+
+    # make sure the read only instance is stopped
+    mysql_common_stop
+    rc=$?
+    if [ $rc -ne $OCF_SUCCESS ] && [ $rc -ne $OCF_NOT_RUNNING ]; then
+        ocf_exit_reason "Failed to stop read-only galera instance during promotion to Master"
+        return $rc
+    fi
+
+    sleep 4
+
+    mysql_common_prepare_dirs
+    mysql_common_start "$extra_opts"
+    rc=$?
+    if [ $rc != $OCF_SUCCESS ]; then
+        return $rc
+    fi
+
+    galera_monitor
+    rc=$?
+    if [ $rc != $OCF_SUCCESS -a $rc != $OCF_RUNNING_MASTER ]; then
+        ocf_exit_reason "Failed initial monitor action"
+        return $rc
+    fi
+
+    is_readonly
+    if [ $? -eq 0 ]; then
+        ocf_exit_reason "Failure. Master instance started in read-only mode, check configuration."
+        return $OCF_ERR_GENERIC
+    fi
+
+    is_primary
+    if [ $? -ne 0 ]; then
+        ocf_exit_reason "Failure. Master instance started, but is not in Primary mode."
+        return $OCF_ERR_GENERIC
+    fi
+
+    if ocf_is_true $bootstrap; then
+        promote_everyone
+        clear_bootstrap_node
+        ocf_log info "Bootstrap complete, promoting the rest of the galera instances."
+    else
+        # if this is not the bootstrap node, make sure this instance
+        # syncs with the rest of the cluster before promotion returns.
+        wait_for_sync
+    fi
+
+    # last commit is no longer relevant once promoted
+    clear_last_commit
+
+    ocf_log info "Galera started"
+    return $OCF_SUCCESS
+}
+
+galera_demote()
+{
+    mysql_common_stop
+    rc=$?
+    if [ $rc -ne $OCF_SUCCESS ] && [ $rc -ne $OCF_NOT_RUNNING ]; then
+        ocf_exit_reason "Failed to stop Master galera instance during demotion to Master"
+        return $rc
+    fi
+
+    # if this node was previously a bootstrap node, that is no longer the case.
+    clear_bootstrap_node
+
+    # start again in slave mode so the new last commit is recorded
+    galera_start
+}
+
+galera_start()
+{
+    local extra_opts='--read-only=true'
+    local last_commit
+
+    echo $OCF_RESKEY_wsrep_cluster_address | grep -q $NODENAME
+    if [ $? -ne 0 ]; then
+        ocf_exit_reason "local node <${NODENAME}> must be a member of the wsrep_cluster_address <${OCF_RESKEY_wsrep_cluster_address}>to start this galera instance"
+        return $OCF_ERR_CONFIGURED
+    fi
+
+    mysql_common_prepare_dirs
+    mysql_common_start "$extra_opts"
+
+    is_readonly
+    if [ $? -ne 0 ]; then
+        ocf_exit_reason "Slave instance did not start correctly in read-only mode, Make sure local galera.cnf does not have wsrep_cluster_address set."
+        return $OCF_ERR_GENERIC
+    fi
+
+    ocf_log info "attempting to detect last commit version"
+    while [ -z "$last_commit" ]; do
+        last_commit=$(get_status_variable "wsrep_last_committed")
+        if [ -z "$last_commit" ]; then
+            sleep 1
+        fi
+    done
+    ocf_log info "Last commit version found:  $last_commit"
+
+    set_last_commit $last_commit
+
+    master_exists
+    if [ $? -eq 0 ]; then
+        ocf_log info "Master instances are already up, setting master score so this instance will join galera cluster."
+        set_master_score $NODENAME
+    else
+        clear_master_score
+        detect_first_master
+    fi
+
+    return $OCF_SUCCESS
+}
+
+galera_monitor()
+{
+    local rc
+    local status_loglevel="err"
+
+    # Set loglevel to info during probe
+    if ocf_is_probe; then
+        status_loglevel="info"
+    fi
+ 
+    mysql_common_status $status_loglevel
+    rc=$?
+
+    # If status returned an error, return that immediately
+    if [ $rc -ne $OCF_SUCCESS ]; then
+        return $rc
+    fi
+
+    echo $OCF_RESKEY_wsrep_cluster_address | grep -q $NODENAME
+    if [ $? -ne 0 ]; then
+        ocf_exit_reason "local node <${NODENAME}> is started, but is not a member of the wsrep_cluster_address <${OCF_RESKEY_wsrep_cluster_address}>"
+        return $OCF_ERR_GENERIC
+    fi
+
+    is_readonly
+    if [ $? -ne 0 ]; then
+        is_primary
+        if [ $? -ne 0 ]; then
+            ocf_exit_reason "local node <${NODENAME}> is neither in primary mode nor in read_only mode. Unknown state."
+            return $OCF_ERR_GENERIC
+        fi
+
+        if ocf_is_probe; then
+            # restore master score during probe
+            # if we detect this is a master instance
+            set_master_score
+        fi
+        rc=$OCF_RUNNING_MASTER
+    else 
+        master_exists
+        if [ $? -ne 0 ]; then
+            detect_first_master
+        else
+            # a master instance exists and is healthy, promote this
+            # local read only instance
+            # so it can join the master galera cluster.
+            set_master_score
+        fi
+    fi
+    # TODO look at what is done in the wait script
+
+    return $rc
+}
+
+galera_stop()
+{
+    local rc
+    # make sure the process is stopped
+    mysql_common_stop
+    rc=$1
+
+    clear_last_commit
+    clear_master_score
+    clear_bootstrap_node
+    return $rc
+}
+
+galera_validate()
+{
+    if ! ocf_is_ms; then
+        ocf_exit_reason "Galera must be configured as a multistate Master/Slave resource."
+        return $OCF_ERR_CONFIGURED
+    fi
+
+    if [ -z "$OCF_RESKEY_wsrep_cluster_address" ]; then
+        ocf_exit_reason "Galera must be configured with a wsrep_cluster_address value."
+        return $OCF_ERR_CONFIGURED
+    fi
+
+    mysql_common_validate
+}
+
+case "$1" in
+  meta-data)    meta_data
+        exit $OCF_SUCCESS;;
+  usage|help)   usage
+        exit $OCF_SUCCESS;;
+esac
+
+galera_validate
+rc=$?
+LSB_STATUS_STOPPED=3
+if [ $rc -ne 0 ]; then
+    case "$1" in
+        stop) exit $OCF_SUCCESS;;
+        monitor) exit $OCF_NOT_RUNNING;;
+        status) exit $LSB_STATUS_STOPPED;;
+        *) exit $rc;;
+    esac
+fi
+
+if [ -z "${OCF_RESKEY_check_passwd}" ]; then
+    # This value is automatically sourced from /etc/sysconfig/checkcluster if available
+    OCF_RESKEY_check_passwd=${MYSQL_PASSWORD}
+fi
+if [ -z "${OCF_RESKEY_check_user}" ]; then
+    # This value is automatically sourced from /etc/sysconfig/checkcluster if available
+    OCF_RESKEY_check_user=${MYSQL_USERNAME}
+fi
+: ${OCF_RESKEY_check_user="root"}
+
+MYSQL_OPTIONS_CHECK="-nNE --user=${OCF_RESKEY_check_user}"
+if [ -n "${OCF_RESKEY_check_passwd}" ]; then
+    MYSQL_OPTIONS_CHECK="$MYSQL_OPTIONS_CHECK --password=${OCF_RESKEY_check_passwd}"
+fi
+
+# What kind of method was invoked?
+case "$1" in
+  start)    galera_start;;
+  stop)     galera_stop;;
+  status)   mysql_common_status err;;
+  monitor)  galera_monitor;;
+  promote)  galera_promote;;
+  demote)   galera_demote;;
+  validate-all) exit $OCF_SUCCESS;;
+
+ *)     usage
+        exit $OCF_ERR_UNIMPLEMENTED;;
+esac
+
+# vi:sw=4:ts=4:et:
diff --git a/heartbeat/iSCSILogicalUnit b/heartbeat/iSCSILogicalUnit
index 0ab4722..ffd66ff 100755
--- a/heartbeat/iSCSILogicalUnit
+++ b/heartbeat/iSCSILogicalUnit
@@ -3,10 +3,10 @@
 #
 #   iSCSILogicalUnit OCF RA. Exports and manages iSCSI Logical Units.
 #
-#   (c) 2013	LINBIT, Lars Ellenberg
+#   (c) 2013 LINBIT, Lars Ellenberg
 #   (c) 2009-2010 Florian Haas, Dejan Muhamedagic,
-#                 and Linux-HA contributors
-#				  
+#       and Linux-HA contributors
+#
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of version 2 of the GNU General Public License as
@@ -36,11 +36,13 @@
 # Defaults
 # Set a default implementation based on software installed
 if have_binary ietadm; then
-    OCF_RESKEY_implementation_default="iet"
+	OCF_RESKEY_implementation_default="iet"
 elif have_binary tgtadm; then
-    OCF_RESKEY_implementation_default="tgt"
+	OCF_RESKEY_implementation_default="tgt"
 elif have_binary lio_node; then
-    OCF_RESKEY_implementation_default="lio"
+	OCF_RESKEY_implementation_default="lio"
+elif have_binary targetcli; then
+	OCF_RESKEY_implementation_default="lio-t"
 fi
 : ${OCF_RESKEY_implementation=${OCF_RESKEY_implementation_default}}
 
@@ -91,9 +93,9 @@ an SCSI Target, exported via a daemon that speaks the iSCSI protocol.
 <parameter name="implementation" required="0" unique="0">
 <longdesc lang="en">
 The iSCSI target daemon implementation. Must be one of "iet", "tgt",
-or "lio".  If unspecified, an implementation is selected based on the
+"lio", or "lio-t".  If unspecified, an implementation is selected based on the
 availability of management utilities, with "iet" being tried first,
-then "tgt", then "lio".
+then "tgt", then "lio", then "lio-t".
 </longdesc>
 <shortdesc lang="en">iSCSI target daemon implementation</shortdesc>
 <content type="string" default="${OCF_RESKEY_implementation_default}"/>
@@ -228,12 +230,12 @@ in /sys/kernel/config/target/core/.
 </parameters>
 
 <actions>
-<action name="start"        timeout="10" />
-<action name="stop"         timeout="10" />
-<action name="status"       timeout="10" interval="10" depth="0" />
-<action name="monitor"      timeout="10" interval="10" depth="0" />
-<action name="meta-data"    timeout="5" />
-<action name="validate-all"   timeout="10" />
+<action name="start"         timeout="10" />
+<action name="stop"          timeout="10" />
+<action name="status"        timeout="10" interval="10" depth="0" />
+<action name="monitor"       timeout="10" interval="10" depth="0" />
+<action name="meta-data"     timeout="5" />
+<action name="validate-all"  timeout="10" />
 </actions>
 </resource-agent>
 END
@@ -250,108 +252,150 @@ END
 }
 
 iSCSILogicalUnit_start() {
-    iSCSILogicalUnit_monitor
-    if [ $? =  $OCF_SUCCESS ]; then
-	return $OCF_SUCCESS
-    fi
+	iSCSILogicalUnit_monitor
+	if [ $? =  $OCF_SUCCESS ]; then
+		return $OCF_SUCCESS
+	fi
 
-    local params
+	local params
 
-    case $OCF_RESKEY_implementation in
+	case $OCF_RESKEY_implementation in
 	iet)
-	    params="Path=${OCF_RESKEY_path}"
-	    # use blockio if path points to a block device, fileio
-	    # otherwise.
-	    if [ -b "${OCF_RESKEY_path}" ]; then
-		params="${params} Type=blockio"
-	    else
-		params="${params} Type=fileio"
-	    fi
-	    # in IET, we have to set LU parameters on creation
-	    if [ -n "${OCF_RESKEY_scsi_id}" ]; then
-		params="${params} ScsiId=${OCF_RESKEY_scsi_id}"
-	    fi
-	    if [ -n "${OCF_RESKEY_scsi_sn}" ]; then
-		params="${params} ScsiSN=${OCF_RESKEY_scsi_sn}"
-	    fi
-	    params="${params} ${OCF_RESKEY_additional_parameters}"
-	    ocf_run ietadm --op new \
-		--tid=${TID} \
-		--lun=${OCF_RESKEY_lun} \
-		--params ${params// /,} || exit $OCF_ERR_GENERIC
-	    ;;
+		params="Path=${OCF_RESKEY_path}"
+		# use blockio if path points to a block device, fileio
+		# otherwise.
+		if [ -b "${OCF_RESKEY_path}" ]; then
+			params="${params} Type=blockio"
+		else
+			params="${params} Type=fileio"
+		fi
+		# in IET, we have to set LU parameters on creation
+		if [ -n "${OCF_RESKEY_scsi_id}" ]; then
+			params="${params} ScsiId=${OCF_RESKEY_scsi_id}"
+		fi
+		if [ -n "${OCF_RESKEY_scsi_sn}" ]; then
+			params="${params} ScsiSN=${OCF_RESKEY_scsi_sn}"
+		fi
+		params="${params} ${OCF_RESKEY_additional_parameters}"
+		ocf_run ietadm --op new \
+			--tid=${TID} \
+			--lun=${OCF_RESKEY_lun} \
+			--params ${params// /,} || exit $OCF_ERR_GENERIC
+		;;
 	tgt)
-	    # tgt requires that we create the LU first, then set LU
-	    # parameters
-	    params=""
-	    local var
-            local envar
-	    for var in scsi_id scsi_sn vendor_id product_id; do
-		envar="OCF_RESKEY_${var}"
-		if [ -n "${!envar}" ]; then
-		    params="${params} ${var}=${!envar}"
+		# tgt requires that we create the LU first, then set LU
+		# parameters
+		params=""
+		local var
+		local envar
+		for var in scsi_id scsi_sn vendor_id product_id; do
+			envar="OCF_RESKEY_${var}"
+			if [ -n "${!envar}" ]; then
+				params="${params} ${var}=${!envar}"
+			fi
+		done
+		params="${params} ${OCF_RESKEY_additional_parameters}"
+
+		# cleanup: tgt (as of tgtadm version 1.0.24) does not like an explicit "bsoflags=direct"
+		# when used with "bstype=aio" (which always uses O_DIRECT)
+		[[ $OCF_RESKEY_tgt_bstype/$OCF_RESKEY_tgt_bsoflags = "aio/direct" ]] && OCF_RESKEY_tgt_bsoflags=""
+
+		tgt_args=""
+		[[ $OCF_RESKEY_tgt_bstype ]]	&& tgt_args="$tgt_args --bstype=$OCF_RESKEY_tgt_bstype"
+		[[ $OCF_RESKEY_tgt_bsoflags ]]	&& tgt_args="$tgt_args --bsoflags=$OCF_RESKEY_tgt_bsoflags"
+		[[ $OCF_RESKEY_tgt_device_type ]]	&& tgt_args="$tgt_args --device-type=$OCF_RESKEY_tgt_device_type"
+
+		ocf_run tgtadm --lld iscsi --op new --mode logicalunit \
+			--tid=${TID} \
+			--lun=${OCF_RESKEY_lun} \
+			$tgt_args \
+			--backing-store ${OCF_RESKEY_path} || exit $OCF_ERR_GENERIC
+		if [ -z "$params" ]; then
+			return $OCF_SUCCESS
+		else
+			ocf_run tgtadm --lld iscsi --op update --mode logicalunit \
+				--tid=${TID} \
+				--lun=${OCF_RESKEY_lun} \
+				--params ${params// /,} || exit $OCF_ERR_GENERIC
+		fi
+		;;
+	lio)
+		# For lio, we first have to create a target device, then
+		# add it to the Target Portal Group as an LU.
+
+		block_configfs_path="/sys/kernel/config/target/core/iblock_${OCF_RESKEY_lio_iblock}/${OCF_RESOURCE_INSTANCE}/udev_path"
+		if [ ! -e "${block_configfs_path}" ]; then
+			ocf_run tcm_node --createdev=iblock_${OCF_RESKEY_lio_iblock}/${OCF_RESOURCE_INSTANCE} \
+				${OCF_RESKEY_path} || exit $OCF_ERR_GENERIC
+		elif [ -e "$block_configfs_path" ] && [ $(cat "$block_configfs_path") != "${OCF_RESKEY_path}" ]; then
+			ocf_exit_reason "Existing iblock_${OCF_RESKEY_lio_iblock}/${OCF_RESOURCE_INSTANCE} has incorrect path: $(cat "$block_configfs_path") != ${OCF_RESKEY_path}"
+			exit $OCF_ERR_GENERIC
+		else
+			ocf_log info "iscsi iblock already exists: ${block_configfs_path}"
 		fi
-	    done
-	    params="${params} ${OCF_RESKEY_additional_parameters}"
 
-	    # cleanup: tgt (as of tgtadm version 1.0.24) does not like an explicit "bsoflags=direct"
-	    # when used with "bstype=aio" (which always uses O_DIRECT)
-	    [[ $OCF_RESKEY_tgt_bstype/$OCF_RESKEY_tgt_bsoflags = "aio/direct" ]] && OCF_RESKEY_tgt_bsoflags=""
+		if [ -n "${OCF_RESKEY_scsi_sn}" ]; then
+			ocf_run tcm_node --setunitserial=iblock_${OCF_RESKEY_lio_iblock}/${OCF_RESOURCE_INSTANCE} \
+				${OCF_RESKEY_scsi_sn} || exit $OCF_ERR_GENERIC
+		fi
 
-	    tgt_args=""
-	    [[ $OCF_RESKEY_tgt_bstype ]]	&& tgt_args="$tgt_args --bstype=$OCF_RESKEY_tgt_bstype"
-	    [[ $OCF_RESKEY_tgt_bsoflags ]]	&& tgt_args="$tgt_args --bsoflags=$OCF_RESKEY_tgt_bsoflags"
-	    [[ $OCF_RESKEY_tgt_device_type ]]	&& tgt_args="$tgt_args --device-type=$OCF_RESKEY_tgt_device_type"
+		lun_configfs_path="/sys/kernel/config/target/iscsi/${OCF_RESKEY_target_iqn}/tpgt_1/lun/lun_${OCF_RESKEY_lun}/${OCF_RESOURCE_INSTANCE}/udev_path"
+		if [ ! -e "${lun_configfs_path}" ]; then
+			ocf_run lio_node --addlun=${OCF_RESKEY_target_iqn} 1 ${OCF_RESKEY_lun} \
+				${OCF_RESOURCE_INSTANCE} iblock_${OCF_RESKEY_lio_iblock}/${OCF_RESOURCE_INSTANCE} || exit $OCF_ERR_GENERIC
+		else
+			ocf_log info "iscsi lun already exists: ${lun_configfs_path}"
+		fi
 
-	    ocf_run tgtadm --lld iscsi --op new --mode logicalunit \
-		--tid=${TID} \
-		--lun=${OCF_RESKEY_lun} \
-		$tgt_args \
-	    	--backing-store ${OCF_RESKEY_path} || exit $OCF_ERR_GENERIC
-	    if [ -z "$params" ]; then
-		return $OCF_SUCCESS
-	    else
-		ocf_run tgtadm --lld iscsi --op update --mode logicalunit \
-		    --tid=${TID} \
-		    --lun=${OCF_RESKEY_lun} \
-		    --params ${params// /,} || exit $OCF_ERR_GENERIC
-	    fi
-	    ;;
-	lio)
-	    # For lio, we first have to create a target device, then
-	    # add it to the Target Portal Group as an LU.
-	    ocf_run tcm_node --createdev=iblock_${OCF_RESKEY_lio_iblock}/${OCF_RESOURCE_INSTANCE} \
-		${OCF_RESKEY_path} || exit $OCF_ERR_GENERIC
-	    if [ -n "${OCF_RESKEY_scsi_sn}" ]; then
-		ocf_run tcm_node --setunitserial=iblock_${OCF_RESKEY_lio_iblock}/${OCF_RESOURCE_INSTANCE} \
-		    ${OCF_RESKEY_scsi_sn} || exit $OCF_ERR_GENERIC
-	    fi
-	    ocf_run lio_node --addlun=${OCF_RESKEY_target_iqn} 1 ${OCF_RESKEY_lun} \
-		${OCF_RESOURCE_INSTANCE} iblock_${OCF_RESKEY_lio_iblock}/${OCF_RESOURCE_INSTANCE} || exit $OCF_ERR_GENERIC
-
-           if [ -n "${OCF_RESKEY_allowed_initiators}" ]; then
-               for initiator in ${OCF_RESKEY_allowed_initiators}; do
-                       ocf_run lio_node --addlunacl=${OCF_RESKEY_target_iqn} 1 \
-                       ${initiator} ${OCF_RESKEY_lun} ${OCF_RESKEY_lun} || exit $OCF_ERR_GENERIC
-               done
-           fi
-	    ;;
-    esac
-
-    return $OCF_SUCCESS
+		if [ -n "${OCF_RESKEY_allowed_initiators}" ]; then
+			for initiator in ${OCF_RESKEY_allowed_initiators}; do
+				acl_configfs_path="/sys/kernel/config/target/iscsi/${OCF_RESKEY_target_iqn}/tpgt_1/acls/${initiator}/lun_${OCF_RESKEY_lun}"
+				if [ ! -e "${acl_configfs_path}" ]; then
+					ocf_run lio_node --addlunacl=${OCF_RESKEY_target_iqn} 1 \
+					${initiator} ${OCF_RESKEY_lun} ${OCF_RESKEY_lun} || exit $OCF_ERR_GENERIC
+				else
+					ocf_log info "iscsi acl already exists: ${acl_configfs_path}"
+				fi
+			done
+		fi
+		;;
+	lio-t)
+		# For lio, we first have to create a target device, then
+		# add it to the Target Portal Group as an LU.
+		ocf_run targetcli /backstores/block create name=${OCF_RESOURCE_INSTANCE} dev=${OCF_RESKEY_path} || exit $OCF_ERR_GENERIC
+		if [ -n "${OCF_RESKEY_scsi_sn}" ]; then
+			echo ${OCF_RESKEY_scsi_sn} > /sys/kernel/config/target/core/iblock_${OCF_RESKEY_lio_iblock}/${OCF_RESOURCE_INSTANCE}/wwn/vpd_unit_serial
+		fi
+		ocf_run targetcli /iscsi/${OCF_RESKEY_target_iqn}/tpg1/luns create /backstores/block/${OCF_RESOURCE_INSTANCE} ${OCF_RESKEY_lun} || exit $OCF_ERR_GENERIC
+
+		if [ -n "${OCF_RESKEY_allowed_initiators}" ]; then
+			for initiator in ${OCF_RESKEY_allowed_initiators}; do
+				ocf_run targetcli /iscsi/${OCF_RESKEY_target_iqn}/tpg1/acls create ${initiator} add_mapped_luns=False || exit $OCF_ERR_GENERIC
+				ocf_run targetcli /iscsi/${OCF_RESKEY_target_iqn}/tpg1/acls/${initiator} create ${OCF_RESKEY_lun} ${OCF_RESKEY_lun} || exit $OCF_ERR_GENERIC
+			done
+		fi
+		;;
+	esac
+
+	# Force the monitor operation to pass before start is considered a success.
+	iSCSILogicalUnit_monitor
 }
 
 iSCSILogicalUnit_stop() {
-    iSCSILogicalUnit_monitor
-    if [ $? =  $OCF_SUCCESS ]; then
+	iSCSILogicalUnit_monitor
+	if [ $? -eq $OCF_NOT_RUNNING ]; then
+		return $OCF_SUCCESS
+	fi
+
 	case $OCF_RESKEY_implementation in
-	    iet)
-		# IET allows us to remove LUs while they are in use
-		ocf_run ietadm --op delete \
-		    --tid=${TID} \
-		    --lun=${OCF_RESKEY_lun} || exit $OCF_ERR_GENERIC
+
+	iet)
+	# IET allows us to remove LUs while they are in use
+	ocf_run ietadm --op delete \
+		--tid=${TID} \
+		--lun=${OCF_RESKEY_lun} || exit $OCF_ERR_GENERIC
 		;;
-	    tgt)
+	tgt)
 		# tgt will fail to remove an LU while it is in use,
 		# but at the same time does not allow us to
 		# selectively shut down a connection that is using a
@@ -359,202 +403,240 @@ iSCSILogicalUnit_stop() {
 		# decides that the LU is no longer in use, or we get
 		# timed out by the LRM.
 		while ! ocf_run -warn tgtadm --lld iscsi --op delete --mode logicalunit \
-		    --tid ${TID} \
-		    --lun=${OCF_RESKEY_lun}; do
-		    sleep 1
+			--tid ${TID} \
+			--lun=${OCF_RESKEY_lun}; do
+			sleep 1
+		done
+		;;
+	lio)
+
+		acls_configfs_path="/sys/kernel/config/target/iscsi/${OCF_RESKEY_target_iqn}/tpgt_1/acls"
+		for initiatorpath in ${acls_configfs_path}/*; do
+			initiator=$(basename "${initiatorpath}")
+			if [ -e "${initiatorpath}/lun_${OCF_RESKEY_lun}" ]; then
+				ocf_log info "deleting acl at ${initiatorpath}/lun_${OCF_RESKEY_lun}"
+				ocf_run lio_node --dellunacl=${OCF_RESKEY_target_iqn} 1 \
+					${initiator} ${OCF_RESKEY_lun} || exit $OCF_ERR_GENERIC
+			fi
 		done
+		lun_configfs_path="/sys/kernel/config/target/iscsi/${OCF_RESKEY_target_iqn}/tpgt_1/lun/lun_${OCF_RESKEY_lun}/"
+		if [ -e "${lun_configfs_path}" ]; then
+			ocf_run lio_node --dellun=${OCF_RESKEY_target_iqn} 1 ${OCF_RESKEY_lun} || exit $OCF_ERR_GENERIC
+		fi
+		block_configfs_path="/sys/kernel/config/target/core/iblock_${OCF_RESKEY_lio_iblock}/${OCF_RESOURCE_INSTANCE}/udev_path"
+		if [ -e "${block_configfs_path}" ]; then
+			ocf_run tcm_node --freedev=iblock_${OCF_RESKEY_lio_iblock}/${OCF_RESOURCE_INSTANCE} || exit $OCF_ERR_GENERIC
+		fi
+		;;
+	lio-t)
+		ocf_run targetcli /iscsi/${OCF_RESKEY_target_iqn}/tpg1/luns delete ${OCF_RESKEY_lun} || exit $OCF_ERR_GENERIC
+		if [ -n "${OCF_RESKEY_allowed_initiators}" ]; then
+			for initiator in ${OCF_RESKEY_allowed_initiators}; do
+				if targetcli /iscsi/${OCF_RESKEY_target_iqn}/tpg1/acls/${initiator} status | grep "Mapped LUNs: 0" >/dev/null ; then
+					ocf_run targetcli /iscsi/${OCF_RESKEY_target_iqn}/tpg1/acls/ delete ${initiator}
+				fi
+			done
+		fi
+
+		ocf_run targetcli /backstores/block delete ${OCF_RESOURCE_INSTANCE} || exit $OCF_ERR_GENERIC
 		;;
-	    lio)
-               if [ -n "${OCF_RESKEY_allowed_initiators}" ]; then
-                       for initiator in ${OCF_RESKEY_allowed_initiators}; do
-                               ocf_run lio_node --dellunacl=${OCF_RESKEY_target_iqn} 1 \
-                               ${initiator} ${OCF_RESKEY_lun} || exit $OCF_ERR_GENERIC
-                       done
-               fi
-		ocf_run lio_node --dellun=${OCF_RESKEY_target_iqn} 1 ${OCF_RESKEY_lun} || exit $OCF_ERR_GENERIC
-		ocf_run tcm_node --freedev=iblock_${OCF_RESKEY_lio_iblock}/${OCF_RESOURCE_INSTANCE} || exit $OCF_ERR_GENERIC
 	esac
-    fi
-    
-    return $OCF_SUCCESS
+
+	return $OCF_SUCCESS
 }
 
 iSCSILogicalUnit_monitor() {
-    case $OCF_RESKEY_implementation in
+	case $OCF_RESKEY_implementation in
 	iet)
- 	    # Figure out and set the target ID
-	    TID=`sed -ne "s/tid:\([[:digit:]]\+\) name:${OCF_RESKEY_target_iqn}$/\1/p" < /proc/net/iet/volume`
-	    if [ -z "${TID}" ]; then
-		# Our target is not configured, thus we're not
-		# running.
-		return $OCF_NOT_RUNNING
-	    fi
-	    # FIXME: this looks for a matching LUN and path, but does
-	    # not actually test for the correct target ID.
-	    grep -E -q "[[:space:]]+lun:${OCF_RESKEY_lun}.*path:${OCF_RESKEY_path}$" /proc/net/iet/volume && return $OCF_SUCCESS
-	    ;;
+		# Figure out and set the target ID
+		TID=`sed -ne "s/tid:\([[:digit:]]\+\) name:${OCF_RESKEY_target_iqn}$/\1/p" < /proc/net/iet/volume`
+		if [ -z "${TID}" ]; then
+			# Our target is not configured, thus we're not
+			# running.
+			return $OCF_NOT_RUNNING
+		fi
+		# FIXME: this looks for a matching LUN and path, but does
+		# not actually test for the correct target ID.
+		grep -E -q "[[:space:]]+lun:${OCF_RESKEY_lun}.*path:${OCF_RESKEY_path}$" /proc/net/iet/volume && return $OCF_SUCCESS
+		;;
 	tgt)
-	    # Figure out and set the target ID
-	    TID=`tgtadm --lld iscsi --op show --mode target \
-		| sed -ne "s/^Target \([[:digit:]]\+\): ${OCF_RESKEY_target_iqn}$/\1/p"`
-	    if [ -z "$TID" ]; then
-		# Our target is not configured, thus we're not
-		# running.
-		return $OCF_NOT_RUNNING
-	    fi
-	    # This only looks for the backing store, but does not test
-	    # for the correct target ID and LUN.
-	    tgtadm --lld iscsi --op show --mode target \
-		| grep -E -q "[[:space:]]+Backing store.*: ${OCF_RESKEY_path}$" && return $OCF_SUCCESS
-	    ;;
+		# Figure out and set the target ID
+		TID=`tgtadm --lld iscsi --op show --mode target \
+			| sed -ne "s/^Target \([[:digit:]]\+\): ${OCF_RESKEY_target_iqn}$/\1/p"`
+		if [ -z "$TID" ]; then
+			# Our target is not configured, thus we're not
+			# running.
+			return $OCF_NOT_RUNNING
+		fi
+		# This only looks for the backing store, but does not test
+		# for the correct target ID and LUN.
+		tgtadm --lld iscsi --op show --mode target \
+			| grep -E -q "[[:space:]]+Backing store.*: ${OCF_RESKEY_path}$" && return $OCF_SUCCESS
+		;;
 	lio)
-	    configfs_path="/sys/kernel/config/target/iscsi/${OCF_RESKEY_target_iqn}/tpgt_1/lun/lun_${OCF_RESKEY_lun}/${OCF_RESOURCE_INSTANCE}/udev_path"
-	    [ -e ${configfs_path} ] && [ `cat ${configfs_path}` = "${OCF_RESKEY_path}" ] && return $OCF_SUCCESS
-	    ;;
-    esac
-    
-    return $OCF_NOT_RUNNING
+		configfs_path="/sys/kernel/config/target/iscsi/${OCF_RESKEY_target_iqn}/tpgt_1/lun/lun_${OCF_RESKEY_lun}/${OCF_RESOURCE_INSTANCE}/udev_path"
+		[ -e ${configfs_path} ] && [ `cat ${configfs_path}` = "${OCF_RESKEY_path}" ] && return $OCF_SUCCESS
+
+		# if we aren't activated, is a block device still left over?
+		block_configfs_path="/sys/kernel/config/target/core/iblock_${OCF_RESKEY_lio_iblock}/${OCF_RESOURCE_INSTANCE}/udev_path"
+		[ -e ${block_configfs_path} ] && ocf_log warn "existing block without an active lun: ${block_configfs_path}"
+		[ -e ${block_configfs_path} ] && return $OCF_ERR_GENERIC
+
+		;;
+	lio-t)
+		configfs_path="/sys/kernel/config/target/iscsi/${OCF_RESKEY_target_iqn}/tpgt_1/lun/lun_${OCF_RESKEY_lun}/*/udev_path"
+		[ -e ${configfs_path} ] && [ `cat ${configfs_path}` = "${OCF_RESKEY_path}" ] && return $OCF_SUCCESS
+		;;
+	esac
+
+	return $OCF_NOT_RUNNING
 }
 
 iSCSILogicalUnit_validate() {
-    # Do we have all required variables?
-    for var in target_iqn lun path; do
+	# Do we have all required variables?
+	for var in target_iqn lun path; do
 	param="OCF_RESKEY_${var}"
 	if [ -z "${!param}" ]; then
-	    ocf_log error "Missing resource parameter \"$var\"!"
-	    exit $OCF_ERR_CONFIGURED
+		ocf_exit_reason "Missing resource parameter \"$var\"!"
+		exit $OCF_ERR_CONFIGURED
 	fi
-    done
+	done
 
-    # Is the configured implementation supported?
-    case "$OCF_RESKEY_implementation" in
-	"iet"|"tgt"|"lio")
-	    ;;
+	# Is the configured implementation supported?
+	case "$OCF_RESKEY_implementation" in
+	"iet"|"tgt"|"lio"|"lio-t")
+		;;
 	"")
-	    # The user didn't specify an implementation, and we were
-	    # unable to determine one from installed binaries (in
-	    # other words: no binaries for any supported
-	    # implementation could be found)
-	    ocf_log error "Undefined iSCSI target implementation"
-	    exit $OCF_ERR_INSTALLED
-	    ;;
+		# The user didn't specify an implementation, and we were
+		# unable to determine one from installed binaries (in
+		# other words: no binaries for any supported
+		# implementation could be found)
+		ocf_exit_reason "Undefined iSCSI target implementation"
+		exit $OCF_ERR_INSTALLED
+		;;
 	*)
-	    ocf_log error "Unsupported iSCSI target implementation \"$OCF_RESKEY_implementation\"!"
-	    exit $OCF_ERR_CONFIGURED
-	    ;;
-    esac
+		ocf_exit_reason "Unsupported iSCSI target implementation \"$OCF_RESKEY_implementation\"!"
+		exit $OCF_ERR_CONFIGURED
+		;;
+	esac
 
-    # Do we have a valid LUN?
-    case $OCF_RESKEY_implementation in
+	# Do we have a valid LUN?
+	case $OCF_RESKEY_implementation in
 	iet)
-	    # IET allows LUN 0 and up
-	    [ $OCF_RESKEY_lun -ge 0 ]
-	    case $? in
+		# IET allows LUN 0 and up
+		[ $OCF_RESKEY_lun -ge 0 ]
+		case $? in
 		0)
-	            # OK
-		    ;;
+			# OK
+			;;
 		1)
-		    ocf_log err "Invalid LUN $OCF_RESKEY_lun (must be a non-negative integer)."
-		    exit $OCF_ERR_CONFIGURED
-		    ;;
+			ocf_log err "Invalid LUN $OCF_RESKEY_lun (must be a non-negative integer)."
+			exit $OCF_ERR_CONFIGURED
+			;;
 		*)
-		    ocf_log err "Invalid LUN $OCF_RESKEY_lun (must be an integer)."
-		    exit $OCF_ERR_CONFIGURED
-		    ;;
-	    esac
-	    ;;
+			ocf_log err "Invalid LUN $OCF_RESKEY_lun (must be an integer)."
+			exit $OCF_ERR_CONFIGURED
+			;;
+		esac
+		;;
 	tgt)
-	    # tgt reserves LUN 0 for its own purposes
-	    [ $OCF_RESKEY_lun -ge 1 ]
-	    case $? in
+		# tgt reserves LUN 0 for its own purposes
+		[ $OCF_RESKEY_lun -ge 1 ]
+		case $? in
 		0)
-	            # OK
-		    ;;
+				# OK
+			;;
 		1)
-		    ocf_log err "Invalid LUN $OCF_RESKEY_lun (must be greater than 0)."
-		    exit $OCF_ERR_CONFIGURED
-		    ;;
+			ocf_log err "Invalid LUN $OCF_RESKEY_lun (must be greater than 0)."
+			exit $OCF_ERR_CONFIGURED
+			;;
 		*)
-		    ocf_log err "Invalid LUN $OCF_RESKEY_lun (must be an integer)."
-		    exit $OCF_ERR_CONFIGURED
-		    ;;
-	    esac
-	    ;;
-    esac
-
-    # Do we have any configuration parameters that the current
-    # implementation does not support?
-    local unsupported_params
-    local var
-    local envar
-    case $OCF_RESKEY_implementation in
+			ocf_log err "Invalid LUN $OCF_RESKEY_lun (must be an integer)."
+			exit $OCF_ERR_CONFIGURED
+			;;
+		esac
+		;;
+	esac
+
+	# Do we have any configuration parameters that the current
+	# implementation does not support?
+	local unsupported_params
+	local var
+	local envar
+	case $OCF_RESKEY_implementation in
 	iet)
-	    # IET does not support setting the vendor and product ID
-	    # (it always uses "IET" and "VIRTUAL-DISK")
-	    unsupported_params="vendor_id product_id allowed_initiators lio_iblock tgt_bstype tgt_bsoflags tgt_device_type"
-	    ;;
+		# IET does not support setting the vendor and product ID
+		# (it always uses "IET" and "VIRTUAL-DISK")
+		unsupported_params="vendor_id product_id allowed_initiators lio_iblock tgt_bstype tgt_bsoflags tgt_device_type"
+		;;
 	tgt)
-	    unsupported_params="allowed_initiators lio_iblock"
-	    ;;
+		unsupported_params="allowed_initiators lio_iblock"
+		;;
 	lio)
-	    unsupported_params="scsi_id vendor_id product_id tgt_bstype tgt_bsoflags tgt_device_type"
-	    ;;
-    esac
-    for var in ${unsupported_params}; do
-	envar=OCF_RESKEY_${var}
-	defvar=OCF_RESKEY_${var}_default
-	if [ -n "${!envar}" ]; then
-	    if  [[ "${!envar}" != "${!defvar}" ]];then 
-	        case "$__OCF_ACTION" in
-                    start|validate-all)
-                      ocf_log warn "Configuration parameter \"${var}\"" \
-                       "is not supported by the iSCSI implementation" \
-                       "and will be ignored." ;;
-	        esac
-	    fi
-	fi
-    done
+		unsupported_params="scsi_id vendor_id product_id tgt_bstype tgt_bsoflags tgt_device_type"
+		;;
+	lio-t)
+		unsupported_params="scsi_id vendor_id product_id tgt_bstype tgt_bsoflags tgt_device_type lio_iblock"
+		;;
+	esac
+
+	for var in ${unsupported_params}; do
+		envar=OCF_RESKEY_${var}
+		defvar=OCF_RESKEY_${var}_default
+		if [ -n "${!envar}" ]; then
+			if  [[ "${!envar}" != "${!defvar}" ]];then 
+				case "$__OCF_ACTION" in
+				start|validate-all)
+					ocf_log warn "Configuration parameter \"${var}\"" \
+						"is not supported by the iSCSI implementation" \
+						"and will be ignored." ;;
+				esac
+			fi
+		fi
+	done
 
-    if ! ocf_is_probe; then
-    # Do we have all required binaries?
+	if ! ocf_is_probe; then
+	# Do we have all required binaries?
 	case $OCF_RESKEY_implementation in
-	    iet)
+	iet)
 		check_binary ietadm
 		;;
-	    tgt)
+	tgt)
 		check_binary tgtadm
 		;;
-	    lio)
+	lio)
 		check_binary tcm_node
 		check_binary lio_node
 		;;
+	lio-t)
+		check_binary targetcli
+		;;
 	esac
 
-        # Is the required kernel functionality available?
+	# Is the required kernel functionality available?
 	case $OCF_RESKEY_implementation in
-	    iet)
+	iet)
 		[ -d /proc/net/iet ]
 		if [ $? -ne 0 ]; then
-		    ocf_log err "/proc/net/iet does not exist or is not a directory -- check if required modules are loaded."
-		    exit $OCF_ERR_INSTALLED
+			ocf_log err "/proc/net/iet does not exist or is not a directory -- check if required modules are loaded."
+			exit $OCF_ERR_INSTALLED
 		fi
 		;;
-	    tgt)
-	        # tgt is userland only
+	tgt)
+		# tgt is userland only
 		;;
 	esac
-    fi
+	fi
 
-    return $OCF_SUCCESS
+	return $OCF_SUCCESS
 }
 
-
 case $1 in
-  meta-data)
+meta-data)
 	meta_data
 	exit $OCF_SUCCESS
 	;;
-  usage|help)
+usage|help)
 	iSCSILogicalUnit_usage
 	exit $OCF_SUCCESS
 	;;
@@ -568,13 +650,14 @@ start)		iSCSILogicalUnit_start;;
 stop)		iSCSILogicalUnit_stop;;
 monitor|status)	iSCSILogicalUnit_monitor;;
 reload)		ocf_log err "Reloading..."
-	        iSCSILogicalUnit_start
+			iSCSILogicalUnit_start
 		;;
 validate-all)	;;
 *)		iSCSILogicalUnit_usage
 		exit $OCF_ERR_UNIMPLEMENTED
 		;;
 esac
+
 rc=$?
 ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION : $rc"
 exit $rc
diff --git a/heartbeat/iSCSITarget b/heartbeat/iSCSITarget
index a988fb2..72ec64a 100755
--- a/heartbeat/iSCSITarget
+++ b/heartbeat/iSCSITarget
@@ -1,7 +1,7 @@
 #!/bin/bash
 #
 #
-# 	iSCSITarget OCF RA. Exports and manages iSCSI targets.
+#     iSCSITarget OCF RA. Exports and manages iSCSI targets.
 #
 #   (c) 2009-2010 Florian Haas, Dejan Muhamedagic,
 #                 and Linux-HA contributors
@@ -34,11 +34,13 @@
 # Defaults
 # Set a default implementation based on software installed
 if have_binary ietadm; then
-    OCF_RESKEY_implementation_default="iet"
+	OCF_RESKEY_implementation_default="iet"
 elif have_binary tgtadm; then
-    OCF_RESKEY_implementation_default="tgt"
+	OCF_RESKEY_implementation_default="tgt"
 elif have_binary lio_node; then
-    OCF_RESKEY_implementation_default="lio"
+	OCF_RESKEY_implementation_default="lio"
+elif have_binary targetcli; then
+	OCF_RESKEY_implementation_default="lio-t"
 fi
 : ${OCF_RESKEY_implementation=${OCF_RESKEY_implementation_default}}
 
@@ -67,12 +69,12 @@ Units (LUs) exported via a daemon that speaks the iSCSI protocol.
 <parameter name="implementation" required="0" unique="0">
 <longdesc lang="en">
 The iSCSI target daemon implementation. Must be one of "iet", "tgt",
-or "lio".  If unspecified, an implementation is selected based on the
+"lio", or "lio-t".  If unspecified, an implementation is selected based on the
 availability of management utilities, with "iet" being tried first,
-then "tgt", then "lio".
+then "tgt", then "lio", then "lio-t".
 </longdesc>
 <shortdesc lang="en">Specifies the iSCSI target implementation
-("iet", "tgt" or "lio").</shortdesc>
+("iet", "tgt", "lio", or "lio-t").</shortdesc>
 <content type="string" default="${OCF_RESKEY_implementation_default}"/>
 </parameter>
 
@@ -148,11 +150,11 @@ dependent. Neither the name nor the value may contain whitespace.
 </parameters>
 
 <actions>
-<action name="start"        timeout="10" />
-<action name="stop"         timeout="10" />
-<action name="status"       timeout="10" interval="10" depth="0" />
-<action name="monitor"      timeout="10" interval="10" depth="0" />
-<action name="meta-data"    timeout="5" />
+<action name="start"		timeout="10" />
+<action name="stop"		 timeout="10" />
+<action name="status"	   timeout="10" interval="10" depth="0" />
+<action name="monitor"	  timeout="10" interval="10" depth="0" />
+<action name="meta-data"	timeout="5" />
 <action name="validate-all"   timeout="10" />
 </actions>
 </resource-agent>
@@ -170,182 +172,233 @@ END
 }
 
 iSCSITarget_start() {
-    iSCSITarget_monitor
-    if [ $? =  $OCF_SUCCESS ]; then
-	return $OCF_SUCCESS
-    fi
+	iSCSITarget_monitor
+	if [ $? =  $OCF_SUCCESS ]; then
+		return $OCF_SUCCESS
+	fi
 
-    local param
-    local name
-    local value
-    local initiator
-    local portal
+	local param
+	local name
+	local value
+	local initiator
+	local portal
 
-    case $OCF_RESKEY_implementation in
+	case $OCF_RESKEY_implementation in
 	iet)
-	    local lasttid
-	    local tid
-	    if [ "${OCF_RESKEY_tid}" ]; then
-		tid="${OCF_RESKEY_tid}"
-	    else
-    	        # Figure out the last used target ID, add 1 to get the new
-	        # target ID.
-		ocf_take_lock $LOCKFILE
-		ocf_release_lock_on_exit $LOCKFILE
-		lasttid=`sed -ne "s/tid:\([[:digit:]]\+\) name:.*/\1/p" < /proc/net/iet/volume | sort -n | tail -n1`
-		[ -z "${lasttid}" ] && lasttid=0
-		tid=$((++lasttid))
-	    fi
-	    # Create the target.
-	    ocf_run ietadm --op new \
-		--tid=${tid} \
-		--params Name=${OCF_RESKEY_iqn} || exit $OCF_ERR_GENERIC
-	    # Set additional parameters.
-	    for param in ${OCF_RESKEY_additional_parameters}; do
-		name=${param%=*}
-		value=${param#*=}
-		ocf_run ietadm --op update \
-		    --tid=${tid} \
-		    --params ${name}=${value} || exit $OCF_ERR_GENERIC
-	    done
-	    # Legacy versions of IET allow targets by default, current
-	    # versions deny. To be safe we manage both the .allow and
-	    # .deny files.
-	    if [ -n "${OCF_RESKEY_allowed_initiators}" ]; then
-		echo "${OCF_RESKEY_iqn} ALL" >> /etc/initiators.deny
-		echo "${OCF_RESKEY_iqn} ${OCF_RESKEY_allowed_initiators// /,}" >> /etc/initiators.allow
-	    else
-		echo "${OCF_RESKEY_iqn} ALL" >> /etc/initiators.allow
-	    fi
-	    # In iet, adding a new user and assigning it to a target
-	    # is one operation.
-	    if [ -n "${OCF_RESKEY_incoming_username}" ]; then
-		ocf_run ietadm --op new --user \
-		    --tid=${tid} \
-		    --params=IncomingUser=${OCF_RESKEY_incoming_username},Password=${OCF_RESKEY_incoming_password} \
-		    || exit $OCF_ERR_GENERIC
-	    fi
-	    ;;
+		local lasttid
+		local tid
+		if [ "${OCF_RESKEY_tid}" ]; then
+			tid="${OCF_RESKEY_tid}"
+		else
+			# Figure out the last used target ID, add 1 to get the new
+			# target ID.
+			ocf_take_lock $LOCKFILE
+			ocf_release_lock_on_exit $LOCKFILE
+			lasttid=`sed -ne "s/tid:\([[:digit:]]\+\) name:.*/\1/p" < /proc/net/iet/volume | sort -n | tail -n1`
+			[ -z "${lasttid}" ] && lasttid=0
+			tid=$((++lasttid))
+		fi
+
+		# Create the target.
+		ocf_run ietadm --op new \
+			--tid=${tid} \
+			--params Name=${OCF_RESKEY_iqn} || exit $OCF_ERR_GENERIC
+
+		# Set additional parameters.
+		for param in ${OCF_RESKEY_additional_parameters}; do
+			name=${param%=*}
+			value=${param#*=}
+			ocf_run ietadm --op update \
+				--tid=${tid} \
+				--params ${name}=${value} || exit $OCF_ERR_GENERIC
+		done
+
+		# Legacy versions of IET allow targets by default, current
+		# versions deny. To be safe we manage both the .allow and
+		# .deny files.
+		if [ -n "${OCF_RESKEY_allowed_initiators}" ]; then
+			echo "${OCF_RESKEY_iqn} ALL" >> /etc/initiators.deny
+			echo "${OCF_RESKEY_iqn} ${OCF_RESKEY_allowed_initiators// /,}" >> /etc/initiators.allow
+		else
+			echo "${OCF_RESKEY_iqn} ALL" >> /etc/initiators.allow
+		fi
+		# In iet, adding a new user and assigning it to a target
+		# is one operation.
+		if [ -n "${OCF_RESKEY_incoming_username}" ]; then
+			ocf_run ietadm --op new --user \
+				--tid=${tid} \
+				--params=IncomingUser=${OCF_RESKEY_incoming_username},Password=${OCF_RESKEY_incoming_password} \
+				|| exit $OCF_ERR_GENERIC
+		fi
+		;;
 	tgt)
-	    local tid
-	    tid="${OCF_RESKEY_tid}"
-	    # Create the target.
-	    ocf_run tgtadm --lld iscsi --op new --mode target \
-		--tid=${tid} \
-		--targetname ${OCF_RESKEY_iqn} || exit $OCF_ERR_GENERIC
-	    # Set parameters.
-	    for param in ${OCF_RESKEY_additional_parameters}; do
-		name=${param%=*}
-		value=${param#*=}
-		ocf_run tgtadm --lld iscsi --op update --mode target \
-		    --tid=${tid} \
-		    --name=${name} --value=${value} || exit $OCF_ERR_GENERIC
-	    done
-	    # For tgt, we always have to add access per initiator;
-	    # access to targets is denied by default. If
-	    # "allowed_initiators" is unset, we must use the special
-	    # keyword ALL.
-	    for initiator in ${OCF_RESKEY_allowed_initiators=ALL}; do
-		ocf_run tgtadm --lld iscsi --op bind --mode target \
-		    --tid=${tid} \
-		    --initiator-address=${initiator} || exit $OCF_ERR_GENERIC
-	    done
-	    # In tgt, we must first create a user account, then assign
-	    # it to a target using the "bind" operation.
-	    if [ -n "${OCF_RESKEY_incoming_username}" ]; then
-		ocf_run tgtadm --lld iscsi --mode account --op new \
-		    --user=${OCF_RESKEY_incoming_username} \
-		    --password=${OCF_RESKEY_incoming_password} || exit $OCF_ERR_GENERIC
-		ocf_run tgtadm --lld iscsi --mode account --op bind \
-		    --tid=${tid} \
-		    --user=${OCF_RESKEY_incoming_username} || exit $OCF_ERR_GENERIC
-	    fi
-	    ;;
+		local tid
+		tid="${OCF_RESKEY_tid}"
+		# Create the target.
+		ocf_run tgtadm --lld iscsi --op new --mode target \
+			--tid=${tid} \
+			--targetname ${OCF_RESKEY_iqn} || exit $OCF_ERR_GENERIC
+
+		# Set parameters.
+		for param in ${OCF_RESKEY_additional_parameters}; do
+			name=${param%=*}
+			value=${param#*=}
+			ocf_run tgtadm --lld iscsi --op update --mode target \
+				--tid=${tid} \
+				--name=${name} --value=${value} || exit $OCF_ERR_GENERIC
+		done
+
+		# For tgt, we always have to add access per initiator;
+		# access to targets is denied by default. If
+		# "allowed_initiators" is unset, we must use the special
+		# keyword ALL.
+		for initiator in ${OCF_RESKEY_allowed_initiators=ALL}; do
+			ocf_run tgtadm --lld iscsi --op bind --mode target \
+				--tid=${tid} \
+				--initiator-address=${initiator} || exit $OCF_ERR_GENERIC
+		done
+
+		# In tgt, we must first create a user account, then assign
+		# it to a target using the "bind" operation.
+		if [ -n "${OCF_RESKEY_incoming_username}" ]; then
+			ocf_run tgtadm --lld iscsi --mode account --op new \
+				--user=${OCF_RESKEY_incoming_username} \
+				--password=${OCF_RESKEY_incoming_password} || exit $OCF_ERR_GENERIC
+			ocf_run tgtadm --lld iscsi --mode account --op bind \
+				--tid=${tid} \
+				--user=${OCF_RESKEY_incoming_username} || exit $OCF_ERR_GENERIC
+		fi
+		;;
 	lio)
-	    # lio distinguishes between targets and target portal
-	    # groups (TPGs). We will always create one TPG, with the
-	    # number 1. In lio, creating a network portal
-	    # automatically creates the corresponding target if it
-	    # doesn't already exist.
-	    for portal in ${OCF_RESKEY_portals}; do
-		ocf_run lio_node --addnp ${OCF_RESKEY_iqn} 1 \
-		    ${portal} || exit $OCF_ERR_GENERIC
-	    done
-	    # in lio, we can set target parameters by manipulating
-	    # the appropriate configfs entries
-	    for param in ${OCF_RESKEY_additional_parameters}; do
-		name=${param%=*}
-		value=${param#*=}
-		configfs_path="/sys/kernel/config/target/iscsi/${OCF_RESKEY_iqn}/tpgt_1/param/${name}"
-		if [ -e ${configfs_path} ]; then
-		    echo ${value} > ${configfs_path} || exit $OCF_ERR_GENERIC
+		# lio distinguishes between targets and target portal
+		# groups (TPGs). We will always create one TPG, with the
+		# number 1. In lio, creating a network portal
+		# automatically creates the corresponding target if it
+		# doesn't already exist.
+		for portal in ${OCF_RESKEY_portals}; do
+			ocf_run lio_node --addnp ${OCF_RESKEY_iqn} 1 \
+				${portal} || exit $OCF_ERR_GENERIC
+		done
+
+		# in lio, we can set target parameters by manipulating
+		# the appropriate configfs entries
+		for param in ${OCF_RESKEY_additional_parameters}; do
+			name=${param%=*}
+			value=${param#*=}
+			configfs_path="/sys/kernel/config/target/iscsi/${OCF_RESKEY_iqn}/tpgt_1/param/${name}"
+			if [ -e ${configfs_path} ]; then
+				echo ${value} > ${configfs_path} || exit $OCF_ERR_GENERIC
+			else
+				ocf_log warn "Unsupported iSCSI target parameter ${name}: will be ignored."
+			fi
+		done
+
+		# lio does per-initiator filtering by default. To disable
+		# this, we need to switch the target to "permissive mode".
+		if [ -n "${OCF_RESKEY_allowed_initiators}" ]; then
+			for initiator in ${OCF_RESKEY_allowed_initiators}; do
+				ocf_run lio_node --addnodeacl ${OCF_RESKEY_iqn} 1 \
+				${initiator} || exit $OCF_ERR_GENERIC
+			done
 		else
-		    ocf_log warn "Unsupported iSCSI target parameter ${name}: will be ignored."
+			ocf_run lio_node --permissive ${OCF_RESKEY_iqn} 1 || exit $OCF_ERR_GENERIC
+			# permissive mode enables read-only access by default,
+			# so we need to change that to RW to be in line with
+			# the other implementations.
+			echo 0 > "/sys/kernel/config/target/iscsi/${OCF_RESKEY_iqn}/tpgt_1/attrib/demo_mode_write_protect"
+			if [ `cat /sys/kernel/config/target/iscsi/${OCF_RESKEY_iqn}/tpgt_1/attrib/demo_mode_write_protect` -ne 0 ]; then
+				ocf_log err "Failed to disable write protection for target ${OCF_RESKEY_iqn}."
+				exit $OCF_ERR_GENERIC
+			fi
 		fi
-	    done
-	    # lio does per-initiator filtering by default. To disable
-	    # this, we need to switch the target to "permissive mode".
-	    if [ -n "${OCF_RESKEY_allowed_initiators}" ]; then
-		for initiator in ${OCF_RESKEY_allowed_initiators}; do
-		    ocf_run lio_node --addnodeacl ${OCF_RESKEY_iqn} 1 \
-			${initiator} || exit $OCF_ERR_GENERIC
+
+		# TODO: add CHAP authentication support when it gets added
+		# back into LIO
+		ocf_run lio_node --disableauth ${OCF_RESKEY_iqn} 1 || exit $OCF_ERR_GENERIC
+		# Finally, we need to enable the target to allow
+		# initiators to connect
+		ocf_run lio_node --enabletpg=${OCF_RESKEY_iqn} 1 || exit $OCF_ERR_GENERIC
+		;;
+	lio-t)
+		# lio distinguishes between targets and target portal
+		# groups (TPGs). We will always create one TPG, with the
+		# number 1. In lio, creating a network portal
+		# automatically creates the corresponding target if it
+		# doesn't already exist.
+		for portal in ${OCF_RESKEY_portals}; do
+			ocf_run targetcli /iscsi create ${OCF_RESKEY_iqn} || exit $OCF_ERR_GENERIC
+			if [ $portal != ${OCF_RESKEY_portals_default} ] ; then
+				IFS=':' read -a sep_portal <<< "$portal"
+				ocf_run targetcli /iscsi/${OCF_RESKEY_iqn}/tpg1/portals create "${sep_portal[0]}" "${sep_portal[1]}" || exit $OCF_ERR_GENERIC
+			fi
 		done
-	    else
-		ocf_run lio_node --permissive ${OCF_RESKEY_iqn} 1 || exit $OCF_ERR_GENERIC
-		# permissive mode enables read-only access by default,
-		# so we need to change that to RW to be in line with
-		# the other implementations.
-		echo 0 > "/sys/kernel/config/target/iscsi/${OCF_RESKEY_iqn}/tpgt_1/attrib/demo_mode_write_protect"
-		if [ `cat /sys/kernel/config/target/iscsi/${OCF_RESKEY_iqn}/tpgt_1/attrib/demo_mode_write_protect` -ne 0 ]; then
-		    ocf_log err "Failed to disable write protection for target ${OCF_RESKEY_iqn}."
-		    exit $OCF_ERR_GENERIC
+		# in lio, we can set target parameters by manipulating
+		# the appropriate configfs entries
+		for param in ${OCF_RESKEY_additional_parameters}; do
+			name=${param%=*}
+			value=${param#*=}
+			configfs_path="/sys/kernel/config/target/iscsi/${OCF_RESKEY_iqn}/tpgt_1/param/${name}"
+			if [ -e ${configfs_path} ]; then
+				echo ${value} > ${configfs_path} || exit $OCF_ERR_GENERIC
+			else
+				ocf_log warn "Unsupported iSCSI target parameter ${name}: will be ignored."
+			fi
+		done
+		# lio does per-initiator filtering by default. To disable
+		# this, we need to switch the target to "permissive mode".
+		if [ -n "${OCF_RESKEY_allowed_initiators}" ]; then
+			for initiator in ${OCF_RESKEY_allowed_initiators}; do
+				ocf_run targetcli /iscsi/${OCF_RESKEY_iqn}/tpg1/acls create ${initiator} || exit $OCF_ERR_GENERIC
+			done
+		else
+			ocf_run targetcli /iscsi/${OCF_RESKEY_iqn}/tpg1/ set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1 || exit $OCF_ERR_GENERIC
 		fi
-	    fi
-	    # TODO: add CHAP authentication support when it gets added
-	    # back into LIO
-	    ocf_run lio_node --disableauth ${OCF_RESKEY_iqn} 1 || exit $OCF_ERR_GENERIC
-	    # Finally, we need to enable the target to allow
-	    # initiators to connect
-	    ocf_run lio_node --enabletpg=${OCF_RESKEY_iqn} 1 || exit $OCF_ERR_GENERIC
-	    ;;
-    esac
-
-    return $OCF_SUCCESS
+		# TODO: add CHAP authentication support when it gets added
+		# back into LIO
+		ocf_run targetcli /iscsi/${OCF_RESKEY_iqn}/tpg1/ set attribute authentication=0 || exit $OCF_ERR_GENERIC
+#			   ocf_run targetcli /iscsi 
+		;;
+	esac
+
+	iSCSITarget_monitor
 }
 
 iSCSITarget_stop() {
-    iSCSITarget_monitor
-    if [ $? =  $OCF_SUCCESS ]; then
+	iSCSITarget_monitor
+	if [ $? -eq $OCF_NOT_RUNNING ]; then
+		return $OCF_SUCCESS
+	fi
+
 	local tid
 	case $OCF_RESKEY_implementation in
-	    iet)
+	iet)
 		# Figure out the target ID
 		tid=`sed -ne "s/tid:\([[:digit:]]\+\) name:${OCF_RESKEY_iqn}/\1/p" < /proc/net/iet/volume`
 		if [ -z "${tid}" ]; then
-		    ocf_log err "Failed to retrieve target ID for IQN ${OCF_RESKEY_iqn}"
-		    exit $OCF_ERR_GENERIC
+			ocf_log err "Failed to retrieve target ID for IQN ${OCF_RESKEY_iqn}"
+			exit $OCF_ERR_GENERIC
 		fi
 		# Close existing connections. There is no other way to
 		# do this in IET than to parse the contents of
 		# /proc/net/iet/session.
 		set -- $(sed -ne '/^tid:'${tid}' /,/^tid/ {
-                          /^[[:space:]]*sid:\([0-9]\+\)/ {
-                             s/^[[:space:]]*sid:\([0-9]*\).*/--sid=\1/; h;
-                          };
-                          /^[[:space:]]*cid:\([0-9]\+\)/ { 
-                              s/^[[:space:]]*cid:\([0-9]*\).*/--cid=\1/; G; p; 
-                          }; 
-                      }' < /proc/net/iet/session)
+						  /^[[:space:]]*sid:\([0-9]\+\)/ {
+							 s/^[[:space:]]*sid:\([0-9]*\).*/--sid=\1/; h;
+						  };
+						  /^[[:space:]]*cid:\([0-9]\+\)/ { 
+							  s/^[[:space:]]*cid:\([0-9]*\).*/--cid=\1/; G; p; 
+						  }; 
+					  }' < /proc/net/iet/session)
 		while [[ -n $2 ]]; do
-                    # $2 $1 looks like "--sid=X --cid=Y"
-		    ocf_run ietadm --op delete \
-			     --tid=${tid} $2 $1
-		    shift 2
+			# $2 $1 looks like "--sid=X --cid=Y"
+			ocf_run ietadm --op delete \
+				 --tid=${tid} $2 $1
+			shift 2
 		done
-   	        # In iet, unassigning a user from a target and
+			# In iet, unassigning a user from a target and
 		# deleting the user account is one operation.
 		if [ -n "${OCF_RESKEY_incoming_username}" ]; then
-		    ocf_run ietadm --op delete --user \
+			ocf_run ietadm --op delete --user \
 			--tid=${tid} \
 			--params=IncomingUser=${OCF_RESKEY_incoming_username} \
 			|| exit $OCF_ERR_GENERIC
@@ -353,216 +406,226 @@ iSCSITarget_stop() {
 		# Loop on delete. Keep trying until we time out, if
 		# necessary.
 		while true; do
-		    if ietadm --op delete --tid=${tid}; then
-			ocf_log debug "Removed target ${OCF_RESKEY_iqn}."
-			break
-		    else
-			ocf_log warn "Failed to remove target ${OCF_RESKEY_iqn}, retrying."
-			sleep 1
-		    fi
+			if ietadm --op delete --tid=${tid}; then
+				ocf_log debug "Removed target ${OCF_RESKEY_iqn}."
+				break
+			else
+				ocf_log warn "Failed to remove target ${OCF_RESKEY_iqn}, retrying."
+				sleep 1
+			fi
 		done
 		# Avoid stale /etc/initiators.{allow,deny} entries
 		# for this target
 		if [ -e /etc/initiators.deny ]; then
-		    ocf_run sed -e "/^${OCF_RESKEY_iqn}[[:space:]]/d" \
+			ocf_run sed -e "/^${OCF_RESKEY_iqn}[[:space:]]/d" \
 			-i /etc/initiators.deny
 		fi
 		if [ -e /etc/initiators.allow ]; then
-		    ocf_run sed -e "/^${OCF_RESKEY_iqn}[[:space:]]/d" \
+			ocf_run sed -e "/^${OCF_RESKEY_iqn}[[:space:]]/d" \
 			-i /etc/initiators.allow
 		fi
 		;;
-	    tgt)
+	tgt)
 		tid="${OCF_RESKEY_tid}"
 		# Close existing connections. There is no other way to
 		# do this in tgt than to parse the output of "tgtadm --op
 		# show".
 		set -- $(tgtadm --lld iscsi --op show --mode target \
-		    | sed -ne '/^Target '${tid}':/,/^Target/ {
-                          /^[[:space:]]*I_T nexus: \([0-9]\+\)/ {
-                             s/^.*: \([0-9]*\).*/--sid=\1/; h;
-                          };
-                          /^[[:space:]]*Connection: \([0-9]\+\)/ { 
-                              s/^.*: \([0-9]*\).*/--cid=\1/; G; p; 
-                          }; 
-                          /^[[:space:]]*LUN information:/ q; 
-                      }')
+			| sed -ne '/^Target '${tid}':/,/^Target/ {
+						  /^[[:space:]]*I_T nexus: \([0-9]\+\)/ {
+							 s/^.*: \([0-9]*\).*/--sid=\1/; h;
+						  };
+						  /^[[:space:]]*Connection: \([0-9]\+\)/ { 
+							  s/^.*: \([0-9]*\).*/--cid=\1/; G; p; 
+						  }; 
+						  /^[[:space:]]*LUN information:/ q; 
+					  }')
 		while [[ -n $2 ]]; do
-                    # $2 $1 looks like "--sid=X --cid=Y"
-		    ocf_run tgtadm --lld iscsi --op delete --mode connection \
+			# $2 $1 looks like "--sid=X --cid=Y"
+			ocf_run tgtadm --lld iscsi --op delete --mode connection \
 			--tid=${tid} $2 $1
-		    shift 2
+			shift 2
 		done
-	        # In tgt, we must first unbind the user account from
+			# In tgt, we must first unbind the user account from
 		# the target, then remove the account itself.
 		if [ -n "${OCF_RESKEY_incoming_username}" ]; then
-		    ocf_run tgtadm --lld iscsi --mode account --op unbind \
-			--tid=${tid} \
-			--user=${OCF_RESKEY_incoming_username} || exit $OCF_ERR_GENERIC
-		    ocf_run tgtadm --lld iscsi --mode account --op delete \
-			--user=${OCF_RESKEY_incoming_username} || exit $OCF_ERR_GENERIC
+			ocf_run tgtadm --lld iscsi --mode account --op unbind \
+				--tid=${tid} \
+				--user=${OCF_RESKEY_incoming_username} || exit $OCF_ERR_GENERIC
+			ocf_run tgtadm --lld iscsi --mode account --op delete \
+				--user=${OCF_RESKEY_incoming_username} || exit $OCF_ERR_GENERIC
 		fi
 		# Loop on delete. Keep trying until we time out, if
 		# necessary.
 		while true; do
-		    if tgtadm --lld iscsi --op delete --mode target --tid=${tid}; then
-			ocf_log debug "Removed target ${OCF_RESKEY_iqn}."
-			break
-		    else
-			ocf_log warn "Failed to remove target ${OCF_RESKEY_iqn}, retrying."
-			sleep 1
-		    fi
+			if tgtadm --lld iscsi --op delete --mode target --tid=${tid}; then
+				ocf_log debug "Removed target ${OCF_RESKEY_iqn}."
+				break
+			else
+				ocf_log warn "Failed to remove target ${OCF_RESKEY_iqn}, retrying."
+				sleep 1
+			fi
 		done
 		# In tgt, we don't have to worry about our ACL
 		# entries. They are automatically removed upon target
 		# deletion.
 		;;
-	    lio)
+	lio)
 		# In lio, removing a target automatically removes all
 		# associated TPGs, network portals, and LUNs.
 		ocf_run lio_node --deliqn ${OCF_RESKEY_iqn} || exit $OCF_ERR_GENERIC
 		;;
+	lio-t)
+		ocf_run targetcli /iscsi delete ${OCF_RESKEY_iqn} || exit $OCF_ERR_GENERIC
+		;;
 	esac
-    fi
 
-    return $OCF_SUCCESS
+	return $OCF_SUCCESS
 }
 
 iSCSITarget_monitor() {
-    case $OCF_RESKEY_implementation in
+	case $OCF_RESKEY_implementation in
 	iet)
-	    grep -Eq "tid:[0-9]+ name:${OCF_RESKEY_iqn}" /proc/net/iet/volume && return $OCF_SUCCESS
-	    ;;
+		grep -Eq "tid:[0-9]+ name:${OCF_RESKEY_iqn}" /proc/net/iet/volume && return $OCF_SUCCESS
+		;;
 	tgt)
-	    tgtadm --lld iscsi --op show --mode target \
+		tgtadm --lld iscsi --op show --mode target \
 		| grep -Eq "Target [0-9]+: ${OCF_RESKEY_iqn}" && return $OCF_SUCCESS
-	    ;;
-	lio)
-	    # if we have no configfs entry for the target, it's
-	    # definitely stopped
-	    [ -d /sys/kernel/config/target/iscsi/${OCF_RESKEY_iqn} ] || return $OCF_NOT_RUNNING
-	    # if the target is there, but its TPG is not enabled, then
-	    # we also consider it stopped
-	    [ `cat /sys/kernel/config/target/iscsi/${OCF_RESKEY_iqn}/tpgt_1/enable` -eq 1 ] || return $OCF_NOT_RUNNING
-	    return $OCF_SUCCESS
-	    ;;
-    esac
-    
-    return $OCF_NOT_RUNNING
+		;;
+	lio | lio-t)
+		# if we have no configfs entry for the target, it's
+		# definitely stopped
+		[ -d /sys/kernel/config/target/iscsi/${OCF_RESKEY_iqn} ] || return $OCF_NOT_RUNNING
+		# if the target is there, but its TPG is not enabled, then
+		# we also consider it stopped
+		[ `cat /sys/kernel/config/target/iscsi/${OCF_RESKEY_iqn}/tpgt_1/enable` -eq 1 ] || return $OCF_NOT_RUNNING
+		return $OCF_SUCCESS
+		;;
+	esac
+	
+	return $OCF_NOT_RUNNING
 }
 
 iSCSITarget_validate() {
-    # Do we have all required variables?
-    local required_vars
-    case $OCF_RESKEY_implementation in
+	# Do we have all required variables?
+	local required_vars
+	case $OCF_RESKEY_implementation in
 	iet)
-	    required_vars="iqn"
-	    ;;
+		required_vars="iqn"
+		;;
 	tgt)
-	    required_vars="iqn tid"
-	    ;;
-    esac
-    for var in ${required_vars}; do
-	param="OCF_RESKEY_${var}"
-	if [ -z "${!param}" ]; then
-	    ocf_log error "Missing resource parameter \"$var\"!"
-	    exit $OCF_ERR_CONFIGURED
-	fi
-    done
+		required_vars="iqn tid"
+		;;
+	esac
+
+	for var in ${required_vars}; do
+		param="OCF_RESKEY_${var}"
+		if [ -z "${!param}" ]; then
+			ocf_exit_reason "Missing resource parameter \"$var\"!"
+			exit $OCF_ERR_CONFIGURED
+		fi
+	done
 
-    # Is the configured implementation supported?
-    case "$OCF_RESKEY_implementation" in
-	"iet"|"tgt"|"lio")
-	    ;;
+	# Is the configured implementation supported?
+	case "$OCF_RESKEY_implementation" in
+	"iet"|"tgt"|"lio"|"lio-t")
+		;;
 	"")
-	    # The user didn't specify an implementation, and we were
-	    # unable to determine one from installed binaries (in
-	    # other words: no binaries for any supported
-	    # implementation could be found)
-	    ocf_log error "Undefined iSCSI target implementation"
-	    exit $OCF_ERR_INSTALLED
-	    ;;
+		# The user didn't specify an implementation, and we were
+		# unable to determine one from installed binaries (in
+		# other words: no binaries for any supported
+		# implementation could be found)
+		ocf_exit_reason "Undefined iSCSI target implementation"
+		exit $OCF_ERR_INSTALLED
+		;;
 	*)
-	    ocf_log error "Unsupported iSCSI target implementation \"$OCF_RESKEY_implementation\"!"
-	    exit $OCF_ERR_CONFIGURED
-	    ;;
-    esac
-
-    # Do we have any configuration parameters that the current
-    # implementation does not support?
-    local unsupported_params
-    local var
-    local envar
-    case $OCF_RESKEY_implementation in
+		ocf_exit_reason "Unsupported iSCSI target implementation \"$OCF_RESKEY_implementation\"!"
+		exit $OCF_ERR_CONFIGURED
+		;;
+	esac
+
+	# Do we have any configuration parameters that the current
+	# implementation does not support?
+	local unsupported_params
+	local var
+	local envar
+	case $OCF_RESKEY_implementation in
 	iet|tgt)
-	    # IET and tgt do not support binding a target portal to a
-	    # specific IP address.
-	    unsupported_params="portals"
-	    ;;
-	lio)
-	    # TODO: Remove incoming_username and incoming_password
-	    # from this check when LIO 3.0 gets CHAP authentication
-	    unsupported_params="tid incoming_username incoming_password"
-	    ;;
-    esac
-    for var in ${unsupported_params}; do
-	envar=OCF_RESKEY_${var}
-	defvar=OCF_RESKEY_${var}_default
-	if [ -n "${!envar}" ]; then
-            if  [[ "${!envar}" != "${!defvar}" ]];then
-                    case "$__OCF_ACTION" in
-                        start|validate-all)
-                          ocf_log warn "Configuration parameter \"${var}\"" \
-                            "is not supported by the iSCSI implementation" \
-                            "and will be ignored." ;;
-                    esac
-            fi
-	fi
-    done
+		# IET and tgt do not support binding a target portal to a
+		# specific IP address.
+		unsupported_params="portals"
+		;;
+	lio|lio-t)
+		# TODO: Remove incoming_username and incoming_password
+		# from this check when LIO 3.0 gets CHAP authentication
+		unsupported_params="tid incoming_username incoming_password"
+		;;
+	esac
 
-    if ! ocf_is_probe; then
-        # Do we have all required binaries?
+	for var in ${unsupported_params}; do
+		envar=OCF_RESKEY_${var}
+		defvar=OCF_RESKEY_${var}_default
+		if [ -n "${!envar}" ]; then
+			if  [[ "${!envar}" != "${!defvar}" ]];then
+				case "$__OCF_ACTION" in
+					start|validate-all)
+						ocf_log warn "Configuration parameter \"${var}\"" \
+							"is not supported by the iSCSI implementation" \
+							"and will be ignored." ;;
+				esac
+			fi
+		fi
+	done
+
+	if ! ocf_is_probe; then
+		# Do we have all required binaries?
 	case $OCF_RESKEY_implementation in
-	    iet)
+	iet)
 		check_binary ietadm
 		;;
-	    tgt)
+	tgt)
 		check_binary tgtadm
 		;;
-	    lio)
+	lio)
 		check_binary tcm_node
 		check_binary lio_node
 		;;
+	lio-t)
+		check_binary targetcli
+		;;
 	esac
 
-        # Is the required kernel functionality available?
+	# Is the required kernel functionality available?
 	case $OCF_RESKEY_implementation in
-	    iet)
+	iet)
 		[ -d /proc/net/iet ]
 		if [ $? -ne 0 ]; then
-		    ocf_log err "/proc/net/iet does not exist or is not a directory -- check if required modules are loaded."
-		    exit $OCF_ERR_INSTALLED
+			ocf_log err "/proc/net/iet does not exist or is not a directory -- check if required modules are loaded."
+			exit $OCF_ERR_INSTALLED
 		fi
 		;;
-	    tgt)
-	        # tgt is userland only
+	tgt)
+			# tgt is userland only
 		;;
-	    lio)
-	        # lio needs configfs to be mounted
+	lio)
+			# lio needs configfs to be mounted
 		if ! grep -Eq "^.*/sys/kernel/config[[:space:]]+configfs" /proc/mounts; then
-		    ocf_log err "configfs not mounted at /sys/kernel/config -- check if required modules are loaded."
-		    exit $OCF_ERR_INSTALLED
+			ocf_log err "configfs not mounted at /sys/kernel/config -- check if required modules are loaded."
+			exit $OCF_ERR_INSTALLED
 		fi
-	        # check for configfs entries created by target_core_mod
+			# check for configfs entries created by target_core_mod
 		if [ ! -d /sys/kernel/config/target ]; then
-		    ocf_log err "/sys/kernel/config/target does not exist or is not a directory -- check if required modules are loaded."
-		    exit $OCF_ERR_INSTALLED
+			ocf_log err "/sys/kernel/config/target does not exist or is not a directory -- check if required modules are loaded."
+			exit $OCF_ERR_INSTALLED
 		fi
 		;;
+	lio-t)
+		#targetcli loads the needed kernel modules
+		;;
 	esac
-    fi
+	fi
 
-    return $OCF_SUCCESS
+	return $OCF_SUCCESS
 }
 
 
@@ -585,7 +648,7 @@ start)		iSCSITarget_start;;
 stop)		iSCSITarget_stop;;
 monitor|status)	iSCSITarget_monitor;;
 reload)		ocf_log err "Reloading..."
-	        iSCSITarget_start
+			iSCSITarget_start
 		;;
 validate-all)	;;
 *)		iSCSITarget_usage
diff --git a/heartbeat/iscsi b/heartbeat/iscsi
index de3f4c0..36ed908 100755
--- a/heartbeat/iscsi
+++ b/heartbeat/iscsi
@@ -179,7 +179,7 @@ open_iscsi_setup() {
 	discovery=open_iscsi_discovery
 	add_disk=open_iscsi_add
 	remove_disk=open_iscsi_remove
-	disk_status=open_iscsi_status
+	disk_status=open_iscsi_monitor
 	iscsiadm=${OCF_RESKEY_iscsiadm}
 
 	have_binary ${iscsiadm} ||
@@ -268,7 +268,8 @@ open_iscsi_add() {
 }
 open_iscsi_get_session_id() {
 	local target="$1"
-	$iscsiadm -m session 2>/dev/null | grep "$target$" |
+	$iscsiadm -m session 2>/dev/null |
+		grep -E "$target($|[[:space:]])" |
 		awk '{print $2}' | tr -d '[]'
 }
 open_iscsi_remove() {
@@ -282,7 +283,13 @@ open_iscsi_remove() {
 		return 1
 	fi
 }
-open_iscsi_status() {
+# open_iscsi_monitor return codes:
+#   0: target running (logged in)
+#   1: target not running and target record exists
+#   2: iscsiadm -m session error (unexpected)
+#   3: target record does not exist (discovery necessary)
+#
+open_iscsi_monitor() {
 	local target="$1"
 	local session_id conn_state outp
 	local prev_state
@@ -291,8 +298,13 @@ open_iscsi_status() {
 	recov=${2:-$OCF_RESKEY_try_recovery}
 	session_id=`open_iscsi_get_session_id "$target"`
 	prev_state=""
-	[ -z "$session_id" ] &&
-		return 1
+	if [ -z "$session_id" ]; then
+		if $iscsiadm -m node -p $OCF_RESKEY_portal -T $target >/dev/null 2>&1; then
+			return 1 # record found
+		else
+			return 3
+		fi
+	fi
 	while :; do
 		outp=`$iscsiadm -m session -r $session_id -P 1` ||
 			return 2
@@ -325,6 +337,7 @@ open_iscsi_status() {
 }
 
 disk_discovery() {
+	discovery_type=${OCF_RESKEY_discovery_type}
 	$discovery  # discover and setup the real portal string (address)
 	case $? in
 	0) ;;
@@ -350,22 +363,29 @@ wait_for_udev() {
 		sleep 1
 	done
 }
-iscsi_status() {
+iscsi_monitor() {
 	$disk_status $OCF_RESKEY_target $*
 	case $? in
 		0) return $OCF_SUCCESS;;
-		1) return $OCF_NOT_RUNNING;;
+		1|3) return $OCF_NOT_RUNNING;;
 		2) return $OCF_ERR_GENERIC;;
 	esac
 }
 iscsi_start() {
-	iscsi_status
-	case $? in
-	$OCF_SUCCESS)
+	local rc
+	$disk_status $OCF_RESKEY_target
+	rc=$?
+	if [ $rc -eq 3 ]; then
+		disk_discovery
+		$disk_status $OCF_RESKEY_target
+		rc=$?
+	fi
+	case $rc in
+	0)
 		ocf_log info "iscsi $PORTAL $OCF_RESKEY_target already running"
 		return $OCF_SUCCESS
 	;;
-	$OCF_NOT_RUNNING)
+	1)
 		$add_disk $PORTAL $OCF_RESKEY_target ||
 			return $OCF_ERR_GENERIC
 		case "$OCF_RESKEY_udev" in
@@ -379,7 +399,7 @@ iscsi_start() {
 		ocf_log warning "iscsi $PORTAL $OCF_RESKEY_target in failed state"
 	;;
 	esac
-	iscsi_status 1 # enforce wait
+	iscsi_monitor 1 # enforce wait
 	if [ $? -eq $OCF_SUCCESS ]; then
 		return $OCF_SUCCESS
 	else
@@ -387,11 +407,11 @@ iscsi_start() {
 	fi
 }
 iscsi_stop() {
-	iscsi_status
+	iscsi_monitor
 	if [ $? -ne $OCF_NOT_RUNNING ] ; then
 		$remove_disk $OCF_RESKEY_target ||
 			return $OCF_ERR_GENERIC
-		iscsi_status
+		iscsi_monitor
 		if [ $? -ne $OCF_NOT_RUNNING ] ; then
 			return $OCF_ERR_GENERIC
 		else
@@ -403,14 +423,6 @@ iscsi_stop() {
 	fi
 }
 
-iscsi_monitor() {
-	if $disk_status $OCF_RESKEY_target; then
-		return $OCF_SUCCESS
-	else
-		return $OCF_NOT_RUNNING
-	fi
-}
-
 #
 #	'main' starts here...
 #
@@ -448,6 +460,7 @@ Linux) setup=open_iscsi_setup
 ;;
 esac
 
+PORTAL="$OCF_RESKEY_portal" # updated by discovery
 LSB_STATUS_STOPPED=3
 $setup
 setup_rc=$?
@@ -469,13 +482,11 @@ fi
 # which method was invoked?
 case "$1" in
 	start)
-		discovery_type=${OCF_RESKEY_discovery_type}
-		disk_discovery
 		iscsi_start
 	;;
 	stop)	iscsi_stop
 	;;
-	status)	iscsi_status
+	status)	iscsi_monitor
 		rc=$?
 		case $rc in
 		$OCF_SUCCESS)
@@ -490,7 +501,7 @@ case "$1" in
 		esac
 		exit $rc
 		;;
-	monitor)	iscsi_status
+	monitor)	iscsi_monitor
 	;;
 	validate-all)	# everything already validated
 		# just exit successfully here.
diff --git a/heartbeat/kamailio b/heartbeat/kamailio
new file mode 100755
index 0000000..e200d8c
--- /dev/null
+++ b/heartbeat/kamailio
@@ -0,0 +1,688 @@
+#!/bin/bash
+#
+#       OCF resource agent for Kamailio for pacemaker
+#
+
+# Copyright (c) 2013 FREQUENTIS AG,
+#                    Authors: Stefan Wenk
+#                             Rainer Brestan
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of version 2 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# Further, this software is distributed without any warranty that it is
+# free of the rightful claim of any third person regarding infringement
+# or the like.  Any license provided herein, whether implied or
+# otherwise, applies only to this software file.  Patent licenses, if
+# any, provided herein do not apply to combinations of this program with
+# other software, or any other product whatsoever.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
+#
+
+# OCF input parameters:
+#  OCF_RESKEY_binary
+#  OCF_RESKEY_conffile
+#  OCF_RESKEY_pidfile
+#  OCF_RESKEY_monitoring_ip
+#  OCF_RESKEY_listen_address
+#  OCF_RESKEY_port
+#  OCF_RESKEY_proto
+#  OCF_RESKEY_sipsak
+#  OCF_RESKEY_kamctlrc
+#  OCF_RESKEY_kamuser
+
+# Initialization:
+
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+
+#######################################################################
+# Defaults
+
+RESKEY_binary_default="/usr/sbin/kamailio"
+RESKEY_conffile_default="/etc/kamailio/kamailio.cfg"
+RESKEY_pidfile_default="/var/run/kamailio_${OCF_RESOURCE_INSTANCE}/kamailio.pid"
+RESKEY_monitoring_ip_default=127.0.0.1
+RESKEY_port_default=5060
+RESKEY_proto_default="udptcp"
+RESKEY_sipsak_default="/usr/bin/sipsak"
+RESKEY_kamctlrc_default="/etc/kamailio/kamctlrc"
+RESKEY_kamuser_default=""
+
+#######################################################################
+: ${OCF_RESKEY_binary=${RESKEY_binary_default}}
+: ${OCF_RESKEY_conffile=${RESKEY_conffile_default}}
+: ${OCF_RESKEY_pidfile=${RESKEY_pidfile_default}}
+: ${OCF_RESKEY_monitoring_ip=${RESKEY_monitoring_ip_default}}
+: ${OCF_RESKEY_port=${RESKEY_port_default}}
+: ${OCF_RESKEY_proto=${RESKEY_proto_default}}
+: ${OCF_RESKEY_sipsak=${RESKEY_sipsak_default}}
+: ${OCF_RESKEY_kamctlrc=${RESKEY_kamctlrc_default}}
+: ${OCF_RESKEY_kamuser=${RESKEY_kamuser_default}}
+
+#######################################################################
+usage() {
+	cat <<END
+usage: $0 {start|stop|status|monitor|validate-all|meta-data}
+Expects to have a fully populated OCF RA-compliant environment set.
+END
+}
+
+meta_data() {
+	cat <<END
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="kamailio" version="1.0">
+<version>1.0</version>
+
+<longdesc lang="en">
+ Resource agent for the Kamailio SIP proxy/registrar.
+ Multiple instances are possible when using following parameter combinations:
+
+ Parameters for Kamailio instance 1:
+    listen_address=192.168.159.128
+    monitoring_ip=192.168.159.128
+    proto=udptcp
+    port=5060 
+
+ Parameters for Kamailio instance 2:
+    listen_address=192.168.159.128
+    monitoring_ip=192.168.159.128
+    proto=udp
+    port=5070
+    conffile=/etc/kamailio/kamailio2.cfg
+    kamctlrc=""
+
+ Only one instance can be monitored via the command "kamctl monitor"
+ because the kamctl tool of kamailio 4.x is not designed for multiple
+ instances. Therefore, the provided kamctrlrc file path needs to be 
+ empty for instance 2, 3 ...
+
+Parameters for a third Kamailio instance:
+    listen_address=192.168.159.128
+    monitoring_ip=192.168.159.128
+    proto=tcp
+    port=5080
+    conffile=/etc/kamailio/kamailio3.cfg
+    kamctlrc=""
+</longdesc>
+
+<shortdesc lang="en">Resource agent for Kamailio</shortdesc>
+
+<parameters>
+  <parameter name="binary" unique="0" required="0">
+   <longdesc lang="en">The kamailio binary</longdesc>
+   <shortdesc lang="en">The kamailio  binary</shortdesc>
+   <content type="string" default="${RESKEY_binary_default}" />
+  </parameter>
+
+  <parameter name="conffile" unique="0" required="0">
+   <longdesc lang="en">
+    The kamailio configuration file name with full path. 
+    For example, "/etc/kamailio/kamailio.cfg" , which is the default value.
+    Make sure to use unique names in case of having multiple instances.
+   </longdesc>
+   <shortdesc lang="en">Configuration file name with full path</shortdesc>
+   <content type="string" default="${RESKEY_conffile_default}" />
+  </parameter>
+
+  <parameter name="pidfile" unique="0" required="0">
+   <longdesc lang="en">
+     The kamailio PID file. The directory used must be writable by kamailio
+     process user. Be sure to use unique name for running more than one
+     instance. Try to use absolute path names.
+     If empty, resource agent create a unique directory from the resource
+     instance name for the PID file and assign it to the process user.
+   </longdesc>
+   <shortdesc lang="en">PID file</shortdesc>
+   <content type="string" default="${RESKEY_pidfile_default}" />
+  </parameter>
+
+  <parameter name="monitoring_ip" unique="0" required="0">
+  <longdesc lang="en">
+   SIP IP Address of the kamailio instance used for SIP OPTIONS polling monitoring.
+   Usually the same IP address value as for parameter listen_address should be
+   provided.
+
+   In order to respond with a 200 OK response to the SIP OOPTION requests, 
+   the kamailio.cfg file needs to contain following section:
+   Note: The following "kamailio.cfg" code sniplet is part of an XML section. 
+         Therefore it contains two & characters, which need to be replaced 
+         with two ampersand characters within "kamailio.cfg":
+
+   if (is_method("OPTIONS") && ($ru=~"sip:monitor at .*")) {
+        ##
+        ## If the method is an OPTIONS we are simply going to respond
+        ## with a 200 OK.  
+        # xlog("L_INFO", "Method is an OPTIONS, probably just monitoring\n");   
+        sl_send_reply("200", "Kamailio is alive");
+        exit;
+    }
+
+  </longdesc>
+  <shortdesc lang="en">Monitoring IP address used for SIP OPTIONS polling.</shortdesc>
+   <content type="string" default="${RESKEY_monitoring_ip_default}" />
+  </parameter>
+
+  <parameter name="listen_address" unique="0" required="1">
+  <longdesc lang="en">
+    SIP IP address the kamailio will listen on.
+  </longdesc>
+  <shortdesc lang="en">Listening SIP address</shortdesc>
+  <content type="string" />
+  </parameter>
+
+  <parameter name="port" unique="0" required="0">
+   <longdesc lang="en">
+    SIP port for the kamailio instance.
+   </longdesc>
+   <shortdesc lang="en">SIP Port</shortdesc>
+   <content type="string" default="${RESKEY_port_default}" />
+  </parameter>
+
+  <parameter name="proto" unique="0" required="0">
+   <longdesc lang="en">
+    The protocol used for SIP proto  =  udp|tcp|udptcp. 
+   </longdesc>
+   <shortdesc lang="en">protocol</shortdesc>
+   <content type="string" default="${RESKEY_proto_default}" />
+  </parameter>
+
+  <parameter name="sipsak" unique="0" required="0">
+   <longdesc lang="en">
+    The installation path of the sipsak tool, which is used 
+    for monitoring Kamailio via SIP OPTIONS polling. 
+   </longdesc>
+   <shortdesc lang="en">protocol</shortdesc>
+   <content type="string" default="${RESKEY_sipsak_default}" />
+  </parameter>
+
+  <parameter name="kamctlrc" unique="0" required="0">
+   <longdesc lang="en">
+    The location of the "kamctlrc" file for the Kamailio instance.
+    The file "kamctlrc" is the Kamailio configuration file for its "kamctl" control tool.
+
+    This parameter only needs to be provided in case of using multiple Kamailio server
+    instances on a single cluster node:
+
+    In case that the parameter "kamctlrc" is not empty, this ressource agent monitors 
+    the health state of the Kamailio server via the command "kamctl monitor 1". This 
+    setting is recommended in case of using a single Kamailio server instance.
+
+    In case that the parameter "kamctlrc" is empty, the ressource agent does not 
+    monitor the health state of the Kamailio server instance via the "kamctl" command.
+
+    Please note that the "kamctl" control command of Kamailio 4.x does not support 
+    running multiple Kamailio instances on one host. Nevertheless this resource agent 
+    does allow multiple Kamailio instances per host. The result of the "kamctl" 
+    limitation in terms of number of Kamailio server instances is that the health 
+    check via "kamctl monitor 1" can be configured for a single Kamailio instance
+    only.
+
+    Please refer to the long description of this resoure agent for an example
+    of parameter combinations in case that multiple instances are to be 
+    configured per cluster node.
+
+   </longdesc>
+   <shortdesc lang="en">protocol</shortdesc>
+   <content type="string" default="${RESKEY_kamctlrc_default}" />
+  </parameter>
+
+  <parameter name="kamuser" unique="0" required="0">
+   <longdesc lang="en">
+    The user account for kamailio process to run with.
+    Uses the current user, if not specified or empty.
+    There is no check, if running kamailio with the specified user account is possible.
+   </longdesc>
+   <shortdesc lang="en">kamailio user</shortdesc>
+   <content type="string" default="${RESKEY_kamuser_default}" />
+  </parameter>
+</parameters>
+
+ <actions>
+  <action name="start" timeout="60" />
+  <action name="stop" timeout="30" />
+  <action name="status" timeout="30" interval="10" />
+  <action name="monitor" timeout="30" interval="10" />
+  <action name="meta-data" timeout="5" />
+  <action name="validate-all" timeout="5" />
+  <action name="notify" timeout="5" />
+ </actions>
+</resource-agent>
+END
+
+    exit $OCF_SUCCESS
+}
+
+#######################################################################
+
+###
+#Check if a process with given PID is running
+# Parameter 1: PID
+###
+isRunning_PID()
+{
+    kill -s 0 "$1" > /dev/null 2>&1
+}
+
+###
+#Check if an instance with given command line is running
+# Parameter 1: command line.
+###
+isRunning_cmd()
+{
+    pkill -s 0 "$1" > /dev/null 2>&1
+}
+
+###
+# Formats the result of a command.
+#
+# Parameter 1: Exit status.
+# Parameter 2: Standard output (stdout).
+# Parameter 3: Error output (stderr).
+#     Returns: Formatted result.
+kamailio_format_result() {
+    local exitstatus="$1"
+    local value="$2"
+    local error="$3"
+
+    echo -n "exit status: ${exitstatus}"
+    if [ -n "$value" ]; then
+        echo -n ", value: ${value}"
+    fi
+
+    if [ -n "$error" ]; then
+        echo -n ", error: ${error}"
+    fi
+    echo
+}
+
+###
+# Put the command line, how the kamailio process is started according 
+# to the configured parameters, into the variable "kam_cmd".
+###
+
+kamailio_cmd()
+{
+    case ${OCF_RESKEY_proto} in
+    udp)  listen_param="-T -l udp:${OCF_RESKEY_listen_address}:${OCF_RESKEY_port} -l udp:127.0.0.1:${OCF_RESKEY_port}"
+           ;;
+    tcp)  listen_param="-l tcp:${OCF_RESKEY_listen_address}:${OCF_RESKEY_port} -l tcp:127.0.0.1:${OCF_RESKEY_port}"
+           ;;
+    udptcp)  listen_param1="-l udp:${OCF_RESKEY_listen_address}:${OCF_RESKEY_port} -l udp:127.0.0.1:${OCF_RESKEY_port}"
+              listen_param2="-l tcp:${OCF_RESKEY_listen_address}:${OCF_RESKEY_port} -l tcp:127.0.0.1:${OCF_RESKEY_port}"
+              listen_param="${listen_param1} ${listen_param2}"
+           ;;
+    *)  listen_param="-T"
+           ;;
+    esac
+
+    kam_cmd="${OCF_RESKEY_binary} -P ${OCF_RESKEY_pidfile} -f ${OCF_RESKEY_conffile} $listen_param"
+}
+
+###
+# Gets the PID for the running Kamailio instance.
+#
+#     Returns: The variable $PID contains the found PID value or an empty string.
+# Exit Status: Zero if the PID file was found and this process run under
+#              the command line parameters of our instance.
+#              1) if the PID file is not present and no process running under
+#                 our command line options is active.
+#              2) in all other fatal cases, which we classify in the followig 
+#                 as OCF_ERR_genering. These are folloing cases:
+#                a) The PID file contains a PID value which does no match to 
+#                   to our instance
+#                b) The PID contains a empty string in its first line
+#                c) The PID file contains some text and some processeses
+#                   from our instance are still active
+
+kamailio_get_pid() {
+    if [ -f ${OCF_RESKEY_pidfile} ]; then
+        PID=`head -n 1 $OCF_RESKEY_pidfile`
+        if [ ! -z "$PID" ]; then
+            #Cross check if the PID file really contains a process of our kamailio instance:
+            kamailio_cmd
+            CROSSPID=`pgrep -o -f "${kam_cmd}"`
+            if [ x"$PID" == x"$CROSSPID" ]; then
+                #ocf_log debug "Found kamailio process PID with value: $PID."
+                return 0
+            fi
+            #ocf_log debug "PID file does not contain a PID of a $OCF_RESKEY_binary process!"
+            return 2
+        fi
+
+        #PID file does not contain a valid PID
+        rm -f ${OCF_RESKEY_pidfile}
+        return 2
+    fi
+
+
+    # No PID file found!
+    # Check if still a process exists even though we don't have the PID any longer:
+    kamailio_cmd
+    pgrep -f "${kam_cmd}"
+    if [ $? -eq 0 ]; then
+        ocf_log info "PID file does not contain a valid PID, but kamailio process is still active"
+        return 2
+    fi
+
+    ocf_log info "No PID file found and our kamailio instance is not active"
+    return 1
+}
+
+kamailio_status() {
+    local not_running_log_level="warn"
+    local errorfile error output
+
+    if [ "$__OCF_ACTION" = "start" ]; then
+        not_running_log_level="debug"
+    fi
+
+    kamailio_get_pid >/dev/null
+    RET=$?
+    if [ $RET -ne 0 ]; then
+        if [ $RET -eq 2 ]; then
+            ocf_log $not_running_log_level "PID file does not contain a PID of a ${OCF_RESKEY_binary} process!"
+            return $OCF_ERR_GENERIC
+        fi
+        return $OCF_NOT_RUNNING
+    fi
+
+    PID=`head -n 1 $OCF_RESKEY_pidfile`
+    isRunning_PID "$PID"
+    RET=$?
+    if [ "$RET" -ne 0 ]; then
+        ocf_log $not_running_log_level "PID from $PID from ${OCF_RESKEY_pidfile} not running"
+        rm -f ${OCF_RESKEY_pidfile}
+        return $OCF_NOT_RUNNING
+    fi
+
+    rc=0
+    # In case that OCF_RESKEY_kamctlrc we perfom a health check via "kamctl monitor 1"
+    if [ ! -z ${OCF_RESKEY_kamctlrc} ]; then
+        # PID is running now but it is not save to check via kamctl without care, because 
+        # the implementation analysis in the case that we kill all running processes
+        # shows that in case that the fifo cannot be read, then kamctl blocks. This needs
+        # to be avoided.
+        # In order to be on the safe side, we run this check therefore under "timeout" control:
+        rc=1
+        timeout 3 kamctl monitor 1 |grep "Up since" ; rc=$?
+    fi
+
+    if [ $rc -ne 0 ]; then
+        ocf_log $not_running_log_level "Kamailio is not up according to kamctl monitor!"
+        return $OCF_NOT_RUNNING
+    fi
+
+    errorfile=`mktemp`
+    case ${OCF_RESKEY_proto} in
+    udp)  output=`$OCF_RESKEY_sipsak -s sip:monitor@$OCF_RESKEY_monitoring_ip:${OCF_RESKEY_port} -H localhost --transport udp>/dev/null 2>>$errorfile`
+          result=$?
+          ;;
+    tcp)  output=`$OCF_RESKEY_sipsak -s sip:monitor@$OCF_RESKEY_monitoring_ip:${OCF_RESKEY_port} -H localhost --transport tcp>/dev/null 2>>$errorfile`
+          result=$?
+          ;;
+    udptcp) output=`$OCF_RESKEY_sipsak -s sip:monitor@$OCF_RESKEY_monitoring_ip:${OCF_RESKEY_port} -H localhost --transport tcp>/dev/null 2>>$errorfile`
+            result=$?
+            if [ $result -eq 0 ]; then
+                output=`$OCF_RESKEY_sipsak -s sip:monitor@$OCF_RESKEY_monitoring_ip:${OCF_RESKEY_port} -H localhost --transport udp>/dev/null 2>>$errorfile`
+                result=$?
+            fi
+            ;;
+    *)  output=`$OCF_RESKEY_sipsak -s sip:monitor@$OCF_RESKEY_monitoring_ip:${OCF_RESKEY_port} -H localhost --transport udp>/dev/null 2>>$errorfile`
+        result=$?
+        ;;
+    esac
+
+    error=`cat $errorfile`
+    rm -f $errorfile
+
+    if [ $result -ne 0 ]; then
+        ocf_log $not_running_log_level "Kamailio is running, but not functional as sipsak ${OCF_RESKEY_proto} failed with $(kamailio_format_result $result "$output" "$error")"
+        return $OCF_ERR_GENERIC
+    fi
+
+    return $OCF_SUCCESS
+}
+
+kamailio_monitor() {
+    kamailio_status
+}
+
+kamailio_start() {
+    local errorfile error output piddir
+
+    if
+        kamailio_status
+    then
+        ocf_log info "kamailio already running."
+        return $OCF_SUCCESS
+    fi
+
+    # if pidfile directory does not exist, create it with kamailio process owner
+    piddir=`dirname "${OCF_RESKEY_pidfile}"`
+    if [ ! -d "$piddir" ]; then
+        mkdir -p "$piddir"
+        if [ "$OCF_RESKEY_kamuser" != "" ]; then
+            chown ${OCF_RESKEY_kamuser} "$piddir"
+        fi
+    fi
+
+    kamailio_cmd
+    if [ "$OCF_RESKEY_kamuser" != "" ]; then
+        kam_cmd="su -s /bin/bash $OCF_RESKEY_kamuser -c \"$kam_cmd\""
+    fi
+
+    ocf_log info "start kamailio with $kam_cmd."
+    errorfile=`mktemp`
+    output=$(eval ${kam_cmd} 2>>$errorfile)
+    result=$?
+    error=`cat $errorfile`
+    rm -f $errorfile
+
+    if [ $result -eq 0 ]; then
+        result=1
+        while [ $result -ne 0 ]; do
+            sleep 1
+            kamailio_get_pid >/dev/null
+            result=$?
+        done
+
+        ocf_log info "kamailio instance PID=$PID started."
+        # check with monitor operation if running correctly
+        result=$OCF_ERR_GENERIC
+        while [ $result -ne $OCF_SUCCESS ]; do
+            sleep 1
+            kamailio_monitor
+            result=$?
+            ocf_log info "monitor in start returned $result"
+        done
+        ocf_log info "kamailio started successful."
+
+    else
+        ocf_log err "kamailio instance could not be started, $(kamailio_format_result $result "$output" "$error")"
+        result=$OCF_ERR_GENERIC
+    fi
+
+    return $result
+}
+
+kamailio_stop() {
+    local piddir
+    local TRIES=0
+    result=$OCF_SUCCESS
+
+    kamailio_cmd
+
+    ocf_log info "Stopping kamailio by sending SIGTERM to ${kam_cmd}"
+    pkill -SIGTERM -x -f "${kam_cmd}"
+    if [ $? -eq 1 ]; then
+        # already stopped. no processes found
+        # in case of not specified pidfile, delete the created directory
+        # otherwise only the pidfile itself
+        if [ "${OCF_RESKEY_pidfile}" == "${RESKEY_pidfile_default}" ]; then
+            piddir=`dirname "${OCF_RESKEY_pidfile}"`
+            rm -rf "$piddir"
+        else
+            rm -f "${OCF_RESKEY_pidfile}"
+        fi
+        return $result
+    fi
+
+    if [ "$OCF_RESKEY_CRM_meta_timeout" != "" ]; then
+        KAMAILIO_STOP_TIMEOUT=$(( ($OCF_RESKEY_CRM_meta_timeout/1000) - 7 ))
+    else
+        KAMAILIO_STOP_TIMEOUT=20
+    fi
+
+    while isRunning_cmd "${kam_cmd}" && [ "$TRIES" -lt "${KAMAILIO_STOP_TIMEOUT}" ]
+    do
+        sleep 1
+        ocf_log info "kamailio ${kam_cmd} is still running after SIGTERM"
+        ((TRIES++))
+    done
+
+    isRunning_cmd "${kam_cmd}"
+    RET=$?
+
+    if [ "$RET" -eq 0 ]; then
+        ocf_log info "Killing ${kam_cmd} with SIGKILL"
+        TRIES=0
+        pkill -SIGKILL -x -f "${kam_cmd}" > /dev/null 2>&1
+
+        while isRunning_cmd "${kam_cmd}" && [ "$TRIES" -lt 3 ]
+        do
+            sleep 1
+            ocf_log info "kamailio ${kam_cmd} is still running after SIGKILL"
+            ((TRIES++))
+        done
+
+        isRunning_cmd "${kam_cmd}"
+        RET=$?
+        if [ "$RET" -eq 0 ]; then
+            ocf_log fatal "kamailio is still running even after SIGKILL"
+            result=$OCF_ERR_GENERIC
+        fi
+    else
+        ocf_log info "${kam_cmd} has stopped."
+    fi
+
+    # in case of not specified pidfile, delete the created directory
+    # otherwise only the pidfile itself
+    if [ "${OCF_RESKEY_pidfile}" == "${RESKEY_pidfile_default}" ]; then
+        piddir=`dirname "${OCF_RESKEY_pidfile}"`
+        rm -rf "$piddir"
+    else
+        rm -f "${OCF_RESKEY_pidfile}"
+    fi
+    return $result
+
+}
+
+kamailio_validate_all() {
+    # Check if kamailio configuration is valid before starting the server
+
+    if [ ! -f $OCF_RESKEY_binary ]; then
+       ocf_log err "File OCF_RESKEY_binary [${OCF_RESKEY_binary}] does not exist!"
+       return $OCF_NOT_INSTALLED
+    fi
+  
+    out=$($OCF_RESKEY_binary -c 2>&1 > /dev/null)
+    retcode=$?
+    if [ "$retcode" -ne '0' ]; then
+        ocf_log info "Not starting kamailio: $OCF_RESKEY_binary does not start!"
+        return $OCF_ERR_CONFIGURED
+    fi
+
+    case $OCF_RESKEY_monitoring_ip in
+    "")    ocf_log err "Required parameter OCF_RESKEY_monitoring_ip is missing!"
+           return $OCF_ERR_CONFIGURED
+           ;;
+    [0-9]*.[0-9]*.[0-9]*.[0-9]*)     : OK
+            ;;
+    *)     ocf_log err "Parameter OCF_RESKEY_monitoring_ip [$OCF_RESKEY_monitoring_ip] is not an IP address!"
+           return $OCF_ERR_CONFIGURED
+           ;;
+    esac 
+
+    case $OCF_RESKEY_listen_address in
+    "")    ocf_log err "Required parameter $OCF_RESKEY_listen_address is missing!"
+           return $OCF_ERR_CONFIGURED
+           ;;
+    [0-9]*.[0-9]*.[0-9]*.[0-9]*)    : OK
+           ;;
+    *)     ocf_log err "Parameter OCF_RESKEY_listen_address [$OCF_RESKEY_listen_address] not an IP address!"
+           return $OCF_ERR_CONFIGURED
+           ;;
+    esac 
+
+    if [ ! -f ${OCF_RESKEY_sipsak} ]; then
+        ocf_log err "sipsak [${OCF_RESKEY_sipsak}] does not exist!"
+        return $OCF_NOT_INSTALLED
+    fi
+
+    if [ ! -z ${OCF_RESKEY_kamctlrc} ]; then
+        if [ ! -f ${OCF_RESKEY_kamctlrc} ]; then
+            ocf_log err "kamctlrc file [${kamctlrc}] does not exist!"
+            return $OCF_NOT_INSTALLED
+        fi
+    else
+        ocf_log debug "No monitoring via kamctl monitor because the parameter [kamctlrc] is empty."
+    fi
+
+    if [ ! -f ${OCF_RESKEY_conffile} ]; then
+        ocf_log err "Kamailio configuration file provided in the parameter conffile [${OCF_RESKEY_conffile}] does not exist!"
+        return $OCF_ERR_CONFIGURED
+    fi
+
+    case $OCF_RESKEY_proto in
+    "")   ocf_log err "Parameter $OCF_RESKEY_proto is empty!"
+          return $OCF_ERR_CONFIGURED
+          ;;
+    udp|tcp|udptcp)    : OK
+        ;;
+    *)   ocf_log err "Parameter value $OCF_RESKEY_proto for parameter [proto] not yet supported!"
+         return $OCF_ERR_CONFIGURED
+         ;;
+    esac
+
+    return $OCF_SUCCESS
+}
+
+if [ $# -ne 1 ]; then
+    usage
+    exit $OCF_ERR_ARGS
+fi
+
+case $__OCF_ACTION in
+    meta-data) meta_data
+        exit $OCF_SUCCESS
+    ;;
+    start|stop|status|monitor)
+        kamailio_${__OCF_ACTION}
+    ;;
+    validate-all) kamailio_validate_all
+    ;;
+    notify) exit $OCF_SUCCESS
+    ;;
+    usage) usage
+        exit $OCF_SUCCESS
+    ;;
+#   reload) #Not supported by Kamailio, but not needed by pacemaker
+#   ;;
+#   recover #Not needed by pacemaker
+#   ;;
+    *) usage
+        exit $OCF_ERR_UNIMPLEMENTED
+        ;;
+esac
+
+exit $?
diff --git a/heartbeat/lxc b/heartbeat/lxc
index 53a80df..811ae2b 100755
--- a/heartbeat/lxc
+++ b/heartbeat/lxc
@@ -113,6 +113,14 @@ LXC_usage() {
 END
 }
 
+lxc_version() {
+  if have_binary lxc-version ; then
+    lxc-version | cut -d' ' -f 3
+  else # since LXC 1.0.0 all commands knows about --version
+    lxc-info --version
+  fi
+}
+
 cgroup_mounted() {
 # test cgroup_mounted, mount if required
 	# Various possible overrides to cgroup mount point.
@@ -189,21 +197,56 @@ LXC_stop() {
 	# If the container is running "init" and is able to perform and orderly shutdown, then it should be done.
 	# It is 'assumed' that the 'init' system will do an orderly shudown if presented with a 'kill -PWR' signal.
 	# On a 'sysvinit' this would require the container to have an inittab file containing "p0::powerfail:/sbin/init 0"
-	typeset -i PID=0
-	# This should work for traditional 'sysvinit' and 'upstart'
-	lxc-ps --name "${OCF_RESKEY_container}" -- -C init -o pid,comm |while read CN PID CMD ;do
-		[ $PID -gt 1 ] || continue
-		[ "$CMD" = "init" ] || continue
-		ocf_log info "Sending \"OS shut down\" instruction to" ${OCF_RESKEY_container} "as it was found to be using \"sysV init\" or \"upstart\""
-		kill -PWR $PID
-	done
-	# This should work for containers using 'systemd' instead of 'init'
-	lxc-ps --name "${OCF_RESKEY_container}" -- -C systemd -o pid,comm |while read CN PID CMD ;do
-		[ $PID -gt 1 ] || continue
-		[ "$CMD" = "systemd" ] || continue
-		ocf_log info "Sending \"OS shut down\" instruction to" ${OCF_RESKEY_container} "as it was found to be using \"systemd\""
-		kill -PWR $PID
-	done
+	declare -i PID=0
+	declare CMD=
+
+	# LXC prior 1.0.0
+	if ocf_version_cmp "`lxc_version`" 1.0.0 ; then
+		# This should work for traditional 'sysvinit' and 'upstart'
+		lxc-ps --name "${OCF_RESKEY_container}" -- -C init -o pid,comm |while read CN PID CMD ;do
+			[ $PID -gt 1 ] || continue
+			[ "$CMD" = "init" ] || continue
+			ocf_log info "Sending \"OS shut down\" instruction to" ${OCF_RESKEY_container} "as it was found to be using \"sysV init\" or \"upstart\""
+			kill -PWR $PID
+		done
+		# This should work for containers using 'systemd' instead of 'init'
+		lxc-ps --name "${OCF_RESKEY_container}" -- -C systemd -o pid,comm |while read CN PID CMD ;do
+			[ $PID -gt 1 ] || continue
+			[ "$CMD" = "systemd" ] || continue
+			ocf_log info "Sending \"OS shut down\" instruction to" ${OCF_RESKEY_container} "as it was found to be using \"systemd\""
+			kill -PWR $PID
+		done
+	else
+		PID=$(lxc-info --name "${OCF_RESKEY_container}" -p -H)
+
+		# If there is no PID the container seems to be down which
+		# shouldn't happen.
+		if [ $PID -eq 0 ]; then
+			ocf_log err "${OCF_RESKEY_container} seems to run, but has no PID."
+			exit $OCF_ERR_GENERIC
+		fi
+
+		# Rescue me.
+		if [ $PID -eq 1 ]; then
+			ocf_log err "${OCF_RESKEY_container} seems to run with PID 1 which cannot be."
+			PID=0
+			CMD=
+		else
+			CMD=$(ps -o comm= -p $PID)
+		fi
+
+		# This should work for traditional 'sysvinit' and 'upstart'
+		if [ "$CMD" = "init" ]; then
+			ocf_log info "Sending \"OS shut down\" instruction to" ${OCF_RESKEY_container} "as it was found to be using \"sysV init\" or \"upstart\""
+			kill -PWR $PID
+		fi
+
+		# This should work for containers using 'systemd' instead of 'init'
+		if [ "$CMD" = "systemd" ]; then
+			ocf_log info "Sending \"OS shut down\" instruction to" ${OCF_RESKEY_container} "as it was found to be using \"systemd\""
+			kill -PWR $PID
+		fi
+	fi
 	# The "shutdown_timeout" we use here is the operation
 	# timeout specified in the CIB, minus 5 seconds
 	now=$(date +%s)
@@ -241,7 +284,7 @@ LXC_stop() {
 LXC_status() {
 	# run lxc-info with -s option for LXC-0.7.5 or later
 	local lxc_info_opt="-s"
-	ocf_version_cmp "`lxc-version | cut -d' ' -f 3`" 0.7.5 && lxc_info_opt=""
+	ocf_version_cmp "`lxc_version`" 0.7.5 && lxc_info_opt=""
 	S=`lxc-info $lxc_info_opt -n ${OCF_RESKEY_container}`
 	ocf_log debug "State of ${OCF_RESKEY_container}: $S"
 	if [[ "${S##* }" = "RUNNING" ]] ; then 
@@ -284,7 +327,9 @@ LXC_validate() {
 
 	    check_binary lxc-start
 	    check_binary lxc-stop
-	    check_binary lxc-ps
+	    if ocf_version_cmp "`lxc_version`" 1.0.0 ; then
+	        check_binary lxc-ps
+	    fi
 	    check_binary lxc-info
 	fi
 	return $OCF_SUCCESS
diff --git a/heartbeat/mysql b/heartbeat/mysql
index 5e332a1..c3282fd 100755
--- a/heartbeat/mysql
+++ b/heartbeat/mysql
@@ -51,97 +51,7 @@
 
 : ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
 . ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
-
-#######################################################################
-
-# Attempt to detect a default binary
-OCF_RESKEY_binary_default=$(which mysqld_safe 2> /dev/null)
-if [ "$OCF_RESKEY_binary_default" = "" ]; then
-	OCF_RESKEY_binary_default=$(which safe_mysqld 2> /dev/null)
-fi
-
-# Fill in some defaults if no values are specified
-HOSTOS=`uname`
-if [ "X${HOSTOS}" = "XOpenBSD" ];then
-	if [ "$OCF_RESKEY_binary_default" = "" ]; then
-		OCF_RESKEY_binary_default="/usr/local/bin/mysqld_safe"
-	fi
-	OCF_RESKEY_config_default="/etc/my.cnf"
-	OCF_RESKEY_datadir_default="/var/mysql"
-	OCF_RESKEY_user_default="_mysql"
-	OCF_RESKEY_group_default="_mysql"
-	OCF_RESKEY_log_default="/var/log/mysqld.log"
-	OCF_RESKEY_pid_default="/var/mysql/mysqld.pid"
-	OCF_RESKEY_socket_default="/var/run/mysql/mysql.sock"
-else
-	if [ "$OCF_RESKEY_binary_default" = "" ]; then
-		OCF_RESKEY_binary_default="/usr/bin/safe_mysqld"
-	fi
-	OCF_RESKEY_config_default="/etc/my.cnf"
-	OCF_RESKEY_datadir_default="/var/lib/mysql"
-	OCF_RESKEY_user_default="mysql"
-	OCF_RESKEY_group_default="mysql"
-	OCF_RESKEY_log_default="/var/log/mysqld.log"
-	OCF_RESKEY_pid_default="/var/run/mysql/mysqld.pid"
-	OCF_RESKEY_socket_default="/var/lib/mysql/mysql.sock"
-fi
-OCF_RESKEY_client_binary_default="mysql"
-OCF_RESKEY_test_user_default="root"
-OCF_RESKEY_test_table_default="mysql.user"
-OCF_RESKEY_test_passwd_default=""
-OCF_RESKEY_enable_creation_default=0
-OCF_RESKEY_additional_parameters_default=""
-OCF_RESKEY_replication_port_default="3306"
-OCF_RESKEY_max_slave_lag_default="3600"
-OCF_RESKEY_evict_outdated_slaves_default="false"
-OCF_RESKEY_reader_attribute_default="readable"
-
-: ${OCF_RESKEY_binary=${OCF_RESKEY_binary_default}}
-MYSQL_BINDIR=`dirname ${OCF_RESKEY_binary}`
-
-: ${OCF_RESKEY_client_binary=${OCF_RESKEY_client_binary_default}}
-
-: ${OCF_RESKEY_config=${OCF_RESKEY_config_default}}
-: ${OCF_RESKEY_datadir=${OCF_RESKEY_datadir_default}}
-
-: ${OCF_RESKEY_user=${OCF_RESKEY_user_default}}
-: ${OCF_RESKEY_group=${OCF_RESKEY_group_default}}
-
-: ${OCF_RESKEY_log=${OCF_RESKEY_log_default}}
-: ${OCF_RESKEY_pid=${OCF_RESKEY_pid_default}}
-: ${OCF_RESKEY_socket=${OCF_RESKEY_socket_default}}
-
-: ${OCF_RESKEY_test_user=${OCF_RESKEY_test_user_default}}
-: ${OCF_RESKEY_test_table=${OCF_RESKEY_test_table_default}}
-: ${OCF_RESKEY_test_passwd=${OCF_RESKEY_test_passwd_default}}
-
-: ${OCF_RESKEY_enable_creation=${OCF_RESKEY_enable_creation_default}}
-: ${OCF_RESKEY_additional_parameters=${OCF_RESKEY_additional_parameters_default}}
-
-: ${OCF_RESKEY_replication_user=${OCF_RESKEY_replication_user_default}}
-: ${OCF_RESKEY_replication_passwd=${OCF_RESKEY_replication_passwd_default}}
-: ${OCF_RESKEY_replication_port=${OCF_RESKEY_replication_port_default}}
-
-: ${OCF_RESKEY_max_slave_lag=${OCF_RESKEY_max_slave_lag_default}}
-: ${OCF_RESKEY_evict_outdated_slaves=${OCF_RESKEY_evict_outdated_slaves_default}}
-
-: ${OCF_RESKEY_reader_attribute=${OCF_RESKEY_reader_attribute_default}}
-
-#######################################################################
-# Convenience variables
-
-MYSQL=$OCF_RESKEY_client_binary
-MYSQL_OPTIONS_LOCAL="-S $OCF_RESKEY_socket --connect_timeout=10"
-MYSQL_OPTIONS_REPL="$MYSQL_OPTIONS_LOCAL --user=$OCF_RESKEY_replication_user --password=$OCF_RESKEY_replication_passwd"
-MYSQL_OPTIONS_TEST="$MYSQL_OPTIONS_LOCAL --user=$OCF_RESKEY_test_user --password=$OCF_RESKEY_test_passwd"
-MYSQL_TOO_MANY_CONN_ERR=1040
-
-CRM_MASTER="${HA_SBIN_DIR}/crm_master -l reboot "
-NODENAME=$(ocf_local_nodename)
-CRM_ATTR="${HA_SBIN_DIR}/crm_attribute -N $NODENAME "
-INSTANCE_ATTR_NAME=`echo ${OCF_RESOURCE_INSTANCE}| awk -F : '{print $1}'`
-CRM_ATTR_REPL_INFO="${HA_SBIN_DIR}/crm_attribute --type crm_config --name ${INSTANCE_ATTR_NAME}_REPL_INFO -s mysql_replication"
-
+. ${OCF_FUNCTIONS_DIR}/mysql-common.sh
 #######################################################################
 
 usage() {
@@ -172,6 +82,9 @@ meta_data() {
 Resource script for MySQL.
 May manage a standalone MySQL database, a clone set with externally
 managed replication, or a complete master/slave replication setup.
+Note, when master/slave replication is in use, the resource must
+be setup to use notifications. Set 'notify=true' in the metadata
+attributes when defining a MySQL master/slave instance.
 
 While managing replication, the default behavior is to use uname -n 
 values in the change master to command.  Other IPs can be specified 
@@ -301,9 +214,9 @@ Additional parameters which are passed to the mysqld on startup.
 MySQL replication user. This user is used for starting and stopping
 MySQL replication, for setting and resetting the master host, and for
 setting and unsetting read-only mode. Because of that, this user must
-have SUPER, REPLICATION SLAVE, REPLICATION CLIENT, and PROCESS
-privileges on all nodes within the cluster. Mandatory if you define
-a master-slave resource.
+have SUPER, REPLICATION SLAVE, REPLICATION CLIENT, PROCESS and RELOAD
+privileges on all nodes within the cluster. Mandatory if you define a
+master-slave resource.
 </longdesc>
 <shortdesc lang="en">MySQL replication user</shortdesc>
 <content type="string" default="${OCF_RESKEY_replication_user_default}" />
@@ -484,7 +397,7 @@ get_slave_info() {
         else
             # Instance produced an empty "SHOW SLAVE STATUS" output --
             # instance is not a slave
-            ocf_log err "check_slave invoked on an instance that is not a replication slave."
+            ocf_exit_reason "check_slave invoked on an instance that is not a replication slave."
             return $OCF_ERR_GENERIC
         fi
 
@@ -505,7 +418,7 @@ check_slave() {
             # Whoa. Replication ran into an error. This slave has
             # diverged from its master. Make sure this resource
             # doesn't restart in place.
-            ocf_log err "MySQL instance configured for replication, but replication has failed."
+            ocf_exit_reason "MySQL instance configured for replication, but replication has failed."
             ocf_log err "See $tmpfile for details"
 
             # Just pull the reader VIP away, killing MySQL here would be pretty evil
@@ -544,7 +457,7 @@ check_slave() {
             # We don't have a replication SQL thread running. Not a
             # good thing. Try to recoved by restarting the SQL thread
             # and remove reader vip.  Prevent MySQL restart.
-            ocf_log err "MySQL Slave SQL threads currently not running."
+            ocf_exit_reason "MySQL Slave SQL threads currently not running."
             ocf_log err "See $tmpfile for details"
 
             # Remove reader vip
@@ -561,8 +474,8 @@ check_slave() {
         if ocf_is_true $OCF_RESKEY_evict_outdated_slaves; then
             # We're supposed to bail out if we lag too far
             # behind. Let's check our lag.
-            if [ $secs_behind -gt $OCF_RESKEY_max_slave_lag ]; then
-                ocf_log err "MySQL Slave is $secs_behind seconds behind master (allowed maximum: $OCF_RESKEY_max_slave_lag)."
+            if [ "$secs_behind" = "NULL" ] || [ $secs_behind -gt $OCF_RESKEY_max_slave_lag ]; then
+                ocf_exit_reason "MySQL Slave is $secs_behind seconds behind master (allowed maximum: $OCF_RESKEY_max_slave_lag)."
                 ocf_log err "See $tmpfile for details"
 
                 # Remove reader vip
@@ -570,7 +483,7 @@ check_slave() {
 
                 exit $OCF_ERR_INSTALLED
             fi
-            elif ocf_is_ms; then
+        elif ocf_is_ms; then
             # Even if we're not set to evict lagging slaves, we can
             # still use the seconds behind master value to set our
             # master preference.
@@ -584,7 +497,7 @@ check_slave() {
         fi
 
         # is the slave ok to have a VIP on it
-        if [ $secs_behind -gt $OCF_RESKEY_max_slave_lag ]; then
+        if [ "$secs_behind" = "NULL" ] || [ $secs_behind -gt $OCF_RESKEY_max_slave_lag ]; then
             set_reader_attr 0
         else
             set_reader_attr 1
@@ -597,7 +510,7 @@ check_slave() {
         # instance is not a slave
         # TODO: Needs to handle when get_slave_info will return too many connections error
         rm -f $tmpfile
-        ocf_log err "check_slave invoked on an instance that is not a replication slave."
+        ocf_exit_reason "check_slave invoked on an instance that is not a replication slave."
         exit $OCF_ERR_GENERIC
     fi
 }
@@ -686,7 +599,7 @@ unset_master(){
     ocf_run $MYSQL $MYSQL_OPTIONS_REPL \
         -e "STOP SLAVE IO_THREAD"
     if [ $? -gt 0 ]; then
-        ocf_log err "Error stopping slave IO thread"
+        ocf_exit_reason "Error stopping slave IO thread"
         exit $OCF_ERR_GENERIC
     fi
 
@@ -710,14 +623,14 @@ unset_master(){
     ocf_run $MYSQL $MYSQL_OPTIONS_REPL \
         -e "STOP SLAVE"
     if [ $? -gt 0 ]; then
-        ocf_log err "Error stopping rest slave threads"
+        ocf_exit_reason "Error stopping rest slave threads"
         exit $OCF_ERR_GENERIC
     fi
 
     ocf_run $MYSQL $MYSQL_OPTIONS_REPL \
         -e "RESET SLAVE /*!50516 ALL */;"
     if [ $? -gt 0 ]; then
-        ocf_log err "Failed to reset slave"
+        ocf_exit_reason "Failed to reset slave"
         exit $OCF_ERR_GENERIC
     fi
 }
@@ -794,57 +707,6 @@ get_local_ip() {
 
 # Functions invoked by resource manager actions
 
-mysql_validate() {
-    check_binary $OCF_RESKEY_binary
-    check_binary  $OCF_RESKEY_client_binary
-
-    if [ ! -f $OCF_RESKEY_config ]; then
-        ocf_log err "Config $OCF_RESKEY_config doesn't exist";
-        return $OCF_ERR_INSTALLED;
-    fi
-
-    if [ ! -d $OCF_RESKEY_datadir ]; then
-        ocf_log err "Datadir $OCF_RESKEY_datadir doesn't exist";
-        return $OCF_ERR_INSTALLED;
-    fi
-
-    getent passwd $OCF_RESKEY_user >/dev/null 2>&1
-    if [ ! $? -eq 0 ]; then
-        ocf_log err "User $OCF_RESKEY_user doesn't exit";
-        return $OCF_ERR_INSTALLED;
-    fi
-
-    getent group $OCF_RESKEY_group >/dev/null 2>&1
-    if [ ! $? -eq 0 ]; then
-        ocf_log err "Group $OCF_RESKEY_group doesn't exist";
-        return $OCF_ERR_INSTALLED;
-    fi
-
-    true
-}
-
-mysql_status() {
-    if [ ! -e $OCF_RESKEY_pid ]; then
-        ocf_log $1 "MySQL is not running"
-        return $OCF_NOT_RUNNING;
-    fi
-
-    pid=`cat $OCF_RESKEY_pid`;
-    if [ -d /proc -a -d /proc/1 ]; then
-        [ "u$pid" != "u" -a -d /proc/$pid ]
-    else
-        kill -s 0 $pid >/dev/null 2>&1
-    fi
-
-    if [ $? -eq 0 ]; then
-        return $OCF_SUCCESS;
-    else
-        ocf_log $1 "MySQL not running: removing old PID file"
-        rm -f $OCF_RESKEY_pid
-        return $OCF_NOT_RUNNING;
-    fi
-}
-
 mysql_monitor() {
     local rc
     local status_loglevel="err"
@@ -854,7 +716,7 @@ mysql_monitor() {
         status_loglevel="info"
     fi
  
-    mysql_status $status_loglevel
+    mysql_common_status $status_loglevel
 
     rc=$?
 
@@ -865,7 +727,6 @@ mysql_monitor() {
         return $rc
     fi
 
-
     if [ $OCF_CHECK_LEVEL -gt 0 -a -n "$OCF_RESKEY_test_table" ]; then
         # Check if this instance is configured as a slave, and if so
         # check slave status
@@ -879,7 +740,7 @@ mysql_monitor() {
         rc=$?
 
         if [ $rc -ne 0 ]; then
-            ocf_log err "Failed to select from $test_table";
+            ocf_exit_reason "Failed to select from $test_table";
             return $OCF_ERR_GENERIC;
         fi
     fi
@@ -894,58 +755,20 @@ mysql_monitor() {
 }
 
 mysql_start() {
-    local rc pid
+    local rc
 
     if ocf_is_ms; then
         # Initialize the ReaderVIP attribute, monitor will enable it
         set_reader_attr 0
     fi
 
-    mysql_status info
+    mysql_common_status info
     if [ $? = $OCF_SUCCESS ]; then
         ocf_log info "MySQL already running"
         return $OCF_SUCCESS
     fi
 
-    touch $OCF_RESKEY_log
-    chown $OCF_RESKEY_user:$OCF_RESKEY_group $OCF_RESKEY_log
-    chmod 0640 $OCF_RESKEY_log
-    [ -x /sbin/restorecon ] && /sbin/restorecon $OCF_RESKEY_log
-
-    if ocf_is_true "$OCF_RESKEY_enable_creation" && [ ! -d $OCF_RESKEY_datadir/mysql ] ; then
-        ocf_log info "Initializing MySQL database: "
-        $MYSQL_BINDIR/mysql_install_db --datadir=$OCF_RESKEY_datadir
-        rc=$?
-        if [ $rc -ne 0 ] ; then
-            ocf_log err "Initialization failed: $rc";
-            exit $OCF_ERR_GENERIC
-        fi
-        chown -R $OCF_RESKEY_user:$OCF_RESKEY_group $OCF_RESKEY_datadir
-    fi
-
-    pid_dir=`dirname $OCF_RESKEY_pid`
-    if [ ! -d $pid_dir ] ; then
-        ocf_log info "Creating PID dir: $pid_dir"
-        mkdir -p $pid_dir
-        chown $OCF_RESKEY_user:$OCF_RESKEY_group $pid_dir
-    fi
-
-    socket_dir=`dirname $OCF_RESKEY_socket`
-    if [ ! -d $socket_dir ] ; then
-        ocf_log info "Creating socket dir: $socket_dir"
-        mkdir -p $socket_dir
-        chown $OCF_RESKEY_user:$OCF_RESKEY_group $socket_dir
-    fi
-
-    # Regardless of whether we just created the directory or it
-    # already existed, check whether it is writable by the configured
-    # user
-    for dir in $pid_dir $socket_dir; do
-        if ! su -s /bin/sh - $OCF_RESKEY_user -c "test -w $dir"; then
-            ocf_log err "Directory $dir is not writable by $OCF_RESKEY_user"
-            exit $OCF_ERR_PERM;
-        fi
-    done
+    mysql_common_prepare_dirs
 
     # Uncomment to perform permission clensing
     # - not convinced this should be enabled by default
@@ -958,34 +781,11 @@ mysql_start() {
         mysql_extra_params="--skip-slave-start"
     fi
 
-    ${OCF_RESKEY_binary} --defaults-file=$OCF_RESKEY_config \
-    --pid-file=$OCF_RESKEY_pid \
-    --socket=$OCF_RESKEY_socket \
-    --datadir=$OCF_RESKEY_datadir \
-    --log-error=$OCF_RESKEY_log \
-    --user=$OCF_RESKEY_user $OCF_RESKEY_additional_parameters \
-    $mysql_extra_params >/dev/null 2>&1 &
-    pid=$!
-
-    # Spin waiting for the server to come up.
-    # Let the CRM/LRM time us out if required.
-    start_wait=1
-    while [ $start_wait = 1 ]; do
-        if ! ps $pid > /dev/null 2>&1; then
-            wait $pid
-            ocf_log err "MySQL server failed to start (rc=$?), please check your installation"
-            return $OCF_ERR_GENERIC
-        fi
-        mysql_status info
-        rc=$?
-        if [ $rc = $OCF_SUCCESS ]; then
-            start_wait=0
-        elif [ $rc != $OCF_NOT_RUNNING ]; then
-            ocf_log info "MySQL start failed: $rc"
-            return $rc
-        fi
-        sleep 2
-    done
+    mysql_common_start $mysql_extra_params
+    rc=$?
+    if [ $rc != $OCF_SUCCESS ]; then
+        return $rc
+    fi
 
     if ocf_is_ms; then
         # We're configured as a stateful resource. We must start as
@@ -1003,7 +803,7 @@ mysql_start() {
             set_master
             start_slave
             if [ $? -ne 0 ]; then
-                ocf_log err "Failed to start slave"
+                ocf_exit_reason "Failed to start slave"
                 return $OCF_ERR_GENERIC
             fi
         else
@@ -1026,7 +826,7 @@ mysql_start() {
     mysql_monitor
     rc=$?
     if [ $rc != $OCF_SUCCESS -a $rc != $OCF_RUNNING_MASTER ]; then
-        ocf_log err "Failed initial monitor action"
+        ocf_exit_reason "Failed initial monitor action"
         return $rc
     fi
 
@@ -1035,7 +835,6 @@ mysql_start() {
 }
 
 mysql_stop() {
-
     if ocf_is_ms; then
         # clear preference for becoming master
         $CRM_MASTER -D
@@ -1044,52 +843,13 @@ mysql_stop() {
         set_reader_attr 0
     fi
 
-    if [ ! -f $OCF_RESKEY_pid ]; then
-        ocf_log info "MySQL is not running"
-        return $OCF_SUCCESS
-    fi
-
-    pid=`cat $OCF_RESKEY_pid 2> /dev/null `
-    /bin/kill $pid > /dev/null
-    rc=$?
-    if [ $rc != 0 ]; then
-        ocf_log err "MySQL couldn't be stopped"
-        return $OCF_ERR_GENERIC
-    fi
-    # stop waiting
-    shutdown_timeout=15
-    if [ -n "$OCF_RESKEY_CRM_meta_timeout" ]; then
-        shutdown_timeout=$((($OCF_RESKEY_CRM_meta_timeout/1000)-5))
-    fi
-    count=0
-    while [ $count -lt $shutdown_timeout ]
-    do
-        mysql_status info
-        rc=$?
-        if [ $rc = $OCF_NOT_RUNNING ]; then
-            break
-        fi
-        count=`expr $count + 1`
-        sleep 1
-        ocf_log debug "MySQL still hasn't stopped yet. Waiting..."
-    done
-
-    mysql_status info
-    if [ $? != $OCF_NOT_RUNNING ]; then
-        ocf_log info "MySQL failed to stop after ${shutdown_timeout}s using SIGTERM. Trying SIGKILL..."
-        /bin/kill -KILL $pid > /dev/null
-    fi
-
-    ocf_log info "MySQL stopped";
-    rm -f /var/lock/subsys/mysqld
-    rm -f $OCF_RESKEY_socket
-    return $OCF_SUCCESS
+    mysql_common_stop
 }
 
 mysql_promote() {
     local master_info
 
-    if ( ! mysql_status err ); then
+    if ( ! mysql_common_status err ); then
         return $OCF_NOT_RUNNING
     fi
     ocf_run $MYSQL $MYSQL_OPTIONS_REPL \
@@ -1115,7 +875,7 @@ mysql_promote() {
 }
 
 mysql_demote() {
-    if ! mysql_status err; then
+    if ! mysql_common_status err; then
         return $OCF_NOT_RUNNING
     fi
 
@@ -1164,7 +924,7 @@ mysql_notify() {
 
                 start_slave
                 if [ $? -ne 0 ]; then
-                    ocf_log err "Failed to start slave"
+                    ocf_exit_reason "Failed to start slave"
                     return $OCF_ERR_GENERIC
                 fi
             fi
@@ -1176,7 +936,7 @@ mysql_notify() {
                 ocf_log info "post-demote notification for $demote_host"
                 set_read_only on
                 if [ $? -ne 0 ]; then
-                    ocf_log err "Failed to set read-only";
+                    ocf_exit_reason "Failed to set read-only";
                     return $OCF_ERR_GENERIC;
                 fi
 
@@ -1244,13 +1004,22 @@ case "$1" in
         exit $OCF_SUCCESS;;
 esac
 
-mysql_validate
+mysql_common_validate
 rc=$?
 LSB_STATUS_STOPPED=3
 if [ $rc -ne 0 ]; then
     case "$1" in
-        stop) exit $OCF_SUCCESS;;
-        monitor) exit $OCF_NOT_RUNNING;;
+        stop) ;;
+        monitor)
+            mysql_common_status "info"
+            if [ $? -eq $OCF_SUCCESS ]; then
+                # if validatation fails and pid is active, always treat this as an error
+                ocf_exit_reason "environment validation failed, active pid is in unknown state."
+                exit $OCF_ERR_GENERIC
+            fi
+            # validation failed and pid is not active, it's safe to say this instance is inactive.
+            exit $OCF_NOT_RUNNING;;
+
         status) exit $LSB_STATUS_STOPPED;;
         *) exit $rc;;
     esac
@@ -1260,7 +1029,7 @@ fi
 case "$1" in
   start)    mysql_start;;
   stop)     mysql_stop;;
-  status)   mysql_status err;;
+  status)   mysql_common_status err;;
   monitor)  mysql_monitor;;
   promote)  mysql_promote;;
   demote)   mysql_demote;;
diff --git a/heartbeat/mysql-common.sh b/heartbeat/mysql-common.sh
new file mode 100755
index 0000000..6564c66
--- /dev/null
+++ b/heartbeat/mysql-common.sh
@@ -0,0 +1,287 @@
+#!/bin/sh
+
+#######################################################################
+
+# Attempt to detect a default binary
+OCF_RESKEY_binary_default=$(which mysqld_safe 2> /dev/null)
+if [ "$OCF_RESKEY_binary_default" = "" ]; then
+	OCF_RESKEY_binary_default=$(which safe_mysqld 2> /dev/null)
+fi
+
+# Fill in some defaults if no values are specified
+HOSTOS=`uname`
+if [ "X${HOSTOS}" = "XOpenBSD" ];then
+	if [ "$OCF_RESKEY_binary_default" = "" ]; then
+		OCF_RESKEY_binary_default="/usr/local/bin/mysqld_safe"
+	fi
+	OCF_RESKEY_config_default="/etc/my.cnf"
+	OCF_RESKEY_datadir_default="/var/mysql"
+	OCF_RESKEY_user_default="_mysql"
+	OCF_RESKEY_group_default="_mysql"
+	OCF_RESKEY_log_default="/var/log/mysqld.log"
+	OCF_RESKEY_pid_default="/var/mysql/mysqld.pid"
+	OCF_RESKEY_socket_default="/var/run/mysql/mysql.sock"
+else
+	if [ "$OCF_RESKEY_binary_default" = "" ]; then
+		OCF_RESKEY_binary_default="/usr/bin/safe_mysqld"
+	fi
+	OCF_RESKEY_config_default="/etc/my.cnf"
+	OCF_RESKEY_datadir_default="/var/lib/mysql"
+	OCF_RESKEY_user_default="mysql"
+	OCF_RESKEY_group_default="mysql"
+	OCF_RESKEY_log_default="/var/log/mysqld.log"
+	OCF_RESKEY_pid_default="/var/run/mysql/mysqld.pid"
+	OCF_RESKEY_socket_default="/var/lib/mysql/mysql.sock"
+fi
+OCF_RESKEY_client_binary_default="mysql"
+OCF_RESKEY_test_user_default="root"
+OCF_RESKEY_test_table_default="mysql.user"
+OCF_RESKEY_test_passwd_default=""
+OCF_RESKEY_enable_creation_default=0
+OCF_RESKEY_additional_parameters_default=""
+OCF_RESKEY_replication_port_default="3306"
+OCF_RESKEY_max_slave_lag_default="3600"
+OCF_RESKEY_evict_outdated_slaves_default="false"
+OCF_RESKEY_reader_attribute_default="readable"
+
+: ${OCF_RESKEY_binary=${OCF_RESKEY_binary_default}}
+MYSQL_BINDIR=`dirname ${OCF_RESKEY_binary}`
+
+: ${OCF_RESKEY_client_binary=${OCF_RESKEY_client_binary_default}}
+
+: ${OCF_RESKEY_config=${OCF_RESKEY_config_default}}
+: ${OCF_RESKEY_datadir=${OCF_RESKEY_datadir_default}}
+
+: ${OCF_RESKEY_user=${OCF_RESKEY_user_default}}
+: ${OCF_RESKEY_group=${OCF_RESKEY_group_default}}
+
+: ${OCF_RESKEY_log=${OCF_RESKEY_log_default}}
+: ${OCF_RESKEY_pid=${OCF_RESKEY_pid_default}}
+: ${OCF_RESKEY_socket=${OCF_RESKEY_socket_default}}
+
+: ${OCF_RESKEY_test_user=${OCF_RESKEY_test_user_default}}
+: ${OCF_RESKEY_test_table=${OCF_RESKEY_test_table_default}}
+: ${OCF_RESKEY_test_passwd=${OCF_RESKEY_test_passwd_default}}
+
+: ${OCF_RESKEY_enable_creation=${OCF_RESKEY_enable_creation_default}}
+: ${OCF_RESKEY_additional_parameters=${OCF_RESKEY_additional_parameters_default}}
+
+: ${OCF_RESKEY_replication_user=${OCF_RESKEY_replication_user_default}}
+: ${OCF_RESKEY_replication_passwd=${OCF_RESKEY_replication_passwd_default}}
+: ${OCF_RESKEY_replication_port=${OCF_RESKEY_replication_port_default}}
+
+: ${OCF_RESKEY_max_slave_lag=${OCF_RESKEY_max_slave_lag_default}}
+: ${OCF_RESKEY_evict_outdated_slaves=${OCF_RESKEY_evict_outdated_slaves_default}}
+
+: ${OCF_RESKEY_reader_attribute=${OCF_RESKEY_reader_attribute_default}}
+
+#######################################################################
+# Convenience variables
+
+MYSQL=$OCF_RESKEY_client_binary
+MYSQL_OPTIONS_LOCAL="-S $OCF_RESKEY_socket"
+MYSQL_OPTIONS_REPL="$MYSQL_OPTIONS_LOCAL --user=$OCF_RESKEY_replication_user --password=$OCF_RESKEY_replication_passwd"
+MYSQL_OPTIONS_TEST="$MYSQL_OPTIONS_LOCAL --user=$OCF_RESKEY_test_user --password=$OCF_RESKEY_test_passwd"
+MYSQL_TOO_MANY_CONN_ERR=1040
+
+CRM_MASTER="${HA_SBIN_DIR}/crm_master -l reboot "
+NODENAME=$(ocf_local_nodename)
+CRM_ATTR="${HA_SBIN_DIR}/crm_attribute -N $NODENAME "
+INSTANCE_ATTR_NAME=`echo ${OCF_RESOURCE_INSTANCE}| awk -F : '{print $1}'`
+CRM_ATTR_REPL_INFO="${HA_SBIN_DIR}/crm_attribute --type crm_config --name ${INSTANCE_ATTR_NAME}_REPL_INFO -s mysql_replication"
+
+#######################################################################
+
+mysql_common_validate()
+{
+
+    if ! have_binary "$OCF_RESKEY_binary"; then
+        ocf_exit_reason "Setup problem: couldn't find command: $OCF_RESKEY_binary"
+        return $OCF_ERR_INSTALLED;
+    fi
+
+    if ! have_binary "$OCF_RESKEY_client_binary"; then
+        ocf_exit_reason "Setup problem: couldn't find command: $OCF_RESKEY_client_binary"
+        return $OCF_ERR_INSTALLED;
+    fi
+
+    if [ ! -f $OCF_RESKEY_config ]; then
+        ocf_exit_reason "Config $OCF_RESKEY_config doesn't exist";
+        return $OCF_ERR_INSTALLED;
+    fi
+
+    if [ ! -d $OCF_RESKEY_datadir ]; then
+        ocf_exit_reason "Datadir $OCF_RESKEY_datadir doesn't exist";
+        return $OCF_ERR_INSTALLED;
+    fi
+
+    getent passwd $OCF_RESKEY_user >/dev/null 2>&1
+    if [ ! $? -eq 0 ]; then
+        ocf_exit_reason "User $OCF_RESKEY_user doesn't exit";
+        return $OCF_ERR_INSTALLED;
+    fi
+
+    getent group $OCF_RESKEY_group >/dev/null 2>&1
+    if [ ! $? -eq 0 ]; then
+        ocf_exit_reason "Group $OCF_RESKEY_group doesn't exist";
+        return $OCF_ERR_INSTALLED;
+    fi
+
+    return $OCF_SUCCESS
+}
+
+mysql_common_status() {
+    local loglevel=$1
+    local pid=$2
+    if [ -z "$pid" ]; then
+        if [ ! -e $OCF_RESKEY_pid ]; then
+            ocf_log $loglevel "MySQL is not running"
+            return $OCF_NOT_RUNNING;
+        fi
+
+        pid=`cat $OCF_RESKEY_pid`;
+    fi
+    if [ -d /proc -a -d /proc/1 ]; then
+        [ "u$pid" != "u" -a -d /proc/$pid ]
+    else
+        kill -s 0 $pid >/dev/null 2>&1
+    fi
+
+    if [ $? -eq 0 ]; then
+        return $OCF_SUCCESS;
+    else
+        ocf_log $loglevel "MySQL not running: removing old PID file"
+        rm -f $OCF_RESKEY_pid
+        return $OCF_NOT_RUNNING;
+    fi
+}
+
+mysql_common_prepare_dirs()
+{
+    local rc
+
+    touch $OCF_RESKEY_log
+    chown $OCF_RESKEY_user:$OCF_RESKEY_group $OCF_RESKEY_log
+    chmod 0640 $OCF_RESKEY_log
+    [ -x /sbin/restorecon ] && /sbin/restorecon $OCF_RESKEY_log
+
+    if ocf_is_true "$OCF_RESKEY_enable_creation" && [ ! -d $OCF_RESKEY_datadir/mysql ] ; then
+        ocf_log info "Initializing MySQL database: "
+        $MYSQL_BINDIR/mysql_install_db --datadir=$OCF_RESKEY_datadir
+        rc=$?
+        if [ $rc -ne 0 ] ; then
+            ocf_exit_reason "Initialization failed: $rc";
+            exit $OCF_ERR_GENERIC
+        fi
+        chown -R $OCF_RESKEY_user:$OCF_RESKEY_group $OCF_RESKEY_datadir
+    fi
+
+    pid_dir=`dirname $OCF_RESKEY_pid`
+    if [ ! -d $pid_dir ] ; then
+        ocf_log info "Creating PID dir: $pid_dir"
+        mkdir -p $pid_dir
+        chown $OCF_RESKEY_user:$OCF_RESKEY_group $pid_dir
+    fi
+
+    socket_dir=`dirname $OCF_RESKEY_socket`
+    if [ ! -d $socket_dir ] ; then
+        ocf_log info "Creating socket dir: $socket_dir"
+        mkdir -p $socket_dir
+        chown $OCF_RESKEY_user:$OCF_RESKEY_group $socket_dir
+    fi
+
+    # Regardless of whether we just created the directory or it
+    # already existed, check whether it is writable by the configured
+    # user
+    for dir in $pid_dir $socket_dir; do
+        if ! su -s /bin/sh - $OCF_RESKEY_user -c "test -w $dir"; then
+            ocf_exit_reason "Directory $dir is not writable by $OCF_RESKEY_user"
+            exit $OCF_ERR_PERM;
+        fi
+    done
+}
+
+mysql_common_start()
+{
+    local mysql_extra_params="$1"
+    local pid
+
+    ${OCF_RESKEY_binary} --defaults-file=$OCF_RESKEY_config \
+    --pid-file=$OCF_RESKEY_pid \
+    --socket=$OCF_RESKEY_socket \
+    --datadir=$OCF_RESKEY_datadir \
+    --log-error=$OCF_RESKEY_log \
+    --user=$OCF_RESKEY_user $OCF_RESKEY_additional_parameters \
+    $mysql_extra_params >/dev/null 2>&1 &
+    pid=$!
+
+    # Spin waiting for the server to come up.
+    # Let the CRM/LRM time us out if required.
+    start_wait=1
+    while [ $start_wait = 1 ]; do
+        if ! ps $pid > /dev/null 2>&1; then
+            wait $pid
+            ocf_exit_reason "MySQL server failed to start (pid=$pid) (rc=$?), please check your installation"
+            return $OCF_ERR_GENERIC
+        fi
+        mysql_common_status info
+        rc=$?
+        if [ $rc = $OCF_SUCCESS ]; then
+            start_wait=0
+        elif [ $rc != $OCF_NOT_RUNNING ]; then
+            ocf_log info "MySQL start failed: $rc"
+            return $rc
+        fi
+        sleep 2
+    done
+
+    return $OCF_SUCCESS
+}
+
+mysql_common_stop()
+{
+    local pid
+    local rc
+
+    if [ ! -f $OCF_RESKEY_pid ]; then
+        ocf_log info "MySQL is not running"
+        return $OCF_SUCCESS
+    fi
+
+    pid=`cat $OCF_RESKEY_pid 2> /dev/null `
+    /bin/kill $pid > /dev/null
+    rc=$?
+    if [ $rc != 0 ]; then
+        ocf_exit_reason "MySQL couldn't be stopped"
+        return $OCF_ERR_GENERIC
+    fi
+    # stop waiting
+    shutdown_timeout=15
+    if [ -n "$OCF_RESKEY_CRM_meta_timeout" ]; then
+        shutdown_timeout=$((($OCF_RESKEY_CRM_meta_timeout/1000)-5))
+    fi
+    count=0
+    while [ $count -lt $shutdown_timeout ]
+    do
+        mysql_common_status info $pid
+        rc=$?
+        if [ $rc = $OCF_NOT_RUNNING ]; then
+            break
+        fi
+        count=`expr $count + 1`
+        sleep 1
+        ocf_log debug "MySQL still hasn't stopped yet. Waiting..."
+    done
+
+    mysql_common_status info $pid
+    if [ $? != $OCF_NOT_RUNNING ]; then
+        ocf_log info "MySQL failed to stop after ${shutdown_timeout}s using SIGTERM. Trying SIGKILL..."
+        /bin/kill -KILL $pid > /dev/null
+    fi
+
+    ocf_log info "MySQL stopped";
+    rm -f /var/lock/subsys/mysqld
+    rm -f $OCF_RESKEY_socket
+    return $OCF_SUCCESS
+
+}
diff --git a/heartbeat/named b/heartbeat/named
index ede22df..2c34a15 100755
--- a/heartbeat/named
+++ b/heartbeat/named
@@ -211,21 +211,21 @@ named_validate_all() {
         if ocf_is_probe; then
            ocf_log info "Configuration file ${OCF_RESKEY_named_rootdir}/${OCF_RESKEY_named_config} not readable during probe."
         else
-           ocf_log err "Configuration file ${OCF_RESKEY_named_rootdir}/${OCF_RESKEY_named_config} doesn't exist"
+           ocf_exit_reason "Configuration file ${OCF_RESKEY_named_rootdir}/${OCF_RESKEY_named_config} doesn't exist"
            return $OCF_ERR_INSTALLED
         fi
     fi
 
     getent passwd $OCF_RESKEY_named_user >/dev/null 2>&1
     if [ ! $? -eq 0 ]; then
-        ocf_log err "User $OCF_RESKEY_named_user doesn't exist";
+        ocf_exit_reason "User $OCF_RESKEY_named_user doesn't exist";
         return $OCF_ERR_INSTALLED;
     fi
 
     if [ -z "$OCF_RESKEY_monitor_request" -o \
          -z "$OCF_RESKEY_monitor_response" -o \
          -z "$OCF_RESKEY_monitor_ip" ]; then
-        ocf_log err "None of monitor_request, monitor_response, and monitor_ip can be empty"
+        ocf_exit_reason "None of monitor_request, monitor_response, and monitor_ip can be empty"
         return $OCF_ERR_CONFIGURED
     fi                            
          
@@ -309,7 +309,7 @@ named_monitor() {
 
     if [ $? -ne 0 ] || ! echo $output | grep -q '.* has .*address '"$OCF_RESKEY_monitor_response" 
     then
-       ocf_log err "named didn't answer properly for $OCF_RESKEY_monitor_request."
+       ocf_exit_reason "named didn't answer properly for $OCF_RESKEY_monitor_request."
        ocf_log err "Expected: $OCF_RESKEY_monitor_response."
        ocf_log err "Got: $output"
        return $OCF_ERR_GENERIC
@@ -356,7 +356,7 @@ named_start() {
 
     if ! ${OCF_RESKEY_named} -u ${OCF_RESKEY_named_user} $root_dir_opt ${OCF_RESKEY_named_options}
     then
-        ocf_log err "named failed to start."
+        ocf_exit_reason "named failed to start."
         return $OCF_ERR_GENERIC
     fi
     
@@ -368,7 +368,7 @@ named_start() {
             echo $pid > ${OCF_RESKEY_named_pidfile}
         fi
     else
-        ocf_log err "named failed to start. Probably error in configuration."
+        ocf_exit_reason "named failed to start. Probably error in configuration."
         return $OCF_ERR_GENERIC
     fi
         
@@ -420,7 +420,7 @@ named_stop () {
     
     #If still up    
     if named_status 2>&1; then
-        ocf_log err "named is still up! Killing"
+        ocf_exit_reason "named is still up! Killing"
         kill -9 `cat ${OCF_RESKEY_named_pidfile}`
     fi
     
@@ -460,7 +460,7 @@ then
 fi
 
 if [ `id -u` -ne 0 ]; then
-    ocf_log err "$0 must be run as root"
+    ocf_exit_reason "$0 must be run as root"
     exit $OCF_ERR_GENERIC
 fi
 
diff --git a/heartbeat/nfsnotify b/heartbeat/nfsnotify
new file mode 100755
index 0000000..5f72d58
--- /dev/null
+++ b/heartbeat/nfsnotify
@@ -0,0 +1,315 @@
+#!/bin/bash
+#
+# Copyright (c) 2014 David Vossel <dvossel at redhat.com>
+#                    All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of version 2 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# Further, this software is distributed without any warranty that it is
+# free of the rightful claim of any third person regarding infringement
+# or the like.  Any license provided herein, whether implied or
+# otherwise, applies only to this software file.  Patent licenses, if
+# any, provided herein do not apply to combinations of this program with
+# other software, or any other product whatsoever.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
+#
+
+#######################################################################
+# Initialization:
+
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+. ${OCF_FUNCTIONS_DIR}/ocf-directories
+
+#######################################################################
+
+sbindir=$HA_SBIN_DIR
+if [ -z "$sbindir" ]; then
+	sbindir=/usr/sbin
+fi
+
+SELINUX_ENABLED=-1
+
+NFSNOTIFY_TMP_DIR="${HA_RSCTMP}/nfsnotify_${OCF_RESOURCE_INSTANCE}/"
+HA_STATD_PIDFILE="$NFSNOTIFY_TMP_DIR/rpc.statd_${OCF_RESOURCE_INSTANCE}.pid"
+HA_STATD_PIDFILE_PREV="$NFSNOTIFY_TMP_DIR/rpc.statd_${OCF_RESOURCE_INSTANCE}.pid.prev"
+STATD_PATH="/var/lib/nfs/statd"
+SM_NOTIFY_BINARY="${sbindir}/sm-notify"
+IS_RENOTIFY=0
+
+meta_data() {
+	cat <<END
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="nfsnotify" version="0.9">
+<version>1.0</version>
+
+<longdesc lang="en">
+This agent sends NFSv3 reboot notifications to clients which informs clients to reclaim locks.
+</longdesc>
+<shortdesc lang="en">sm-notify reboot notifications</shortdesc>
+
+<parameters>
+
+<parameter name="source_host" unique="0" required="0">
+<longdesc lang="en">
+Comma separated list of floating IP addresses or host names that clients use
+to access the nfs service.  This will be used to set the source address and
+mon_name of the SN_NOTIFY reboot notifications.
+</longdesc>
+<shortdesc lang="en">source IP addresses</shortdesc>
+<content type="string" default="" />
+</parameter>
+
+<parameter name="notify_args" unique="0" required="0">
+<longdesc lang="en">
+Additional arguments to send to the sm-notify command. By default
+this agent will always set sm-notify's '-f' option.  When the
+source_host option is set, the '-v' option will be used automatically
+to set the proper source address. Any additional sm-notify arguments
+set with this option will be used in addition to the previous default
+arguments.
+</longdesc>
+<shortdesc lang="en">sm-notify arguments</shortdesc>
+<content type="string" default="false" />
+</parameter>
+
+</parameters>
+
+<actions>
+<action name="start"        timeout="90" />
+<action name="stop"         timeout="90" />
+<action name="monitor"      timeout="90" interval="30" depth="0" />
+<action name="reload"       timeout="90" />
+<action name="meta-data"    timeout="10" />
+<action name="validate-all"   timeout="20" />
+</actions>
+</resource-agent>
+END
+}
+
+v3notify_usage()
+{
+	cat <<END
+usage: $0 {start|stop|monitor|validate-all|meta-data}
+
+Expects to have a fully populated OCF RA-compliant environment set.
+END
+}
+
+v3notify_validate()
+{
+	# check_binary will exit with OCF_ERR_INSTALLED when binary is missing
+	check_binary "$SM_NOTIFY_BINARY"
+	check_binary "pgrep"
+	check_binary "killall"
+
+	return $OCF_SUCCESS
+}
+
+killall_smnotify()
+{
+	# killall sm-notify 
+	killall -TERM $SM_NOTIFY_BINARY > /dev/null 2>&1
+	if [ $? -eq 0 ]; then
+		# it is useful to know if sm-notify processes were actually left around
+		# or not during the stop/start operation. Whether this condition is true
+		# or false does not indicate a failure. It does indicate that 
+		# there are probably some unresponsive nfs clients out there that are keeping
+		# the sm-notify processes retrying.
+		ocf_log info "previous sm-notify processes terminated before $__OCF_ACTION action."
+	fi
+}
+
+v3notify_stop()
+{
+	killall_smnotify
+
+	rm -f $HA_STATD_PIDFILE_PREV > /dev/null 2>&1
+	mv $HA_STATD_PIDFILE $HA_STATD_PIDFILE_PREV > /dev/null 2>&1
+
+	return $OCF_SUCCESS
+}
+
+check_statd_pidfile()
+{
+	local binary="rpc.statd"
+	local pidfile="$HA_STATD_PIDFILE"
+
+	ocf_log debug "Checking status for ${binary}."
+	if [ -e "$pidfile" ]; then
+		cat /proc/$(cat $pidfile)/cmdline 2>/dev/null | grep -a "${binary}" > /dev/null 2>&1
+		if [ $? -eq 0 ]; then
+			return $OCF_SUCCESS
+		fi
+
+		ocf_exit_reason "$(cat $pidfile) for $binary is no longer running, sm-notify needs to re-notify clients"
+		return $OCF_ERR_GENERIC
+	fi
+
+	# if we don't have a pid file for rpc.statd, we have not yet sent the notifications
+	return $OCF_NOT_RUNNING
+}
+
+write_statd_pid()
+{
+	local binary="rpc.statd"
+	local pidfile="$HA_STATD_PIDFILE"
+	local pid
+
+	pid=$(pgrep ${binary})
+	case $? in
+		0)
+			ocf_log info "PID file (pid:${pid} at $pidfile) created for ${binary}."
+			mkdir -p $(dirname $pidfile)
+			echo "$pid" > $pidfile
+			return $OCF_SUCCESS;;
+		1)
+			rm -f "$pidfile" > /dev/null 2>&1 
+			ocf_log info "$binary is not running"
+			return $OCF_NOT_RUNNING;;
+		*)
+			rm -f "$pidfile" > /dev/null 2>&1 
+		  	ocf_exit_reason "Error encountered detecting pid status of $binary"
+			return $OCF_ERR_GENERIC;;
+	esac
+}
+
+copy_statd()
+{
+	local src=$1
+	local dest=$2
+
+	if ! [ -d "$dest" ]; then
+		mkdir -p "$dest"
+	fi
+
+	cp -rpfn $src/sm $src/sm.bak $src/state $dest > /dev/null 2>&1
+
+	# make sure folder ownership and selinux lables stay consistent
+	[ -n "`id -u rpcuser`" -a "`id -g rpcuser`" ] && chown rpcuser.rpcuser "$dest"
+	[ $SELINUX_ENABLED -eq 0 ] && chcon -R "$SELINUX_LABEL" "$dest"
+}
+
+v3notify_start()
+{
+	local rc=$OCF_SUCCESS
+	local cur_statd
+	local statd_backup
+	local is_renotify=0
+
+	# monitor, see if we need to notify or not
+	v3notify_monitor
+	if [ $? -eq 0 ]; then
+		return $OCF_SUCCESS
+	fi
+
+	# kill off any other sm-notify processes that might already be running.
+	killall_smnotify
+
+	# record the pid of rpc.statd. if this pid ever changes, we have to re-notify
+	write_statd_pid
+	rc=$?
+	if [ $rc -ne 0 ]; then
+		return $rc
+	fi
+
+	# if the last time we ran nfs-notify, it was with the same statd process,
+	# consider this a re-notification. During re-notifications we do not let the
+	# sm-notify binary have access to the real statd directory.
+	if [ "$(cat $HA_STATD_PIDFILE)" = "$(cat $HA_STATD_PIDFILE_PREV 2>/dev/null)" ]; then
+		ocf_log info "Renotifying clients"
+		is_renotify=1
+	fi
+
+	statd_backup="$STATD_PATH/nfsnotify.bu"
+	copy_statd "$STATD_PATH" "$statd_backup"
+
+	if [ -z "$OCF_RESKEY_source_host" ]; then
+		if [ "$is_renotify" -eq 0 ]; then
+			cur_statd="$STATD_PATH"
+		else 
+			cur_statd="$statd_backup"
+		fi
+		ocf_log info "sending notifications on default source address."
+		$SM_NOTIFY_BINARY -f $OCF_RESKEY_notify_args -P $cur_statd
+		if [ $? -ne 0 ]; then
+			ocf_exit_reason "sm-notify execution failed, view syslog for more information"
+			return $OCF_ERR_GENERIC
+		fi
+		
+		return $OCF_SUCCESS
+	fi
+
+	# do sm-notify for each ip
+	for ip in `echo ${OCF_RESKEY_source_host} | sed 's/,/ /g'`; do
+
+		# have the first sm-notify use the actual statd directory so the
+		# notify list can be managed properly.
+		if [ "$is_renotify" -eq 0 ]; then
+			cur_statd="$STATD_PATH"
+			# everything after the first notify we are considering a renotification
+			# which means we don't use the real statd directory. 
+			is_renotify=1
+		else 
+			# use our copied statd directory for the remaining ip addresses
+			cur_statd="$STATD_PATH/nfsnotify_${OCF_RESOURCE_INSTANCE}_${ip}"
+			copy_statd "$statd_backup" "$cur_statd"
+		fi
+
+		ocf_log info "sending notifications with source address $ip"
+		$SM_NOTIFY_BINARY -f $OCF_RESKEY_notify_args -v $ip -P "$cur_statd"
+		if [ $? -ne 0 ]; then
+			ocf_exit_reason "sm-notify with source host set to [ $ip ] failed. view syslog for more information"
+			return $OCF_ERR_GENERIC
+		fi
+	done
+
+	return $OCF_SUCCESS
+}
+
+v3notify_monitor()
+{
+	# verify rpc.statd is up, and that the rpc.statd pid is the same one we
+	# found during the start. otherwise rpc.statd recovered and we need to notify
+	# again.
+	check_statd_pidfile
+}
+
+case $__OCF_ACTION in
+	meta-data)   meta_data
+		exit $OCF_SUCCESS;;
+	usage|help)    v3notify_usage
+		exit $OCF_SUCCESS;;
+	*)
+		;;
+esac
+
+which restorecon > /dev/null 2>&1 && selinuxenabled
+SELINUX_ENABLED=$?
+if [ $SELINUX_ENABLED -eq 0 ]; then
+	export SELINUX_LABEL="$(ls -ldZ $STATD_PATH | cut -f4 -d' ')"
+fi
+
+case $__OCF_ACTION in
+	start)         v3notify_start;;
+	stop)          v3notify_stop;;
+	monitor)       v3notify_monitor;;
+	validate-all)  v3notify_validate;;
+	*)             v3notify_usage
+	               exit $OCF_ERR_UNIMPLEMENTED;;
+esac
+
+rc=$?
+ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION : $rc"
+exit $rc
+
diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver
index fdb0831..cb8635f 100755
--- a/heartbeat/nfsserver
+++ b/heartbeat/nfsserver
@@ -13,6 +13,11 @@ else
 fi
 
 DEFAULT_INIT_SCRIPT="/etc/init.d/nfsserver"
+if ! [ -f $DEFAULT_INIT_SCRIPT ]; then
+	# On some systems, the script is just called nfs
+	DEFAULT_INIT_SCRIPT="/etc/init.d/nfs"
+fi
+
 DEFAULT_NOTIFY_CMD=`which sm-notify`
 DEFAULT_NOTIFY_CMD=${DEFAULT_NOTIFY_CMD:-"/sbin/sm-notify"}
 DEFAULT_NOTIFY_FOREGROUND="false"
@@ -21,6 +26,9 @@ EXEC_MODE=0
 SELINUX_ENABLED=-1
 STATD_PATH="/var/lib/nfs"
 STATD_DIR=""
+NFS_SYSCONFIG="/etc/sysconfig/nfs"
+NFS_SYSCONFIG_LOCAL_BACKUP="/etc/sysconfig/nfs.ha.bu"
+NFS_SYSCONFIG_AUTOGEN_TAG="AUTOGENERATED by $0 high availability resource-agent"
 
 nfsserver_meta_data() {
 	cat <<END
@@ -54,21 +62,19 @@ Init script for nfsserver
 <content type="string" default="auto detected" />
 </parameter>
 
-<parameter name="nfs_notify_cmd" unique="0" required="0">
+<parameter name="nfs_no_notify" unique="0" required="0">
 <longdesc lang="en">
-The tool to send out NSM reboot notification; it should be either sm-notify or rpc.statd.
-Failover of nfsserver can be considered as rebooting to different machines.
-The nfsserver resource agent use this command to notify all clients about the occurrence of failover.
+Do not send reboot notifications to NFSv3 clients during server startup.
 </longdesc>
 <shortdesc lang="en">
-The tool to send out notification.
+Disable NFSv3 server reboot notifications
 </shortdesc>
-<content type="string" default="$DEFAULT_NOTIFY_CMD" />
+<content type="boolean" default="false" />
 </parameter>
 
 <parameter name="nfs_notify_foreground" unique="0" required="0">
 <longdesc lang="en">
-Keeps the notify tool attached to its controlling terminal and running in the foreground.
+Keeps the sm-notify attached to its controlling terminal and running in the foreground.
 </longdesc>
 <shortdesc lang="en">
 Keeps the notify tool running in the foreground.
@@ -88,25 +94,102 @@ Specifies the length of sm-notify retry time (minutes).
 <content type="integer" default="" />
 </parameter>
 
-<parameter name="nfs_shared_infodir" unique="0" required="1">
+<parameter name="nfs_ip" unique="0" required="0">
 <longdesc lang="en">
-The nfsserver resource agent will save nfs related information in this specific directory.
-And this directory must be able to fail-over before nfsserver itself.
+Comma separated list of floating IP addresses used to access the nfs service
 </longdesc>
 <shortdesc lang="en">
-Directory to store nfs server related information.
+IP addresses.
 </shortdesc>
-<content type="string" default="" />
+<content type="string"/>
 </parameter>
 
-<parameter name="nfs_ip" unique="0" required="1">
+<parameter name="nfsd_args" unique="0" required="0">
 <longdesc lang="en">
-Comma separated list of floating IP addresses used to access the nfs service
+Specifies what arguments to pass to the nfs daemon on startup. View the rpc.nfsd man page for information on what arguments are available.
+Note that setting this value will override all settings placed in the local /etc/sysconfig/nfs file.
 </longdesc>
 <shortdesc lang="en">
-IP addresses.
+rpc.nfsd options
 </shortdesc>
-<content type="string"/>
+<content type="string" />
+</parameter>
+
+<parameter name="lockd_udp_port" unique="0" required="0">
+<longdesc lang="en">
+The udp port lockd should listen on.
+Note that setting this value will override all settings placed in the local /etc/sysconfig/nfs file.
+</longdesc>
+<shortdesc lang="en">
+lockd udp port
+</shortdesc>
+<content type="integer" />
+</parameter>
+
+<parameter name="lockd_tcp_port" unique="0" required="0">
+<longdesc lang="en">
+The tcp port lockd should listen on.
+Note that setting this value will override all settings placed in the local /etc/sysconfig/nfs file.
+</longdesc>
+<shortdesc lang="en">
+lockd tcp port
+</shortdesc>
+<content type="integer" />
+</parameter>
+
+<parameter name="statd_outgoing_port" unique="0" required="0">
+<longdesc lang="en">
+The source port number sm-notify uses when sending reboot notifications.
+Note that setting this value will override all settings placed in the local /etc/sysconfig/nfs file.
+</longdesc>
+<shortdesc lang="en">
+sm-notify source port
+</shortdesc>
+<content type="integer" />
+</parameter>
+
+<parameter name="statd_port" unique="0" required="0">
+<longdesc lang="en">
+The port number used for RPC listener sockets.
+Note that setting this value will override all settings placed in the local /etc/sysconfig/nfs file.
+</longdesc>
+<shortdesc lang="en">
+rpc.statd listener port
+</shortdesc>
+<content type="integer" />
+</parameter>
+
+<parameter name="mountd_port" unique="0" required="0">
+<longdesc lang="en">
+The port number used for rpc.mountd listener sockets.
+Note that setting this value will override all settings placed in the local /etc/sysconfig/nfs file.
+</longdesc>
+<shortdesc lang="en">
+rpc.mountd listener port
+</shortdesc>
+<content type="integer" />
+</parameter>
+
+<parameter name="rquotad_port" unique="0" required="0">
+<longdesc lang="en">
+The port number used for rpc.rquotad.
+Note that setting this value will override all settings placed in the local /etc/sysconfig/nfs file.
+</longdesc>
+<shortdesc lang="en">
+rpc.rquotad port
+</shortdesc>
+<content type="integer" />
+</parameter>
+
+<parameter name="nfs_shared_infodir" unique="0" required="0">
+<longdesc lang="en">
+The nfsserver resource agent will save nfs related information in this specific directory.
+And this directory must be able to fail-over before nfsserver itself.
+</longdesc>
+<shortdesc lang="en">
+Directory to store nfs server related information.
+</shortdesc>
+<content type="string" default="" />
 </parameter>
 
 <parameter name="rpcpipefs_dir" unique="0" required="0">
@@ -229,41 +312,15 @@ set_exec_mode()
 	if which systemctl > /dev/null 2>&1; then
 		if systemctl list-unit-files | grep nfs-server > /dev/null && systemctl list-unit-files | grep nfs-lock > /dev/null; then
 			EXEC_MODE=2
+			# when using systemd, the nfs-lock service file handles nfsv3 locking daemons for us.
 			return 0
 		fi
 	fi
 
-	ocf_log err "No init script or systemd unit file detected for nfs server"
+	ocf_exit_reason "No init script or systemd unit file detected for nfs server"
 	exit $OCF_ERR_INSTALLED
 }
 
-nfs_systemd_exec()
-{
-	local cmd=$1
-	local server_res
-	local lock_res
-
-	if [ "$cmd" = "stop" ]; then
-		systemctl $cmd nfs-server.service
-		server_res=$?
-		systemctl $cmd nfs-lock.service
-		lock_res=$?
-	else
-		systemctl $cmd nfs-lock.service
-		lock_res=$?
-		systemctl $cmd nfs-server.service
-		server_res=$?
-	fi
-
-	if [ $lock_res -ne $server_res ]; then
-		# If one is running and the other isn't, or for whatever other reason
-		# the return code's aren't the same, this is bad.
-		ocf_log err "Systemd services nfs-lock and nfs-server are not in the same state after attempting $cmd command"
-		return $OCF_ERR_GENERIC
-	fi
-	return $server_res
-}
-
 ##
 # wrapper for init script and systemd calls.
 ##
@@ -274,21 +331,45 @@ nfs_exec()
 
 	case $EXEC_MODE in 
 		1) ${OCF_RESKEY_nfs_init_script} $cmd;;
-		2) nfs_systemd_exec $cmd;;
+		2) systemctl $cmd nfs-server.service ;;
 	esac
 }
 
+v3locking_exec()
+{
+	local cmd=$1
+	set_exec_mode
+
+	if [ $EXEC_MODE -eq 2 ]; then
+		systemctl $cmd nfs-lock.service
+	else 
+		case $cmd in
+			start) locking_start;;
+			stop) locking_stop;;
+			status) locking_status;;
+		esac
+	fi
+}
+
 nfsserver_monitor ()
 {
 	fn=`mktemp`
 	nfs_exec status > $fn 2>&1 
 	rc=$?
-	ocf_log debug `cat $fn`
+	ocf_log debug "$(cat $fn)"
 	rm -f $fn
 
-#Adapte LSB status code to OCF return code
+	#Adapte LSB status code to OCF return code
 	if [ $rc -eq 0 ]; then
-		return $OCF_SUCCESS
+		# don't report success if nfs servers are up
+		# without locking daemons.
+		v3locking_exec "status"
+		rc=$?
+		if [ $rc -ne 0 ]; then
+			ocf_exit_reason "NFS server is up, but the locking daemons are down"
+			rc=$OCF_ERR_GENERIC
+		fi
+		return $rc
 	elif [ $rc -eq 3 ]; then
 		return $OCF_NOT_RUNNING
 	else
@@ -296,8 +377,81 @@ nfsserver_monitor ()
 	fi
 }
 
+set_arg()
+{
+	local key="$1"
+	local value="$2"
+	local file="$3"
+	local requires_sysconfig="$4"
+
+	if [ -z "$value" ]; then
+		return
+	fi
+
+	# only write to the tmp /etc/sysconfig/nfs if sysconfig exists.
+	# otherwise this distro does not support setting these options.
+	if [ -d "/etc/sysconfig" ]; then
+		echo "${key}=\"${value}\"" >> $file
+	elif [ "$requires_sysconfig" = "true" ]; then
+		ocf_log warn "/etc/sysconfig/nfs not found, unable to set port and nfsd args."
+	fi
+
+	export ${key}="${value}"
+}
+
+set_env_args()
+{
+	local tmpconfig=$(mktemp ${HA_RSCTMP}/nfsserver-tmp-XXXXX)
+	local statd_args
+
+	# nfsd args
+	set_arg "RPCNFSDARGS" "$OCF_RESKEY_nfsd_args" "$tmpconfig" "true"
+
+	# mountd args
+	if [ -n "$OCF_RESKEY_mountd_port" ]; then
+		set_arg "RPCMOUNTDOPTS" "-p $OCF_RESKEY_mountd_port" "$tmpconfig" "true"
+	fi
+
+	# statd args. we always want to perform the notify using sm-notify after
+	# both rpc.statd and the nfsd daemons are initialized
+	statd_args="--no-notify"
+	if [ -n "$OCF_RESKEY_statd_outgoing_port" ]; then
+		statd_args="$statd_args -o $OCF_RESKEY_statd_outgoing_port"
+	fi
+	if [ -n "$OCF_RESKEY_statd_port" ]; then
+		statd_args="$statd_args -p $OCF_RESKEY_statd_port"
+	fi
+	set_arg "STATDARG" "$statd_args" "$tmpconfig" "false"
+
+	# lockd ports
+	set_arg "LOCKD_UDPPORT" "$OCF_RESKEY_lockd_udp_port" "$tmpconfig" "true"
+	set_arg "LOCKD_TCPPORT" "$OCF_RESKEY_lockd_tcp_port" "$tmpconfig" "true"
+
+	# rquotad_port
+	if [ -n "$OCF_RESKEY_rquotad_port" ]; then
+		set_arg "RPCRQUOTADOPTS" "-p $OCF_RESKEY_rquotad_port" "$tmpconfig" "true"
+	fi
+
+	# override local nfs config. preserve previous local config though.
+	if [ -s $tmpconfig ]; then
+		cat $NFS_SYSCONFIG | grep -e "$NFS_SYSCONFIG_AUTOGEN_TAG"
+		if [ $? -ne 0 ]; then
+			# backup local nfs config if it doesn't have our HA autogen tag in it.
+			mv -f $NFS_SYSCONFIG $NFS_SYSCONFIG_LOCAL_BACKUP
+		fi
+		echo "# $NFS_SYSCONFIG_AUTOGEN_TAG" > $NFS_SYSCONFIG
+		echo "# local config backup stored here, '$NFS_SYSCONFIG_LOCAL_BACKUP'" >> $NFS_SYSCONFIG
+		cat $tmpconfig >> $NFS_SYSCONFIG
+	fi
+	rm -f $tmpconfig
+}
+
 prepare_directory ()
 {
+	if [ -z "$fp" ]; then
+		return
+	fi
+
 	[ -d "$fp" ] || mkdir -p $fp
 	[ -d "$rpcpipefs_make_dir" ] || mkdir -p $rpcpipefs_make_dir
 	[ -d "$fp/v4recovery" ] || mkdir -p $fp/v4recovery
@@ -306,12 +460,15 @@ prepare_directory ()
 	[ -d "$fp/$STATD_DIR/sm" ] || mkdir -p "$fp/$STATD_DIR/sm"
 	[ -d "$fp/$STATD_DIR/sm.ha" ] || mkdir -p "$fp/$STATD_DIR/sm.ha"
 	[ -d "$fp/$STATD_DIR/sm.bak" ] || mkdir -p "$fp/$STATD_DIR/sm.bak"
-	[ -n "`id -u rpcuser`" -a "`id -g rpcuser`" ] && chown -R rpcuser.rpcuser "$fp/$STATD_DIR"
+	[ -n "`id -u rpcuser 2>/dev/null`" -a "`id -g rpcuser 2>/dev/null`" ] &&
+		chown -R rpcuser.rpcuser "$fp/$STATD_DIR"
 
 	[ -f "$fp/etab" ] || touch "$fp/etab"
 	[ -f "$fp/xtab" ] || touch "$fp/xtab"
 	[ -f "$fp/rmtab" ] || touch "$fp/rmtab"
 
+	dd if=/dev/urandom of=$fp/$STATD_DIR/state bs=1 count=4 >/dev/null 2>&1
+	[ -n "`id -u rpcuser`" -a "`id -g rpcuser`" ] && chown rpcuser.rpcuser "$fp/$STATD_DIR/state"
 	[ $SELINUX_ENABLED -eq 0 ] && chcon -R "$SELINUX_LABEL" "$fp"
 }
 
@@ -326,6 +483,10 @@ is_bound ()
 
 bind_tree ()
 {
+	if [ -z "$fp" ]; then
+		return
+	fi
+
 	if is_bound /var/lib/nfs; then
 		ocf_log debug "$fp is already bound to /var/lib/nfs"
 		return 0
@@ -344,68 +505,203 @@ unbind_tree ()
 	fi
 }
 
+binary_status()
+{
+	local binary=$1
+	local pid
+
+	pid=$(pgrep ${binary})
+	case $? in
+		0)
+			echo "$pid"
+			return $OCF_SUCCESS;;
+		1)
+			return $OCF_NOT_RUNNING;;
+		*)
+			return $OCF_ERR_GENERIC;;
+	esac
+}
+
+locking_status()
+{
+	binary_status "rpc.statd" > /dev/null 2>&1
+}
+
+locking_start()
+{
+	local ret=$OCF_SUCCESS
+
+	ocf_log info "Starting rpc.statd."
+
+	rpc.statd $STATDARG
+
+	ret=$?
+	if [ $ret -ne 0 ]; then
+		ocf_log err "Failed to start rpc.statd"
+		return $ret
+	fi
+	touch /var/lock/subsys/nfslock
+
+	return $ret
+}
+
+terminate()
+{
+	local pids
+	local i=0
+
+	while : ; do
+		pids=$(binary_status $1)
+		[ -z "$pids" ] && return 0
+	 	kill $pids
+		sleep 1
+		i=$((i + 1))
+		[ $i -gt 3 ] && return 1
+	done
+}
+
+
+killkill()
+{
+	local pids
+	local i=0
+
+	while : ; do
+		pids=$(binary_status $1)
+		[ -z "$pids" ] && return 0
+	 	kill -9 $pids
+		sleep 1
+		i=$((i + 1))
+		[ $i -gt 3 ] && return 1
+	done
+}
+
+stop_process()
+{
+	local process=$1
+
+	ocf_log info "Stopping $process"
+	if terminate $process; then
+		ocf_log debug "$process is stopped"
+	else
+		if killkill $process; then
+			ocf_log debug "$process is stopped"
+		else
+			ocf_log debug "Failed to stop $process"
+			return 1
+		fi
+	fi
+	return 0
+}
+
+locking_stop()
+{
+	ret=0
+
+	# sm-notify can prevent umount of /var/lib/nfs/statd if
+	# it is still trying to notify unresponsive clients.
+	stop_process sm-notify
+	if [ $? -ne 0 ]; then
+		ret=$OCF_ERR_GENERIC
+	fi
+
+	stop_process rpc.statd
+	if [ $? -ne 0 ]; then
+		ret=$OCF_ERR_GENERIC
+	fi
+
+	return $ret
+}
+
+notify_locks()
+{
+	if ocf_is_true "$OCF_RESKEY_nfs_no_notify"; then
+		# we've been asked not to notify clients
+		return;
+	fi
+
+	# run in foreground, if requested
+	if ocf_is_true "$OCF_RESKEY_nfs_notify_foreground"; then
+		opts="-d"
+	fi
+
+	if [ -n "$OCF_RESKEY_nfs_smnotify_retry_time" ]; then
+		opts="$opts -m $OCF_RESKEY_nfs_smnotify_retry_time"
+	fi
+
+	if [ -n "$OCF_RESKEY_statd_outgoing_port" ]; then
+		opts="$opts -p $OCF_RESKEY_statd_outgoing_port"
+	fi
+
+	# forces re-notificaiton regardless if notifies have already gone out
+	opts="$opts -f"
+
+	ocf_log info "executing sm-notify"
+	if [ -n "$OCF_RESKEY_nfs_ip" ]; then
+		for ip in `echo ${OCF_RESKEY_nfs_ip} | sed 's/,/ /g'`; do
+			cp -rpfn $STATD_PATH/sm.ha/* $STATD_PATH/  > /dev/null 2>&1
+			sm-notify $opts -v $ip
+		done
+	else
+		sm-notify $opts
+	fi
+}
+
 nfsserver_start ()
 {
+	local rc;
+
 	if nfsserver_monitor; then
 		ocf_log debug "NFS server is already started"
 		return $OCF_SUCCESS
 	fi
 
+	set_env_args
 	prepare_directory
 	bind_tree
 
+	# remove the sm-notify pid so sm-notify will be allowed to run again without requiring a reboot.
+	rm -f /var/run/sm-notify.pid
+	#
+	# Synchronize these before starting statd
+	#
+	cp -rpfn $STATD_PATH/sm.ha/* $STATD_PATH/ > /dev/null 2>&1
 	rm -rf $STATD_PATH/sm.ha/* > /dev/null 2>&1
-	cp -rf $STATD_PATH/sm $STATD_PATH/sm.bak /var/lib/nfs/state $STATD_PATH/sm.ha > /dev/null 2>&1
+	cp -rpf $STATD_PATH/sm $STATD_PATH/sm.bak /var/lib/nfs/state $STATD_PATH/sm.ha > /dev/null 2>&1
 
 	ocf_log info "Starting NFS server ..."
 
+	# mounts /proc/fs/nfsd for us
+	lsmod | grep -q nfsd
+	if [ $? -ne 0 ]; then
+		modprobe nfsd
+	fi
+
+	# check to see if we need to start rpc.statd
+	v3locking_exec "status"
+	if [ $? -ne $OCF_SUCCESS ]; then
+		v3locking_exec "start"
+		rc=$?
+		if [ $rc -ne 0 ]; then
+			ocf_exit_reason "Failed to start NFS server locking daemons"
+			return $rc
+		fi
+	else
+		ocf_log info "rpc.statd already up"
+	fi
+
 	fn=`mktemp`
 	nfs_exec start > $fn 2>&1
 	rc=$?
-	ocf_log debug `cat $fn`
+	ocf_log debug "$(cat $fn)"
 	rm -f $fn
 
 	if [ $rc -ne 0 ]; then
-		ocf_log err "Failed to start NFS server"
+		ocf_exit_reason "Failed to start NFS server"
 		return $rc
 	fi	
 
-	#Notify the nfs server has been moved or rebooted
-	#The init script do that already, but with the hostname, which may be ignored by client
-	#we have to do it again with the nfs_ip 
-	local opts
-
-	case ${OCF_RESKEY_nfs_notify_cmd##*/} in 
-	sm-notify)
-		# run in foreground, if requested
-		if ocf_is_true "$OCF_RESKEY_nfs_notify_foreground"; then
-			opts="-d"
-		fi
-
-		if [ -n "$OCF_RESKEY_nfs_smnotify_retry_time" ]; then
-			opts="$opts -m $OCF_RESKEY_nfs_smnotify_retry_time"
-		fi
-
-		opts="$opts -f -v"
-		;;
-
-	rpc.statd)
-		if ocf_is_true "$OCF_RESKEY_nfs_notify_foreground"; then
-			opts="-F"
-		fi
-		opts="$opts -n"
-		;;
-
-	esac
-
-	rm -rf $STATD_PATH/sm.ha.save > /dev/null 2>&1
-	cp -rf $STATD_PATH/sm.ha $STATD_PATH/sm.ha.save > /dev/null 2>&1
-	for ip in `echo ${OCF_RESKEY_nfs_ip} | sed 's/,/ /g'`; do
-	  ${OCF_RESKEY_nfs_notify_cmd} $opts $ip -P $STATD_PATH/sm.ha
-	  rm -rf $STATD_PATH/sm.ha > /dev/null 2>&1
-	  cp -rf $STATD_PATH/sm.ha.save $STATD_PATH/sm.ha > /dev/null 2>&1
-	done
-
+	notify_locks
 
 	ocf_log info "NFS server started"
 	return $OCF_SUCCESS
@@ -415,18 +711,29 @@ nfsserver_stop ()
 {
 	ocf_log info "Stopping NFS server ..."
 
+	# backup the current sm state information to the ha folder before stopping.
+	# the ha folder will be synced after startup, restoring the statd client state
+	rm -rf $STATD_PATH/sm.ha/* > /dev/null 2>&1
+	cp -rpf $STATD_PATH/sm $STATD_PATH/sm.bak /var/lib/nfs/state $STATD_PATH/sm.ha > /dev/null 2>&1
+
 	fn=`mktemp`
 	nfs_exec stop > $fn 2>&1
 	rc=$?
-	ocf_log debug `cat $fn`
+	ocf_log debug "$(cat $fn)"
 	rm -f $fn
 
+	v3locking_exec "stop"
+	if [ $? -ne 0 ]; then
+		ocf_exit_reason "Failed to stop NFS locking daemons"
+		rc=$OCF_ERR_GENERIC
+	fi
+
 	if [ $rc -eq 0 ]; then
 		unbind_tree 
 		ocf_log info "NFS server stopped"
-		return $OCF_SUCCESS
+	else 
+		ocf_exit_reason "Failed to stop NFS server"
 	fi
-	ocf_log err "Failed to stop NFS server"
 	return $rc
 }
 
@@ -438,19 +745,15 @@ nfsserver_validate ()
 	set_exec_mode
 	check_binary ${OCF_RESKEY_nfs_notify_cmd}
 
-	if [ x = x"${OCF_RESKEY_nfs_ip}" ]; then
-		ocf_log err "nfs_ip not set"
-		exit $OCF_ERR_CONFIGURED
-	fi
 
-	if [ x = "x$OCF_RESKEY_nfs_shared_infodir" ]; then
-		ocf_log err "nfs_shared_infodir not set"
+	if [ -n "$OCF_RESKEY_CRM_meta_clone" ] && [ -n "$OCF_RESKEY_nfs_shared_infodir" ]; then
+		ocf_exit_reason "This RA does not support clone mode when a shared info directory is in use."
 		exit $OCF_ERR_CONFIGURED
 	fi
 
 	if [ -n "$OCF_RESKEY_nfs_smnotify_retry_time" ]; then
 		if ! ocf_is_decimal "$OCF_RESKEY_nfs_smnotify_retry_time"; then
-			ocf_log err "Invalid nfs_smnotify_retry_time [$OCF_RESKEY_nfs_smnotify_retry_time]"
+			ocf_exit_reason "Invalid nfs_smnotify_retry_time [$OCF_RESKEY_nfs_smnotify_retry_time]"
 			exit $OCF_ERR_CONFIGURED
 		fi
 	fi
@@ -458,7 +761,7 @@ nfsserver_validate ()
 	case ${OCF_RESKEY_nfs_notify_cmd##*/} in
 	sm-notify|rpc.statd) ;;
 	*)
-		ocf_log err "Invalid nfs_notify_cmd [$OCF_RESKEY_nfs_notify_cmd]"
+		ocf_exit_reason "Invalid nfs_notify_cmd [$OCF_RESKEY_nfs_notify_cmd]"
 		exit $OCF_ERR_CONFIGURED
 		;;
 	esac
@@ -466,11 +769,6 @@ nfsserver_validate ()
 	return $OCF_SUCCESS
 }
 
-if [ -n "$OCF_RESKEY_CRM_meta_clone" ]; then
-	ocf_log err "THIS RA DO NOT SUPPORT CLONE MODE!"
-	exit $OCF_ERR_CONFIGURED
-fi
-
 nfsserver_validate
 
 case $__OCF_ACTION in
diff --git a/heartbeat/nginx b/heartbeat/nginx
index 65fd8f2..a7fb5d0 100755
--- a/heartbeat/nginx
+++ b/heartbeat/nginx
@@ -31,7 +31,7 @@
 #  OCF_RESKEY_status10regex
 #  OCF_RESKEY_status10url
 #  OCF_RESKEY_client
-#  OCF_RESKEY_testurl
+#  OCF_RESKEY_test20url
 #  OCF_RESKEY_test20regex
 #  OCF_RESKEY_test20conffile
 #  OCF_RESKEY_test20name
@@ -416,7 +416,7 @@ start_nginx() {
     return $OCF_SUCCESS
   fi
   if 
-    ocf_run $NGINXD -t -c $CONFIGFILE
+    ocf_run $NGINXD $OPTIONS -t -c $CONFIGFILE
   then
     : Configuration file $CONFIGFILE looks OK
   else
@@ -442,7 +442,7 @@ start_nginx() {
       [ $ec -eq $OCF_NOT_RUNNING ]
     then
       tries=`expr $tries + 1`
-      ocf_log info "Waiting for $NGINXD -c $CONFIGFILE to come up (try $tries)"
+      ocf_log info "Waiting for $NGINXD $OPTIONS -c $CONFIGFILE to come up (try $tries)"
       true
     else
       false
@@ -727,25 +727,25 @@ For example, you can set this paramter to "wget" if you prefer that to curl.
 <content type="string" />
 </parameter>
 
-<parameter name="testurl">
+<parameter name="test20url">
 <longdesc lang="en">
 URL to test. If it does not start with "http", then it's
 considered to be relative to the document root address.
 </longdesc>
-<shortdesc lang="en">Level 10 monitor url</shortdesc>
+<shortdesc lang="en">Level 20 monitor url</shortdesc>
 <content type="string" />
 </parameter>
 
 <parameter name="test20regex">
 <longdesc lang="en">
-Regular expression to match in the output of testurl.
+Regular expression to match in the output of test20url.
 Case insensitive.
 </longdesc>
 <shortdesc lang="en">Level 20 monitor regular expression</shortdesc>
 <content type="string" />
 </parameter>
 
-<parameter name="testconffile">
+<parameter name="test20conffile">
 <longdesc lang="en">
 A file which contains a more complex test configuration. Could be useful if
 you have to check more than one web application or in case sensitive
@@ -838,11 +838,11 @@ validate_all_nginx() {
     exit $OCF_ERR_CONFIGURED
   fi
   if
-    ocf_run $NGINXD -t -c $CONFIGFILE
+    ocf_run $NGINXD $OPTIONS -t -c $CONFIGFILE
   then
     : Cool $NGINXD likes $CONFIGFILE
   else
-    ocf_log err "$NGINXD -t -c $CONFIGFILE reported a configuration error."
+    ocf_log err "$NGINXD $OPTIONS -t -c $CONFIGFILE reported a configuration error."
     return $OCF_ERR_CONFIGURED
   fi
   return $OCF_SUCCESS
@@ -859,7 +859,7 @@ then
   OPTIONS="$OCF_RESKEY_options"
   CLIENT=${OCF_RESKEY_client}
   TESTREGEX=${OCF_RESKEY_status10regex:-'Reading: [0-9]+ Writing: [0-9]+ Waiting: [0-9]+'}
-  TESTURL="$OCF_RESKEY_status10url"
+  TESTURL="$OCF_RESKEY_test20url"
   TESTREGEX20=${OCF_RESKEY_test20regex}
   TESTCONFFILE="$OCF_RESKEY_test20conffile"
   TESTNAME="$OCF_RESKEY_test20name"
diff --git a/heartbeat/ocf-binaries.in b/heartbeat/ocf-binaries.in
index a78a348..cbf70db 100644
--- a/heartbeat/ocf-binaries.in
+++ b/heartbeat/ocf-binaries.in
@@ -56,7 +56,7 @@ check_binary () {
     if ! have_binary "$1"; then
 	if [ "$OCF_NOT_RUNNING" = 7 ]; then
 	    # Chances are we have a fully setup OCF environment
-	    ocf_log err "Setup problem: couldn't find command: $1"
+	    ocf_exit_reason "Setup problem: couldn't find command: $1"
 	else 
 	    echo "Setup problem: couldn't find command: $1"
 	fi
diff --git a/heartbeat/ocf-shellfuncs.in b/heartbeat/ocf-shellfuncs.in
index edeb47f..a0da395 100644
--- a/heartbeat/ocf-shellfuncs.in
+++ b/heartbeat/ocf-shellfuncs.in
@@ -22,7 +22,7 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 # 
 
-# Build version: 32dd79a119834cd3482b01d4bba623e7b8c0d8b4
+# Build version: 02beac55c1da0ad99a5a19bd3b2333bcff7e916c
 
 # TODO: Some of this should probably split out into a generic OCF
 # library for shell scripts, but for the time being, we'll just use it
@@ -181,13 +181,20 @@ set_logtag() {
 	fi
 }
 
-ha_log() {
+__ha_log() {
+	local ignore_stderr=false
 	local loglevel
+
+	[ "x$1" = "x--ignore-stderr" ] && ignore_stderr=true && shift
+
 	[ none = "$HA_LOGFACILITY" ] && HA_LOGFACILITY=""
 	# if we're connected to a tty, then output to stderr
 	if tty >/dev/null; then
 		if [ "x$HA_debug" = "x0" -a "x$loglevel" = xdebug ] ; then
 			return 0
+		elif [ "$ignore_stderr" = "true" ]; then
+			# something already printed this error to stderr, so ignore
+			return 0
 		fi
 		if [ "$HA_LOGTAG" ]; then
 			echo "$HA_LOGTAG: $*"
@@ -226,7 +233,7 @@ ha_log() {
 	  echo "$HA_LOGTAG:	"`hadate`"${*}" >> $HA_LOGFILE
 	fi
 	if
-	  [ -z "$HA_LOGFACILITY" -a -z "$HA_LOGFILE" ]
+	  [ -z "$HA_LOGFACILITY" -a -z "$HA_LOGFILE" ] && ! [ "$ignore_stderr" = "true" ]
 	then
 	  : appending to stderr
 	  echo `hadate`"${*}" >&2
@@ -241,6 +248,11 @@ ha_log() {
         fi
 }
 
+ha_log()
+{
+	__ha_log "$@"
+}
+
 ha_debug() {
 
         if [ "x${HA_debug}" = "x0" ] ; then
@@ -331,6 +343,46 @@ ocf_log() {
 }
 
 #
+# ocf_exit_reason: print exit error string to stderr
+# Usage:           Allows the OCF script to provide a string
+#                  describing why the exit code was returned.
+# Arguments:   reason - required, The string that represents why the error
+#                       occured.
+#
+ocf_exit_reason()
+{
+	local cookie="$OCF_EXIT_REASON_PREFIX"
+	local fmt
+	local msg
+
+	# No argument is likely not intentional.
+	# Just one argument implies a printf format string of just "%s".
+	# "Least surprise" in case some interpolated string from variable
+	# expansion or other contains a percent sign.
+	# More than one argument: first argument is going to be the format string.
+	case $# in
+	0)	ocf_log err "Not enough arguments to ocf_log_exit_msg." ;;
+	1)	fmt="%s" ;;
+
+	*)	fmt=$1
+		shift
+		case $fmt in
+		*%*) : ;; # ok, does look like a format string
+		*) ocf_log warn "Does not look like format string: [$fmt]" ;;
+		esac ;;
+	esac
+
+	if [ -z "$cookie" ]; then
+		# use a default prefix
+		cookie="ocf-exit-reason:"
+	fi
+
+	msg=$(printf "${fmt}" "$@")
+	printf >&2 "%s%s\n" "$cookie" "$msg"
+	__ha_log --ignore-stderr "ERROR: $msg"
+}
+
+#
 # ocf_deprecated: Log a deprecation warning
 # Usage:          ocf_deprecated [param-name]
 # Arguments:      param-name optional, name of a boolean resource
@@ -518,6 +570,7 @@ ocf_local_nodename() {
 	which pacemakerd > /dev/null 2>&1
 	if [ $? -eq 0 ]; then
 		local version=$(pacemakerd -$ | grep "Pacemaker .*" | awk '{ print $2 }')
+		version=$(echo $version | awk -F- '{ print $1 }')
 		ocf_version_cmp "$version" "1.1.8"
 		if [ $? -eq 2 ]; then
 			which crm_node > /dev/null 2>&1
@@ -769,6 +822,7 @@ ocf_start_trace() {
 	fi
 	PS4='+ `date +"%T"`: ${FUNCNAME[0]:+${FUNCNAME[0]}:}${LINENO}: '
 	set -x
+	env=$( echo; printenv | sort )
 }
 ocf_stop_trace() {
 	set +x
@@ -778,3 +832,8 @@ __ocf_set_defaults "$@"
 
 : ${OCF_TRACE_RA:=$OCF_RESKEY_trace_ra}
 ocf_is_true "$OCF_TRACE_RA" && ocf_start_trace
+
+# pacemaker sets HA_use_logd, some others use HA_LOGD :/
+if ocf_is_true "$HA_use_logd"; then
+	: ${HA_LOGD:=yes}
+fi
diff --git a/heartbeat/oracle b/heartbeat/oracle
index d6b2c50..785be46 100755
--- a/heartbeat/oracle
+++ b/heartbeat/oracle
@@ -27,6 +27,9 @@
 #	OCF_RESKEY_ipcrm (optional; defaults to "instance")
 #	OCF_RESKEY_clear_backupmode (optional; default to "false")
 #	OCF_RESKEY_shutdown_method (optional; default to "checkpoint/abort")
+#	OCF_RESKEY_monuser (optional; defaults to "OCFMON")
+#	OCF_RESKEY_monpassword (optional; defaults to "OCFMON")
+#	OCF_RESKEY_monprofile (optional; defaults to "OCFMONPROFILE")
 #
 # Initialization:
 
@@ -56,6 +59,11 @@ oracle_usage() {
 	!
 }
 
+# Defaults
+OCF_RESKEY_monuser_default="OCFMON"
+OCF_RESKEY_monpassword_default="OCFMON"
+OCF_RESKEY_monprofile_default="OCFMONPROFILE"
+
 oracle_meta_data() {
 	cat <<END
 <?xml version="1.0"?>
@@ -100,6 +108,39 @@ If this does not work for you, just set it explicitely.
 <content type="string" default="" />
 </parameter>
 
+<parameter name="monuser" unique="0">
+<longdesc lang="en">
+Monitoring user name. Every connection as
+sysdba is logged in an audit log. This can
+result in a large number of new files created.
+A new user is created (if it doesn't exist) in
+the start action and subsequently used in monitor.
+It should have very limited rights. Make sure
+that the password for this user does not expire.
+</longdesc>
+<shortdesc lang="en">monuser</shortdesc>
+<content type="string" default="$OCF_RESKEY_monuser_default" />
+</parameter>
+
+<parameter name="monpassword" unique="0">
+<longdesc lang="en">
+Password for the monitoring user. Make sure
+that the password for this user does not expire.
+</longdesc>
+<shortdesc lang="en">monpassword</shortdesc>
+<content type="string" default="$OCF_RESKEY_monpassword_default" />
+</parameter>
+
+<parameter name="monprofile" unique="0">
+<longdesc lang="en">
+Profile used by the monitoring user. If the
+profile does not exist, it will be created
+with a non-expiring password.
+</longdesc>
+<shortdesc lang="en">monprofile</shortdesc>
+<content type="string" default="$OCF_RESKEY_monprofile_default" />
+</parameter>
+
 <parameter name="ipcrm" unique="0">
 <longdesc lang="en">
 Sometimes IPC objects (shared memory segments and semaphores)
@@ -250,7 +291,7 @@ dbasql() {
 	runsql "connect / as sysdba" $*
 }
 monsql() {
-	runsql "connect $MONUSR/$MONUSR" $*
+	runsql "connect $MONUSR/\"$MONPWD\"" $*
 }
 # use dbasql_one if the query should result in a single line output
 # at times people stuff commands in oracle .profile
@@ -325,22 +366,73 @@ getipc() {
 	echo "oradebug tracefile_name"
 	echo "oradebug ipc"
 }
+show_mon_profile() {
+	echo "select PROFILE from dba_profiles where PROFILE='$MONPROFILE';"
+}
+mk_mon_profile() {
+	cat<<EOF
+create profile $MONPROFILE limit FAILED_LOGIN_ATTEMPTS UNLIMITED PASSWORD_LIFE_TIME UNLIMITED;
+EOF
+}
 show_mon_user() {
 	echo "select USERNAME, ACCOUNT_STATUS from dba_users where USERNAME='$MONUSR';"
 }
 mk_mon_user() {
 	cat<<EOF
-create user $MONUSR identified by $MONUSR;
+create user $MONUSR identified by "$MONPWD" profile $MONPROFILE;
 grant create session to $MONUSR;
 grant select on v_\$instance to $MONUSR;
 EOF
 }
-check_mon_user() {
+show_mon_user_profile() {
+	echo "select PROFILE from dba_users where USERNAME='$MONUSR';"
+}
+set_mon_user_profile() {
+	echo "alter user $MONUSR profile $MONPROFILE;"
+}
+reset_mon_user_password() {
+	echo "alter user $MONUSR identified by $MONPWD;"
+}
+check_mon_profile() {
 	local output
-	dbasql show_mon_user | grep -w "^$MONUSR" >/dev/null &&
+	output=`dbasql show_mon_profile`
+	if echo "$output" | grep -iw "^$MONPROFILE" >/dev/null; then
 		return 0
+	fi
+	output=`dbasql mk_mon_profile show_mon_profile`
+	if echo "$output" | grep -iw "^$MONPROFILE" >/dev/null; then
+		return 0
+	else
+		ocf_log err "could not create $MONPROFILE oracle profile"
+		ocf_log err "sqlplus output: $output"
+		return 1
+	fi
+}
+check_mon_user() {
+	local output
+	local output2
+
+	output=`dbasql show_mon_user`
+	if echo "$output" | grep -iw "^$MONUSR" >/dev/null; then
+		if echo "$output" | grep -w "EXPIRED" >/dev/null; then
+			dbasql reset_mon_user_password
+		fi
+		output=`dbasql show_mon_user_profile`
+		if echo "$output" | grep -iw "^$MONPROFILE" >/dev/null; then
+			return 0
+		else
+			output=`dbasql set_mon_user_profile`
+			output2=`dbasql show_mon_user_profile`
+			if echo "$output2" | grep -iw "^$MONPROFILE" >/dev/null; then
+				return 0
+			fi
+			ocf_log err "could not set profile for $MONUSR oracle user"
+			ocf_log err "sqlplus output: $output( $output2 )"
+			return 1
+		fi
+	fi
 	output=`dbasql mk_mon_user show_mon_user`
-	if echo "$output" | grep -w "^$MONUSR" >/dev/null; then
+	if echo "$output" | grep -iw "^$MONUSR" >/dev/null; then
 		return 0
 	else
 		ocf_log err "could not create $MONUSR oracle user"
@@ -417,7 +509,7 @@ ipcdesc() {
 }
 rmipc() {
 	local what=$1 id=$2
-	ipcs -$what | filteroraipc | grep -w $id >/dev/null 2>&1 ||
+	ipcs -$what | filteroraipc | grep -iw $id >/dev/null 2>&1 ||
 		return
 	ocf_log info "Removing `ipcdesc $what` $id."
 	ipcrm -$what $id
@@ -447,6 +539,8 @@ is_proc_running() {
 # instance in OPEN state?
 instance_live() {
 	local status=`monsql_one dbstat`
+	[ "$status" = OPEN ] && return 0
+	status=`dbasql_one dbstat`
 	if [ "$status" = OPEN ]; then
 		return 0
 	else
@@ -493,7 +587,7 @@ oracle_getconfig() {
 oracle_start() {
 	local status output
 	if is_proc_running; then
-		status="`monsql_one dbstat`"
+		status="`dbasql_one dbstat`"
 		case "$status" in
 		"OPEN")
 			: nothing to be done, we can leave right now
@@ -541,6 +635,11 @@ oracle_start() {
 	fi
 	output=`dbasql dbopen`
 
+	# check/create the monitor profile
+	if ! check_mon_profile; then
+		return $OCF_ERR_GENERIC
+	fi
+
 	# check/create the monitor user
 	if ! check_mon_user; then
 		return $OCF_ERR_GENERIC
@@ -650,7 +749,12 @@ show_procs() {
 proc_pids() { show_procs | awk '{print $1}'; }
 PROCS_CLEANUP_TIME="30"
 
-MONUSR="OCFMON"
+MONUSR=${OCF_RESKEY_monuser:-$OCF_RESKEY_monuser_default}
+MONPWD=${OCF_RESKEY_monpassword:-$OCF_RESKEY_monpassword_default}
+MONPROFILE=${OCF_RESKEY_monprofile_default:-$OCF_RESKEY_monprofile_default}
+
+MONUSR=$(echo $MONUSR | awk '{print toupper($0)}')
+MONPROFILE=$(echo $MONPROFILE | awk '{print toupper($0)}')
 OCF_REQUIRED_PARAMS="sid"
 OCF_REQUIRED_BINARIES="sqlplus"
 ocf_rarun $*
diff --git a/heartbeat/oralsnr b/heartbeat/oralsnr
index 98fb120..a91eeab 100755
--- a/heartbeat/oralsnr
+++ b/heartbeat/oralsnr
@@ -268,7 +268,7 @@ oralsnr_validate_all() {
 # used in ora-common.sh
 show_procs() {
 	ps -e -o pid,user,args |
-		grep '[t]nslsnr' | grep -w "$listener" | grep -w "$ORACLE_OWNER"
+		grep '[t]nslsnr' | grep -i -w "$listener" | grep -w "$ORACLE_OWNER"
 }
 proc_pids() { show_procs | awk '{print $1}'; }
 PROCS_CLEANUP_TIME="10"
diff --git a/heartbeat/pgsql b/heartbeat/pgsql
index a462627..26bb251 100755
--- a/heartbeat/pgsql
+++ b/heartbeat/pgsql
@@ -252,7 +252,9 @@ Path to PostgreSQL server log output file.
 
 <parameter name="socketdir" unique="0" required="0">
 <longdesc lang="en">
-Unix socket directory for PostgreSQL
+Unix socket directory for PostgreSQL.
+
+If you use PostgreSQL 9.3 or higher and define unix_socket_directories in the postgresql.conf, then you must set socketdir to determine which directory is used for psql command.
 </longdesc>
 <shortdesc lang="en">socketdir</shortdesc>
 <content type="string" default="" />
@@ -507,7 +509,7 @@ pgsql_real_start() {
     # Check if we need to create a log file
     if ! check_log_file $OCF_RESKEY_logfile
     then
-        ocf_log err "PostgreSQL can't write to the log file: $OCF_RESKEY_logfile"
+        ocf_exit_reason "PostgreSQL can't write to the log file: $OCF_RESKEY_logfile"
         return $OCF_ERR_PERM
     fi
 
@@ -545,7 +547,7 @@ pgsql_real_start() {
         # Probably started.....
         ocf_log info "PostgreSQL start command sent."
     else
-        ocf_log err "Can't start PostgreSQL."
+        ocf_exit_reason "Can't start PostgreSQL."
         return $OCF_ERR_GENERIC
     fi
 
@@ -577,7 +579,7 @@ pgsql_replication_start() {
     fi
 
     if [ -f $PGSQL_LOCK ]; then
-        ocf_log err "My data may be inconsistent. You have to remove $PGSQL_LOCK file to force start."
+        ocf_exit_reason "My data may be inconsistent. You have to remove $PGSQL_LOCK file to force start."
         return $OCF_ERR_GENERIC
     fi
 
@@ -607,7 +609,7 @@ pgsql_promote() {
     local rc
 
     if ! is_replication; then
-        ocf_log err "Not in a replication mode."
+        ocf_exit_reason "Not in a replication mode."
         return $OCF_ERR_CONFIGURED
     fi
     rm -f ${XLOG_NOTE_FILE}.*
@@ -630,7 +632,7 @@ pgsql_promote() {
         pgsql_real_start
         rc=$?
         if [ $rc -ne $OCF_RUNNING_MASTER ]; then
-            ocf_log err "Can't start PostgreSQL as primary on promote."
+            ocf_exit_reason "Can't start PostgreSQL as primary on promote."
             if [ $rc -ne $OCF_SUCCESS ]; then
                 change_pgsql_status "$NODENAME" "STOP"
             fi
@@ -641,7 +643,7 @@ pgsql_promote() {
         if [ $? -eq 0 ]; then
             ocf_log info "PostgreSQL promote command sent."
         else
-            ocf_log err "Can't promote PostgreSQL."
+            ocf_exit_reason "Can't promote PostgreSQL."
             return $OCF_ERR_GENERIC
         fi
 
@@ -652,7 +654,7 @@ pgsql_promote() {
             if [ $rc -eq $OCF_RUNNING_MASTER ]; then
                 break;
             elif [ $rc -eq $OCF_ERR_GENERIC ]; then
-                ocf_log err "Can't promote PostgreSQL."
+                ocf_exit_reason "Can't promote PostgreSQL."
                 return $rc
             fi
             sleep 1
@@ -672,7 +674,7 @@ pgsql_demote() {
     local rc
 
     if ! is_replication; then
-        ocf_log err "Not in a replication mode."
+        ocf_exit_reason "Not in a replication mode."
         return $OCF_ERR_CONFIGURED
     fi
 
@@ -873,7 +875,7 @@ pgsql_real_monitor() {
             t)  ocf_log debug "PostgreSQL is running as a hot standby."
                 return $OCF_SUCCESS;;
 
-            *)  ocf_log err "$CHECK_MS_SQL output is $output"
+            *)  ocf_exit_reason "$CHECK_MS_SQL output is $output"
                 return $OCF_ERR_GENERIC;;
         esac
     fi
@@ -984,7 +986,7 @@ pgsql_pre_promote() {
             cmp_location=`printf "$master_baseline\n$my_master_baseline\n" |\
                           sort | head -1`
             if [ "$cmp_location" != "$my_master_baseline" ]; then
-                ocf_log err "My data is newer than new master's one. New master's location : $master_baseline"
+                ocf_exit_reason "My data is newer than new master's one. New master's location : $master_baseline"
                 exec_with_retry 0 $CRM_FAILCOUNT -r $OCF_RESOURCE_INSTANCE -U $NODENAME -v INFINITY
                 return $OCF_ERR_GENERIC
             fi
@@ -1160,7 +1162,7 @@ have_master_right() {
 
     show_xlog_location
     if [ $? -ne 0 ]; then
-        ocf_log err "Failed to show my xlog location."
+        ocf_exit_reason "Failed to show my xlog location."
         exit $OCF_ERR_GENERIC
     fi
 
@@ -1292,7 +1294,7 @@ set_async_mode_all() {
     ocf_log info "Set all nodes into async mode."
     runasowner -q err "echo \"synchronous_standby_names = ''\" > \"$REP_MODE_CONF\""
     if [ $? -ne 0 ]; then
-        ocf_log err "Can't set all nodes into async mode."
+        ocf_exit_reason "Can't set all nodes into async mode."
         return 1
     fi
     return 0
@@ -1330,7 +1332,7 @@ reload_conf() {
     if [ $? -eq 0 ]; then
         ocf_log info "Reload configuration file."
     else
-        ocf_log err "Can't reload configuration file."
+        ocf_exit_reason "Can't reload configuration file."
         return 1
     fi
 
@@ -1350,7 +1352,7 @@ user_recovery_conf() {
 make_recovery_conf() {
     runasowner "touch $RECOVERY_CONF"
     if [ $? -ne 0 ]; then
-        ocf_log err "Can't create recovery.conf."
+        ocf_exit_reason "Can't create recovery.conf."
         return 1
     fi
 
@@ -1471,11 +1473,11 @@ report_psql_error()
 
     ocf_log $loglevel "$message rc=$rc"
     if [ $rc -eq 1 ]; then
-        ocf_log err "Fatal error (out of memory, file not found, etc.) occurred while executing the psql command."
+        ocf_exit_reason "Fatal error (out of memory, file not found, etc.) occurred while executing the psql command."
     elif [ $rc -eq 2 ]; then
         ocf_log $loglevel "Connection error (connection to the server went bad and the session was not interactive) occurred while executing the psql command."
     elif [ $rc -eq 3 ]; then
-        ocf_log err "Script error (the variable ON_ERROR_STOP was set) occurred while executing the psql command."
+        ocf_exit_reason "Script error (the variable ON_ERROR_STOP was set) occurred while executing the psql command."
     fi
 }
 
@@ -1502,7 +1504,7 @@ exec_with_timeout() {
         sleep 1
         count=`expr $count - 1`
         if [ $count -le 0 ]; then
-            ocf_log err "\"$*\" (pid=$func_pid) timed out."
+            ocf_exit_reason "\"$*\" (pid=$func_pid) timed out."
             kill -s 9 $func_pid >/dev/null 2>&1
             return 1
         fi
@@ -1537,7 +1539,7 @@ exec_with_retry() {
         fi
     done
 
-    ocf_log err "giving up executing \"$*\""
+    ocf_exit_reason "giving up executing \"$*\""
     return $rc
 }
 
@@ -1551,7 +1553,7 @@ node_exist() {
 
 check_binary2() {
     if ! have_binary "$1"; then
-        ocf_log err "Setup problem: couldn't find command: $1"
+        ocf_exit_reason "Setup problem: couldn't find command: $1"
         return 1
     fi
     return 0
@@ -1565,7 +1567,7 @@ check_config() {
            ocf_log info "Configuration file is $1 not readable during probe."
            rc=1
         else
-           ocf_log err "Configuration file $1 doesn't exist"
+           ocf_exit_reason "Configuration file $1 doesn't exist"
            rc=2
         fi
     fi
@@ -1578,6 +1580,9 @@ pgsql_validate_all() {
     local version
     local check_config_rc
     local rep_mode_string
+    local socket_directories
+
+    version=`cat $OCF_RESKEY_pgdata/PG_VERSION`
 
     if ! check_binary2 "$OCF_RESKEY_pgctl" || 
        ! check_binary2 "$OCF_RESKEY_psql"; then
@@ -1587,11 +1592,27 @@ pgsql_validate_all() {
     check_config "$OCF_RESKEY_config"
     check_config_rc=$?
     [ $check_config_rc -eq 2 ] && return $OCF_ERR_INSTALLED
-    [ $check_config_rc -eq 0 ] && : ${OCF_RESKEY_socketdir=`get_pgsql_param unix_socket_directory`}
+    if [ $check_config_rc -eq 0 ]; then
+        ocf_version_cmp "$version" "9.3"
+        if [ $? -eq 0 ]; then
+            : ${OCF_RESKEY_socketdir=`get_pgsql_param unix_socket_directory`}
+        else
+            # unix_socket_directories is used by PostgreSQL 9.3 or higher.
+            socket_directories=`get_pgsql_param unix_socket_directories`
+            if [ -n "$socket_directories" ]; then
+                # unix_socket_directories may have multiple socket directories and the pgsql RA can not know which directory is used for psql command.
+                # Therefore, the user must set OCF_RESKEY_socketdir explicitly.
+                if [ -z "$OCF_RESKEY_socketdir" ]; then
+                    ocf_log err "In PostgreSQL 9.3 or higher, socketdir can't be empty if you define unix_socket_directories in the postgresql.conf."
+                    return $OCF_ERR_CONFIGURED
+                fi
+            fi
+        fi
+    fi
 
     getent passwd $OCF_RESKEY_pgdba >/dev/null 2>&1
     if [ ! $? -eq 0 ]; then
-        ocf_log err "User $OCF_RESKEY_pgdba doesn't exist";
+        ocf_exit_reason "User $OCF_RESKEY_pgdba doesn't exist";
         return $OCF_ERR_INSTALLED;
     fi
 
@@ -1599,46 +1620,45 @@ pgsql_validate_all() {
         ocf_log info "Don't check $OCF_RESKEY_pgdata during probe"
     else
         if ! runasowner "test -w $OCF_RESKEY_pgdata"; then
-            ocf_log err "Directory $OCF_RESKEY_pgdata is not writable by $OCF_RESKEY_pgdba"
+            ocf_exit_reason "Directory $OCF_RESKEY_pgdata is not writable by $OCF_RESKEY_pgdba"
             return $OCF_ERR_PERM;
         fi
     fi
 
     if [ -n "$OCF_RESKEY_monitor_user" -a ! -n "$OCF_RESKEY_monitor_password" ]
     then
-        ocf_log err "monitor password can't be empty"
+        ocf_exit_reason "monitor password can't be empty"
         return $OCF_ERR_CONFIGURED
     fi
 
     if [ ! -n "$OCF_RESKEY_monitor_user" -a -n "$OCF_RESKEY_monitor_password" ]
     then
-        ocf_log err "monitor_user has to be set if monitor_password is set"
+        ocf_exit_reason "monitor_user has to be set if monitor_password is set"
         return $OCF_ERR_CONFIGURED
     fi
 
     if is_replication || [ "$OCF_RESKEY_rep_mode" = "slave" ]; then
-        version=`cat $OCF_RESKEY_pgdata/PG_VERSION`
         if [ `printf "$version\n9.1" | sort -n | head -1` != "9.1" ]; then
-            ocf_log err "Replication mode needs PostgreSQL 9.1 or higher."
+            ocf_exit_reason "Replication mode needs PostgreSQL 9.1 or higher."
             return $OCF_ERR_INSTALLED
         fi
         if [ ! -n "$OCF_RESKEY_master_ip" ]; then
-            ocf_log err "master_ip can't be empty."
+            ocf_exit_reason "master_ip can't be empty."
             return $OCF_ERR_CONFIGURED
         fi
     fi
 
     if is_replication; then
         if ! ocf_is_ms; then
-            ocf_log err "Replication(rep_mode=async or sync) requires Master/Slave configuration."
+            ocf_exit_reason "Replication(rep_mode=async or sync) requires Master/Slave configuration."
             return $OCF_ERR_CONFIGURED
         fi
         if [ ! "$OCF_RESKEY_rep_mode" = "sync" -a ! "$OCF_RESKEY_rep_mode" = "async" ]; then
-            ocf_log err "Invalid rep_mode : $OCF_RESKEY_rep_mode"
+            ocf_exit_reason "Invalid rep_mode : $OCF_RESKEY_rep_mode"
             return $OCF_ERR_CONFIGURED
         fi
         if [ ! -n "$NODE_LIST" ]; then
-            ocf_log err "node_list can't be empty."
+            ocf_exit_reason "node_list can't be empty."
             return $OCF_ERR_CONFIGURED
         fi
         if [ $check_config_rc -eq 0 ]; then
@@ -1651,19 +1671,20 @@ pgsql_validate_all() {
             else
                 if grep -q "$rep_mode_string" $OCF_RESKEY_config; then
                     ocf_log info "deleting include directive from $OCF_RESKEY_config"
-                    sed -i "/${rep_mode_string//\//\\/}/d" $OCF_RESKEY_config
+                    rep_mode_string=`echo $rep_mode_string | sed -e 's|/|\\\\/|g'`
+                    sed -i "/$rep_mode_string/d" $OCF_RESKEY_config
                 fi
             fi
         fi
         if ! mkdir -p $OCF_RESKEY_tmpdir || ! chown $OCF_RESKEY_pgdba $OCF_RESKEY_tmpdir || ! chmod 700 $OCF_RESKEY_tmpdir; then
-            ocf_log err "Can't create directory $OCF_RESKEY_tmpdir or it is not readable by $OCF_RESKEY_pgdba"
+            ocf_exit_reason "Can't create directory $OCF_RESKEY_tmpdir or it is not readable by $OCF_RESKEY_pgdba"
             return $OCF_ERR_PERM
         fi
     fi
 
     if [ "$OCF_RESKEY_rep_mode" = "slave" ]; then
         if ocf_is_ms; then
-            ocf_log err "Replication(rep_mode=slave) does not support Master/Slave configuration."
+            ocf_exit_reason "Replication(rep_mode=slave) does not support Master/Slave configuration."
             return $OCF_ERR_CONFIGURED
         fi
     fi
@@ -1698,24 +1719,24 @@ check_log_file() {
 check_socket_dir() {
     if [ ! -d "$OCF_RESKEY_socketdir" ]; then
         if ! mkdir "$OCF_RESKEY_socketdir"; then
-            ocf_log err "Can't create directory $OCF_RESKEY_socketdir"
+            ocf_exit_reason "Can't create directory $OCF_RESKEY_socketdir"
             exit $OCF_ERR_PERM
         fi
 
         if ! chown $OCF_RESKEY_pgdba:`getent passwd \
              $OCF_RESKEY_pgdba | cut -d ":" -f 4` "$OCF_RESKEY_socketdir" 
         then
-            ocf_log err "Can't change ownership for $OCF_RESKEY_socketdir"
+            ocf_exit_reason "Can't change ownership for $OCF_RESKEY_socketdir"
             exit $OCF_ERR_PERM
         fi
 
         if ! chmod 2775 "$OCF_RESKEY_socketdir"; then
-            ocf_log err "Can't change permissions for $OCF_RESKEY_socketdir"
+            ocf_exit_reason "Can't change permissions for $OCF_RESKEY_socketdir"
             exit $OCF_ERR_PERM
         fi
     else
         if ! runasowner "touch $OCF_RESKEY_socketdir/test.$$"; then
-            ocf_log err "$OCF_RESKEY_pgdba can't create files in $OCF_RESKEY_socketdir"
+            ocf_exit_reason "$OCF_RESKEY_pgdba can't create files in $OCF_RESKEY_socketdir"
             exit $OCF_ERR_PERM
         fi
         rm $OCF_RESKEY_socketdir/test.$$
@@ -1805,7 +1826,7 @@ US=`id -u -n`
 
 if [ $US != root -a $US != $OCF_RESKEY_pgdba ]
 then
-    ocf_log err "$0 must be run as root or $OCF_RESKEY_pgdba"
+    ocf_exit_reason "$0 must be run as root or $OCF_RESKEY_pgdba"
     exit $OCF_ERR_GENERIC
 fi
 
diff --git a/heartbeat/portblock b/heartbeat/portblock
index 53b35e9..6085566 100755
--- a/heartbeat/portblock
+++ b/heartbeat/portblock
@@ -24,8 +24,10 @@
 
 # Defaults
 OCF_RESKEY_ip_default="0.0.0.0/0"
+OCF_RESKEY_reset_local_on_unblock_stop_default="false"
 
 : ${OCF_RESKEY_ip=${OCF_RESKEY_ip_default}}
+: ${OCF_RESKEY_reset_local_on_unblock_stop=${OCF_RESKEY_reset_local_on_unblock_stop_default}}
 #######################################################################
 CMD=`basename $0`
 TICKLETCP=$HA_BIN/tickle_tcp
@@ -149,6 +151,26 @@ The action (block/unblock) to be done on the protocol::portno.
 <content type="string" default="" />
 </parameter>
 
+<parameter name="reset_local_on_unblock_stop" unique="0" required="0">
+<content type="boolean" default="${OCF_RESKEY_reset_local_on_unblock_stop_default}" />
+<shortdesc lang="en">(try to) reset server TCP sessions when unblock stops</shortdesc>
+<longdesc>
+If for some reason the long lived server side TCP sessions won't be cleaned up
+by a reconfiguration/flush/stop of whatever services this portblock protects,
+they would linger in the connection table, even after the IP is gone
+and services have been switched over to an other node.
+
+An example would be the default NFS kernel server.
+
+These "known" connections may seriously confuse and delay a later switchback.
+
+Enabling this option will cause this agent to try to get rid of these connections
+by injecting a temporary iptables rule to TCP-reset outgoing packets from the
+blocked ports, and additionally tickle them locally,
+just before it starts to DROP incoming packets on "unblock stop".
+</longdesc>
+</parameter>
+
 <parameter name="ip" unique="0" required="0">
 <longdesc lang="en">
 The IP address used to be blocked/unblocked.
@@ -233,12 +255,34 @@ save_tcp_connections()
 	fi
 }
 
-run_tickle_tcp()
+tickle_remote()
 {
 	[ -z "$OCF_RESKEY_tickle_dir" ] && return
 	echo 1 > /proc/sys/net/ipv4/tcp_tw_recycle
 	f=$OCF_RESKEY_tickle_dir/$OCF_RESKEY_ip
-	[ -f $f ] && cat $f | $TICKLETCP -n 3
+	[ -r $f ] || return
+	$TICKLETCP -n 3 < $f
+}
+
+tickle_local()
+{
+	[ -z "$OCF_RESKEY_tickle_dir" ] && return
+	f=$OCF_RESKEY_tickle_dir/$OCF_RESKEY_ip
+	[ -r $f ] || return
+	# swap "local" and "remote" address,
+	# so we tickle ourselves.
+	# We set up a REJECT with tcp-reset before we do so, so we get rid of
+	# the no longer wanted potentially long lived "ESTABLISHED" connection
+	# entries on the IP we are going to delet in a sec.  These would get in
+	# the way if we switch-over and then switch-back in quick succession.
+	local i
+	awk '{ print $2, $1; }' $f | $TICKLETCP
+	netstat -tn | grep -Fw $OCF_RESKEY_ip || return
+	for i in 0.1 0.5 1 2 4 ; do
+		sleep $i
+		awk '{ print $2, $1; }' $f | $TICKLETCP
+		netstat -tn | grep -Fw $OCF_RESKEY_ip || break
+	done
 }
 
 SayActive()
@@ -304,15 +348,30 @@ IptablesStatus() {
 #IptablesBLOCK  {udp|tcp} portno,portno ip
 IptablesBLOCK()
 {
+  local rc=0
+  local try_reset=false
+  if	[ "$1/$4/$__OCF_ACTION" = tcp/unblock/stop ] &&
+	ocf_is_true $reset_local_on_unblock_stop
+  then
+	try_reset=true
+  fi
   if
     chain_isactive "$1" "$2" "$3"
   then
     : OK -- chain already active
   else
+    if $try_reset ; then
+      $IPTABLES -I OUTPUT -p "$1" -s "$3" -m multiport --sports "$2" -j REJECT --reject-with tcp-reset
+      tickle_local
+    fi
     $IPTABLES -I INPUT -p "$1" -d "$3" -m multiport --dports "$2" -j DROP
+    rc=$?
+    if $try_reset ; then
+      $IPTABLES -D OUTPUT -p "$1" -s "$3" -m multiport --sports "$2" -j REJECT --reject-with tcp-reset
+    fi
   fi
 
-  return $?
+  return $rc
 }
 
 #IptablesUNBLOCK  {udp|tcp} portno,portno ip
@@ -338,7 +397,7 @@ IptablesStart()
     unblock)
 		IptablesUNBLOCK "$@"
 		rc=$?
-		run_tickle_tcp
+		tickle_remote
 		#ignore run_tickle_tcp exit code!
 		return $rc
 		;;
@@ -411,6 +470,17 @@ IptablesValidateAll()
 	exit $OCF_ERR_CONFIGURED
 	;; 
   esac
+
+  if ocf_is_true $reset_local_on_unblock_stop; then
+	if [ $action != unblock ] ; then
+		ocf_log err "reset_local_on_unblock_stop is only relevant with action=unblock"
+		exit $OCF_ERR_CONFIGURED
+	fi
+	if [ -z $OCF_RESKEY_tickle_dir ] ; then
+		ocf_log warn "reset_local_on_unblock_stop works best with tickle_dir enabled as well"
+	fi
+  fi
+
   return $OCF_SUCCESS
 }
 
@@ -451,6 +521,7 @@ protocol=$OCF_RESKEY_protocol
 portno=$OCF_RESKEY_portno
 action=$OCF_RESKEY_action
 ip=$OCF_RESKEY_ip
+reset_local_on_unblock_stop=$OCF_RESKEY_reset_local_on_unblock_stop
 
 case $1 in
   start)	
diff --git a/heartbeat/postfix b/heartbeat/postfix
index 8619af6..72fc371 100755
--- a/heartbeat/postfix
+++ b/heartbeat/postfix
@@ -134,7 +134,7 @@ postfix_start()
     ret=$?
 
     if [ $ret -ne 0 ]; then
-        ocf_log err "Postfix returned error: " $ret
+        ocf_exit_reason "Postfix returned error: " $ret
         return $OCF_ERR_GENERIC
     fi
 
@@ -165,7 +165,7 @@ postfix_stop()
     ret=$?
 
     if [ $ret -ne 0 ]; then
-        ocf_log err "Postfix returned an error while stopping: " $ret
+        ocf_exit_reason "Postfix returned an error while stopping: " $ret
         return $OCF_ERR_GENERIC
     fi
 
@@ -181,14 +181,14 @@ postfix_stop()
     # escalate to abort if we did not stop by now
     # @TODO shall we loop here too?
     if postfix_running info; then
-        ocf_log err "Postfix failed to stop. Escalating to 'abort'."
+        ocf_exit_reason "Postfix failed to stop. Escalating to 'abort'."
 
         $binary $OPTIONS abort >/dev/null 2>&1; ret=$?
         sleep 5
 
         # postfix abort did not succeed
         if postfix_running; then
-            ocf_log err "Postfix failed to abort."
+            ocf_exit_reason "Postfix failed to abort."
             return $OCF_ERR_GENERIC
         fi
     fi
@@ -238,14 +238,14 @@ postfix_validate_all()
                 # skip in-depth directory checks if config file isn't readable during probe
                 dir_check=false
             else
-                ocf_log err "Postfix configuration directory '$config_dir' does not exist or is not readable."
+                ocf_exit_reason "Postfix configuration directory '$config_dir' does not exist or is not readable."
                 return $OCF_ERR_INSTALLED
             fi
         fi
 
         alternate_config_directories=`postconf -h alternate_config_directories 2>/dev/null | grep "$config_dir/\?"`
         if [ "x$alternate_config_directories" = "x" ]; then
-            ocf_log err "Postfix main configuration must contain correct 'alternate_config_directories' parameter."
+            ocf_exit_reason "Postfix main configuration must contain correct 'alternate_config_directories' parameter."
             return $OCF_ERR_INSTALLED
         fi
     fi
@@ -257,7 +257,7 @@ postfix_validate_all()
             if ocf_is_probe; then
                 ocf_log info "Postfix queue directory '$queue_dir' not readable during probe."
             else
-                ocf_log err "Postfix queue directory '$queue_dir' does not exist or is not readable."
+                ocf_exit_reason "Postfix queue directory '$queue_dir' does not exist or is not readable."
                 return $OCF_ERR_INSTALLED
             fi
         fi
@@ -266,14 +266,14 @@ postfix_validate_all()
             data_dir=`postconf $OPTION_CONFIG_DIR -h data_directory 2>/dev/null`
             data_dir_count=`echo "$data_dir" | tr ',' ' ' | wc -w`
             if [ $data_dir_count -gt 1 ]; then
-            	ocf_log err "Postfix data directory '$orig_data_dir' cannot be set to multiple directories."
+            	ocf_exit_reason "Postfix data directory '$orig_data_dir' cannot be set to multiple directories."
                 return $OCF_ERR_INSTALLED
             fi
             if [ ! -d "$data_dir" ]; then
                 if ocf_is_probe; then
                     ocf_log info "Postfix data directory '$data_dir' not readable during probe."
                 else
-                    ocf_log err "Postfix data directory '$data_dir' does not exist or is not readable."
+                    ocf_exit_reason "Postfix data directory '$data_dir' does not exist or is not readable."
                     return $OCF_ERR_INSTALLED
                 fi
             fi
@@ -287,7 +287,7 @@ postfix_validate_all()
                     if ocf_is_probe; then
                         ocf_log info "Directory '$dir' is not writable by user '$user' during probe."
                     else
-                        ocf_log err "Directory '$dir' is not writable by user '$user'."
+                        ocf_exit_reason "Directory '$dir' is not writable by user '$user'."
                         return $OCF_ERR_PERM;
                     fi
                 fi
@@ -300,7 +300,7 @@ postfix_validate_all()
         $binary $OPTIONS check >/dev/null 2>&1
         ret=$?
         if [ $ret -ne 0 ]; then
-            ocf_log err "Postfix 'check' failed: " $ret
+            ocf_exit_reason "Postfix 'check' failed: " $ret
             return $OCF_ERR_GENERIC
         fi
     fi
diff --git a/heartbeat/rsyncd b/heartbeat/rsyncd
index b8cdeb7..86c771e 100755
--- a/heartbeat/rsyncd
+++ b/heartbeat/rsyncd
@@ -127,7 +127,7 @@ rsyncd_status()
 				return $OCF_ERR_GENERIC
 			fi
 		else
-			ocf_log err "PID file empty!"
+			ocf_exit_reason "PID file empty!"
 			return $OCF_ERR_GENERIC
 		fi
 	fi
@@ -145,7 +145,7 @@ rsyncd_start()
 	if [ $retVal -eq $OCF_SUCCESS ]; then
 		exit $OCF_SUCCESS
 	elif [ $retVal -ne $OCF_NOT_RUNNING ]; then
-		ocf_log err "Error. Unknown status."
+		ocf_exit_reason "Error. Unknown status."
 		exit $OCF_ERR_GENERIC
 	fi
 
@@ -164,11 +164,11 @@ rsyncd_start()
 	if  grep -v "^#" "$CONF_FILE" | grep "pid file" > /dev/null ; then
 		$COMMAND;
 		if [ $? -ne 0 ]; then
-			ocf_log err "Error. rsycn daemon returned error $?."
+			ocf_exit_reason "Error. rsync daemon returned error $?."
 			exit $OCF_ERR_GENERIC
 		fi
 	else
-		ocf_log err "Error. \"pid file\" entry required in the rsyncd config file by rsyncd OCF RA."
+		ocf_exit_reason "Error. \"pid file\" entry required in the rsyncd config file by rsyncd OCF RA."
 		return $OCF_ERR_GENERIC
 	fi
 
@@ -186,7 +186,7 @@ rsyncd_stop()
 			if [ $? -ne 0 ]; then
 				kill -s KILL $PID
 				if [ $? -ne 0 ]; then
-					ocf_log err "Error. Could not stop rsync daemon."
+					ocf_exit_reason "Error. Could not stop rsync daemon."
 					return $OCF_ERR_GENERIC
 				fi
 			fi
@@ -205,18 +205,18 @@ rsyncd_monitor()
 rsyncd_validate_all()
 {
 	if [ -n "$OCF_RESKEY_binpath" -a ! -x "$OCF_RESKEY_binpath" ]; then
-		ocf_log err "Binary path $OCF_RESKEY_binpath does not exist."
+		ocf_exit_reason "Binary path $OCF_RESKEY_binpath does not exist."
 		exit $OCF_ERR_ARGS
 	fi
 	if [ -n "$OCF_RESKEY_conffile" -a ! -f "$OCF_RESKEY_conffile" ]; then
-		ocf_log err "Config file $OCF_RESKEY_conffile does not exist."
+		ocf_exit_reason "Config file $OCF_RESKEY_conffile does not exist."
 		exit $OCF_ERR_ARGS
 	fi
 
 	if  grep -v "^#" "$CONF_FILE" | grep "pid file" > /dev/null ; then
 		:
 	else
-		ocf_log err "Error. \"pid file\" entry required in the rsyncd config file by rsyncd OCF RA."
+		ocf_exit_reason "Error. \"pid file\" entry required in the rsyncd config file by rsyncd OCF RA."
 		return $OCF_ERR_GENERIC
 	fi
 
diff --git a/heartbeat/sg_persist b/heartbeat/sg_persist
new file mode 100755
index 0000000..c4af069
--- /dev/null
+++ b/heartbeat/sg_persist
@@ -0,0 +1,673 @@
+#!/bin/bash
+#
+#
+#   OCF Resource Agent compliant PERSISTENT SCSI RESERVATION resource script.
+#
+#
+# Copyright (c) 2011 Evgeny Nifontov and lwang at suse.com All Rights Reserved.
+#
+# "Heartbeat drbd OCF Resource Agent: 2007, Lars Marowsky-Bree" was used 
+# as example of multistate OCF Resource Agent.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of version 2 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# Further, this software is distributed without any warranty that it is
+# free of the rightful claim of any third person regarding infringement
+# or the like.  Any license provided herein, whether implied or
+# otherwise, applies only to this software file.  Patent licenses, if
+# any, provided herein do not apply to combinations of this program with
+# other software, or any other product whatsoever.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
+#
+#
+# OCF instance parameters
+#    OCF_RESKEY_sg_persist_binary
+#    OCF_RESKEY_devs
+#    OCF_RESKEY_required_devs_nof
+#    OCF_RESKEY_reservation_type
+#    OCF_RESKEY_master_score_base
+#    OCF_RESKEY_master_score_dev_factor
+#    OCF_RESKEY_master_score_delay
+#
+# TODO
+# 
+# 1) PROBLEM: devices which were not accessible during 'start' action, will be never registered/reserved 
+#    TODO:    'Master' and 'Salve' registers new devs in 'monitor' action
+#    TODO:    'Master' reserves new devs in 'monitor' action
+
+#######################################################################
+# Initialization:
+
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+
+# set default values
+    : ${sg_persist_binary="sg_persist"}     # binary name for the resource
+    : ${devs=""}                            # device list
+    : ${required_devs_nof=1}                # number of required devices
+    : ${reservation_type=1}                 # reservation type
+    : ${master_score_base=0}                # master score base 
+    : ${master_score_dev_factor=100}        # device factor for master score
+    : ${master_score_delay=30}              # delay for master score
+
+#######################################################################
+
+
+meta_data() {
+    cat <<END
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="sg_persist">
+<version>1.1</version>
+
+<longdesc lang="en">
+This resource agent manages SCSI PERSISTENT RESERVATIONS.
+"sg_persist" from sg3_utils is used, please see its documentation.
+Should be used as multistate (Master/Slave) resource
+Slave registers its node id ("crm_node -i") as reservation key ( --param-rk ) on each device in the "devs" list.
+Master reservs all devices from "devs" list with reservation "--prout-type" value from "reservation_type" parameter.
+</longdesc>
+<shortdesc lang="en">Manages SCSI PERSISTENT RESERVATIONS</shortdesc>
+
+<parameters>
+<parameter name="binary" unique="0">
+<longdesc lang="en">
+The name of the binary that manages the resource
+</longdesc>
+<shortdesc>the binay name of the resource</shortdesc>
+<content type="string" default="sg_persist"/>
+</parameter>
+
+<parameter name="devs" unique="0" required="1">
+<longdesc lang="en">
+Device list. Multiple devices can be listed with blank space as separator. 
+Shell wildcars are allowed.
+</longdesc>
+<shortdesc lang="en">device list</shortdesc>
+<content type="string"/>
+</parameter>
+
+<parameter name="required_devs_nof" unique="0" required="0">
+<longdesc lang="en">
+Minimum number of "working" devices from device list
+      1) existing 
+      2) "sg_persist --read-keys \$device" works (Return code 0)
+resource actions "start","monitor","promote" and "validate-all" return "\$OCF_ERR_INSTALLED"
+if the actual number of "working" devices is less then "required_devs_nof".
+resource actions "stop" and "demote" tries to remove reservations and registration keys from 
+all working devices, but always return "\$OCF_SUCCESS"
+</longdesc>
+<shortdesc lang="en">minimum number of working devices</shortdesc>
+<content type="string" default="1"/>
+</parameter>
+
+<parameter name="reservation_type" unique="0" required="0">
+<longdesc lang="en">
+reservation type 
+</longdesc>
+<shortdesc lang="en">reservation type</shortdesc>
+<content type="string" default="1" />
+</parameter>
+
+<parameter name="master_score_base" unique="0" required="0">
+<longdesc lang="en">
+master_score_base value
+"master_score_base" value is used in "master_score" calculation:
+master_score = \$master_score_base + \$master_score_dev_factor * \$working_devs  
+if set to bigger value in sg_persist resource configuration on some node, this node will be "preferred" for master role. 
+</longdesc>
+<shortdesc lang="en">base master_score value</shortdesc>
+<content type="string" default="0" />
+</parameter>
+
+<parameter name="master_score_dev_factor" unique="0" required="0">
+<longdesc lang="en">
+Working device factor in master_score calculation
+each "working" device provides additional value to "master_score", 
+so the node that sees more devices will be preferred for the "Master"-role
+Setting it to 0 will disable this behavior. 
+</longdesc>
+<shortdesc lang="en">working device factor in master_score calculation</shortdesc>
+<content type="string" default="100" />
+</parameter>
+
+<parameter name="master_score_delay" unique="0" required="0">
+<longdesc lang="en">
+master/slave decreases/increases its master_score after delay of \$master_score_delay seconds
+so if some device gets inaccessible, the slave decreases its master_score first and the resource will no be watched
+and after this device reappears again the master increases its master_score first
+this can work only if the master_score_delay is bigger then monitor interval on both master and slave
+Setting it to 0 will disable this behavior.
+</longdesc>
+<shortdesc lang="en">master_score decrease/increase delay time</shortdesc>
+<content type="string" default="30" />
+</parameter>
+</parameters>
+
+<actions>
+<action name="start"   timeout="30" />
+<action name="promote"   timeout="30" />
+<action name="demote"   timeout="30" />
+<action name="notify"   timeout="30" />
+<action name="stop"    timeout="30" />
+<action name="monitor" depth="0"  timeout="20" interval="29" role="Slave" />
+<action name="monitor" depth="0"  timeout="20" interval="60" role="Master" />
+<action name="meta-data"  timeout="5" />
+<action name="validate-all"  timeout="30" />
+</actions>
+</resource-agent>
+END
+
+    exit $OCF_SUCCESS
+}
+
+sg_persist_init() {
+    
+    if ! ocf_is_root ; then
+        ocf_log err "You must be root to perform this operation."
+        exit $OCF_ERR_PERM
+    fi 
+
+    : ${SG_PERSIST:="$sg_persist_binary"}
+    check_binary $SG_PERSIST
+    
+    ROLE=$OCF_RESKEY_CRM_meta_role
+    NOW=$(date +%s)
+
+    RESOURCE="${OCF_RESOURCE_INSTANCE}"
+    MASTER_SCORE_VAR_NAME="master-${OCF_RESOURCE_INSTANCE}"
+    PENDING_VAR_NAME="pending-$MASTER_SCORE_VAR_NAME"
+    
+    #only works with corocync 
+    CRM_NODE="${HA_SBIN_DIR}/crm_node"
+    NODE_ID_DEC=$($CRM_NODE -i)
+
+    NODE=$($CRM_NODE -l | $GREP $NODE_ID_DEC)
+    NODE=${NODE#$NODE_ID_DEC }
+    NODE=${NODE% *}
+    
+    MASTER_SCORE_ATTRIBUTE="${HA_SBIN_DIR}/crm_attribute --lifetime=reboot --name=$MASTER_SCORE_VAR_NAME --node=$NODE"
+    CRM_MASTER="${HA_SBIN_DIR}/crm_master --lifetime=reboot"
+    PENDING_ATTRIBUTE="${HA_SBIN_DIR}/crm_attribute --lifetime=reboot --name=$PENDING_VAR_NAME --node=$NODE"
+
+    NODE_ID_HEX=$(printf '0x%x' $NODE_ID_DEC)
+
+    if [ -z "$NODE_ID_HEX" ]; then
+        ocf_log err "Couldn't get node id with \"$CRM_NODE\""
+        exit $OCF_ERR_INSTALLED
+    fi
+
+    ocf_log debug "$RESOURCE: NODE:$NODE, ROLE:$ROLE, NODE_ID DEC:$NODE_ID_DEC HEX:$NODE_ID_HEX"
+
+    DEVS=${OCF_RESKEY_devs:=$devs}
+    REQUIRED_DEVS_NOF=${OCF_RESKEY_required_devs_nof:=$required_devs_nof}
+    RESERVATION_TYPE=${OCF_RESKEY_reservation_type:=$reservation_type}
+    MASTER_SCORE_BASE=${OCF_RESKEY_master_score_base:=$master_score_base}
+    MASTER_SCORE_DEV_FACTOR=${OCF_RESKEY_master_score_dev_factor:=$master_score_dev_factor}
+    MASTER_SCORE_DELAY=${OCF_RESKEY_master_score_delay:=$master_score_delay}
+
+    ocf_log debug "$RESOURCE: DEVS=$DEVS"
+    ocf_log debug "$RESOURCE: REQUIRED_DEVS_NOF=$REQUIRED_DEVS_NOF"
+    ocf_log debug "$RESOURCE: RESERVATION_TYPE=$RESERVATION_TYPE"
+    ocf_log debug "$RESOURCE: MASTER_SCORE_BASE=$MASTER_SCORE_BASE"
+    ocf_log debug "$RESOURCE: MASTER_SCORE_DEV_FACTOR=$MASTER_SCORE_DEV_FACTOR"
+    ocf_log debug "$RESOURCE: MASTER_SCORE_DELAY=$MASTER_SCORE_DELAY"
+
+    #expand path wildcards
+    DEVS=$(echo $DEVS)
+
+    if [ -z "$DEVS" ]; then
+        ocf_log err "\"devs\" not defined"
+        exit $OCF_ERR_INSTALLED
+    fi
+ 
+    sg_persist_check_devs
+    sg_persist_get_status
+}
+
+sg_persist_action_usage() {
+    cat <<END
+    usage: $0 {start|stop|monitor|validate-all|promote|demote|notify|meta-data}
+
+    Expects to have a fully populated OCF RA-compliant environment set.
+END
+}
+
+sg_persist_get_status() {
+
+    unset WORKING_DEVS[*]
+    
+    for dev in ${EXISTING_DEVS[*]}
+    do
+        READ_KEYS=`ocf_run $SG_PERSIST --in --read-keys $dev 2>&1`
+        if [ $? -eq $OCF_SUCCESS ]; then 
+            WORKING_DEVS+=($dev)
+            echo $READ_KEYS | $GREP $NODE_ID_HEX >/dev/null
+            if [ $? -eq 0 ]; then 
+                REGISTERED_DEVS+=($dev)
+
+                READ_RESERVATION=`ocf_run $SG_PERSIST --in --read-reservation $dev 2>&1`
+                if [ $? -eq $OCF_SUCCESS ]; then
+                    echo $READ_RESERVATION | $GREP $NODE_ID_HEX >/dev/null
+                    if [ $? -eq 0 ]; then 
+                        RESERVED_DEVS+=($dev)
+                    fi
+
+                    reservation_key=`echo $READ_RESERVATION | $GREP -o 'Key=0x[0-9a-f]*' | $GREP -o '0x[0-9a-f]*'`
+                    if [ -n "$reservation_key" ]; then 
+                        DEVS_WITH_RESERVATION+=($dev)
+                        RESERVATION_KEYS+=($reservation_key)
+                    fi
+                fi
+            fi
+        fi
+    done
+
+    WORKING_DEVS_NOF=${#WORKING_DEVS[*]}
+
+    ocf_log debug "$RESOURCE: working devices: `sg_persist_echo_array ${WORKING_DEVS[*]}`"
+    ocf_log debug "$RESOURCE: number of working devices: $WORKING_DEVS_NOF"
+    
+    ocf_log debug "$RESOURCE: registered devices: `sg_persist_echo_array ${REGISTERED_DEVS[*]}`"
+    ocf_log debug "$RESOURCE: reserved devices: `sg_persist_echo_array ${RESERVED_DEVS[*]}`"
+    ocf_log debug "$RESOURCE: devices with reservation: `sg_persist_echo_array ${DEVS_WITH_RESERVATION[*]}`"
+    ocf_log debug "$RESOURCE: reservation keys: `sg_persist_echo_array ${RESERVATION_KEYS[*]}`"
+    
+    MASTER_SCORE=$(($MASTER_SCORE_BASE + $MASTER_SCORE_DEV_FACTOR*$WORKING_DEVS_NOF))
+    ocf_log debug "$RESOURCE: master_score: $MASTER_SCORE_BASE + $MASTER_SCORE_DEV_FACTOR*$WORKING_DEVS_NOF = $MASTER_SCORE"
+
+}
+
+sg_persist_check_devs() {
+
+    for dev in $DEVS 
+    do
+        if [ -e "$dev" ]; then
+            EXISTING_DEVS+=($dev)
+        fi
+    done
+
+    EXISTING_DEVS_NOF=${#EXISTING_DEVS[*]}
+    if [ $EXISTING_DEVS_NOF -lt $REQUIRED_DEVS_NOF ]; then 
+        ocf_log err "Number of existing devices=$EXISTING_DEVS_NOF less then required_devs_nof=$REQUIRED_DEVS_NOF"
+        exit $OCF_ERR_INSTALLED
+    fi
+
+}
+
+sg_persist_is_registered() {
+    for registered_dev in ${REGISTERED_DEVS[*]}
+    do
+        if [ "$registered_dev" == "$1" ]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+sg_persist_get_reservation_key() {
+    for array_index in ${!DEVS_WITH_RESERVATION[*]}
+    do
+        if [ "${DEVS_WITH_RESERVATION[$array_index]}" == "$1" ]; then
+            echo ${RESERVATION_KEYS[$array_index]}
+            return 0
+        fi
+    done
+    echo ""
+}
+
+sg_persist_echo_array() {
+    str_count=0
+    arr_str=""
+
+    for str in "$@"
+    do
+        arr_str="$arr_str[$str_count]:$str "    
+        str_count=$(($str_count+1))
+    done
+    echo $arr_str
+}
+
+sg_persist_parse_act_pending() {
+    
+    ACT_PENDING_TS=0
+    ACT_PENDING_SCORE=0
+
+    if [ -n "$ACT_PENDING" ]; then
+        ACT_PENDING_TS=${ACT_PENDING%%_*}
+        ACT_PENDING_SCORE=${ACT_PENDING##*_}
+    fi
+}
+
+sg_persist_clear_pending() {
+    if [ -n "$ACT_PENDING" ]; then 
+        DO_PENDING_UPDATE="YES"
+        NEW_PENDING=""
+    fi
+}
+
+sg_persist_new_master_score() {
+    DO_MASTER_SCORE_UPDATE="YES"
+    NEW_MASTER_SCORE=$1
+}
+
+sg_persist_new_pending() {
+    DO_PENDING_UPDATE="YES"
+    NEW_PENDING=$1
+}
+
+
+# Functions invoked by resource manager actions
+
+sg_persist_action_start() {
+
+    ocf_run $MASTER_SCORE_ATTRIBUTE --update=$MASTER_SCORE
+    ocf_run $PENDING_ATTRIBUTE --update=""
+
+    if [ $WORKING_DEVS_NOF -lt $REQUIRED_DEVS_NOF ]; then
+        ocf_log err "$RESOURCE: Number of working devices=$WORKING_DEVS_NOF less then required_devs_nof=$REQUIRED_DEVS_NOF"
+        exit $OCF_ERR_GENERIC
+    fi
+
+    for dev in ${WORKING_DEVS[*]}
+    do
+        if sg_persist_is_registered $dev ; then
+            : OK
+        else
+            ocf_run $SG_PERSIST --out --register --param-rk=0 --param-sark=$NODE_ID_HEX $dev
+            if [ $? -ne $OCF_SUCCESS ]
+            then
+                return $OCF_ERR_GENERIC
+            fi
+        fi
+    done
+
+    return $OCF_SUCCESS
+}
+
+sg_persist_action_stop() {
+
+    if [ ${#REGISTERED_DEVS[*]} -eq 0 ]; then
+        ocf_log debug "$RESOURCE stop: already no registrations"
+    else
+        # Clear preference for becoming master
+        ocf_run $MASTER_SCORE_ATTRIBUTE --delete
+        ocf_run $PENDING_ATTRIBUTE --delete
+
+        for dev in ${REGISTERED_DEVS[*]}
+        do
+            ocf_run $SG_PERSIST --out --register --param-rk=$NODE_ID_HEX --param-sark=0 $dev
+        done
+    fi
+
+    return $OCF_SUCCESS
+}
+
+sg_persist_action_monitor() {
+
+    ACT_MASTER_SCORE=`ocf_run -q $MASTER_SCORE_ATTRIBUTE --query --quiet 2>&1`
+    ocf_log debug "$RESOURCE monitor: ACT_MASTER_SCORE=$ACT_MASTER_SCORE"
+    
+    ACT_PENDING=`ocf_run $PENDING_ATTRIBUTE --query --quiet 2>&1`
+    ocf_log debug "$RESOURCE monitor: ACT_PENDING=$ACT_PENDING"
+
+    sg_persist_parse_act_pending
+    ocf_log debug "$RESOURCE monitor: ACT_PENDING_TS=$ACT_PENDING_TS"
+    ocf_log debug "$RESOURCE monitor: ACT_PENDING_VAL=$ACT_PENDING_SCORE"
+   
+    ocf_log debug "$MASTER_SCORE, $ACT_MASTER_SCORE, $ROLE"
+    
+    DO_MASTER_SCORE_UPDATE="NO"
+    DO_PENDING_UPDATE="NO"
+    if [ -n "$ACT_MASTER_SCORE" ] 
+    then
+        if [ $ACT_MASTER_SCORE -eq $MASTER_SCORE ]; then
+            sg_persist_clear_pending
+        else
+            case $ROLE in
+            Master)  
+                if [ $MASTER_SCORE -lt $ACT_MASTER_SCORE ]; then
+                    if [ -n "$ACT_PENDING" ] 
+                    then
+                        if [ $(($NOW-$ACT_PENDING_TS-$MASTER_SCORE_DELAY)) -ge 0 ]; then
+                            sg_persist_new_master_score $MASTER_SCORE
+                            sg_persist_clear_pending
+                        fi
+                    else
+                        if [ $MASTER_SCORE_DELAY -eq 0 ]; then
+                            sg_persist_new_master_score $MASTER_SCORE
+                            sg_persist_clear_pending
+                        else
+                            sg_persist_new_pending "${NOW}_${MASTER_SCORE}"
+                        fi
+                    fi
+                else
+                    sg_persist_new_master_score $MASTER_SCORE
+                    sg_persist_clear_pending
+                fi
+                ;;
+
+            Slave)
+                if [ $MASTER_SCORE -gt $ACT_MASTER_SCORE ]; then
+                    if [ -n "$ACT_PENDING" ]; then 
+                        if [ $(($NOW-$ACT_PENDING_TS-$MASTER_SCORE_DELAY)) -ge 0 ]; then
+                            sg_persist_new_master_score $MASTER_SCORE
+                            sg_persist_clear_pending
+                        fi
+                    else
+                        if [ $MASTER_SCORE_DELAY -eq 0 ]; then
+                            sg_persist_new_master_score $MASTER_SCORE
+                            sg_persist_clear_pending
+                        else
+                            sg_persist_new_pending "${NOW}_${MASTER_SCORE}"
+                        fi
+                    fi
+                else
+                    sg_persist_new_master_score $MASTER_SCORE
+                    sg_persist_clear_pending
+                fi
+                ;;
+
+            *)
+                ;;
+
+            esac
+        fi
+    fi
+
+    if [ $DO_MASTER_SCORE_UPDATE == "YES" ]; then
+        ocf_run $MASTER_SCORE_ATTRIBUTE --update=$NEW_MASTER_SCORE
+    fi
+
+    if [ $DO_PENDING_UPDATE == "YES" ]; then
+        ocf_run $PENDING_ATTRIBUTE --update=$NEW_PENDING
+    fi
+
+    if [ ${#REGISTERED_DEVS[*]} -eq 0 ]; then
+        ocf_log debug "$RESOURCE monitor: no registrations"
+        return $OCF_NOT_RUNNING
+    fi
+
+    if [ ${#RESERVED_DEVS[*]} -eq ${#WORKING_DEVS[*]} ]; then 
+        return $OCF_RUNNING_MASTER
+    fi
+
+    if [ ${#REGISTERED_DEVS[*]} -eq ${#WORKING_DEVS[*]} ]; then 
+        if [ $RESERVATION_TYPE -eq 7 ] || [ $RESERVATION_TYPE -eq 8 ]; then
+            if [ ${#DEVS_WITH_RESERVATION[*]} -gt 0 ]; then
+                return $OCF_RUNNING_MASTER
+             else
+                return $OCF_SUCCESS
+            fi
+        else
+            return $OCF_SUCCESS
+        fi
+    fi
+
+    ocf_log err "$RESOURCE monitor: unexpected state"
+    
+    return $OCF_ERR_GENERIC
+}
+
+sg_persist_action_promote() {
+
+    if [ ${#RESERVED_DEVS[*]} -gt 0 ]; then 
+        ocf_log info "$RESOURCE promote: already master"
+        return $OCF_SUCCESS
+    fi
+
+    for dev in ${WORKING_DEVS[*]}
+    do
+        reservation_key=`sg_persist_get_reservation_key $dev`
+        case $RESERVATION_TYPE in
+        1|3|5|6)        
+            if [ -z "$reservation_key" ]; then
+                ocf_run $SG_PERSIST --out --reserve --param-rk=$NODE_ID_HEX --prout-type=$RESERVATION_TYPE $dev
+                if [ $? -ne $OCF_SUCCESS ]; then
+                    return $OCF_ERR_GENERIC
+                fi
+            else
+                ocf_run $SG_PERSIST --out --preempt --param-sark=$reservation_key --param-rk=$NODE_ID_HEX --prout-type=$RESERVATION_TYPE $dev
+                if [ $? -ne $OCF_SUCCESS ]; then
+                    return $OCF_ERR_GENERIC
+                fi
+            fi
+            ;;
+
+        7|8) 
+            if [ -z "$reservation_key" ]; then
+                ocf_run $SG_PERSIST --out --reserve --param-rk=$NODE_ID_HEX --prout-type=$RESERVATION_TYPE $dev
+                if [ $? -ne $OCF_SUCCESS ]
+                then
+                    return $OCF_ERR_GENERIC
+                fi
+            else 
+                ocf_log info "$RESOURCE promote: there already exist an reservation holder, all registrants become reservation holders"
+                return $OCF_SUCCESS
+            fi
+            ;;
+  
+        *)
+            return $OCF_ERR_ARGS
+            ;;
+
+        esac
+    done
+
+    return $OCF_SUCCESS
+}
+
+sg_persist_action_demote() {
+    case $RESERVATION_TYPE in
+    1|3|5|6)
+        if [ ${#RESERVED_DEVS[*]} -eq 0 ]; then 
+            ocf_log info "$RESOURCE demote: already slave"
+            return $OCF_SUCCESS
+        fi
+
+        for dev in ${RESERVED_DEVS[*]}
+        do
+            ocf_run $SG_PERSIST --out --release --param-rk=$NODE_ID_HEX --prout-type=$RESERVATION_TYPE $dev
+            if [ $? -ne  $OCF_SUCCESS ]; then
+               return $OCF_ERR_GENERIC
+            fi
+        done
+        ;;
+
+    7|8)  #in case of 7/8, --release won't release the reservation unless unregister the key.    
+        if [ ${#REGISTERED_DEVS[*]} -eq 0 ]; then 
+            ocf_log info "$RESOURCE demote: already slave"
+            return $OCF_SUCCESS
+        fi
+
+        for dev in ${REGISTERED_DEVS[*]}
+        do
+            ocf_run $SG_PERSIST --out --register --param-rk=$NODE_ID_HEX --param-sark=0 $dev
+            if [ $? -ne $OCF_SUCCESS ]; then
+               return $OCF_ERR_GENERIC
+            fi
+        done
+        ;;
+   
+    *)
+        return $OCF_ERR_ARGS
+        ;;
+    esac
+
+    return $OCF_SUCCESS
+}
+
+sg_persist_action_notify() {
+    local n_type="$OCF_RESKEY_CRM_meta_notify_type"
+    local n_op="$OCF_RESKEY_CRM_meta_notify_operation"
+    set -- $OCF_RESKEY_CRM_meta_notify_active_resource
+    local n_active="$#"
+    set -- $OCF_RESKEY_CRM_meta_notify_stop_resource
+    local n_stop="$#"
+    set -- $OCF_RESKEY_CRM_meta_notify_start_resource
+    local n_start="$#"
+
+    ocf_log debug "$RESOURCE notify: $n_type for $n_op - counts: active $n_active - starting $n_start - stopping $n_stop"
+    
+    return $OCF_SUCCESS
+}
+
+sg_persist_action_validate_all () {
+
+    if [ "$OCF_RESKEY_CRM_meta_master_max" != "1" ] && [ "$RESERVATION_TYPE"  != "7" ] && [ "$RESERVATION_TYPE" != "8" ]; then
+        ocf_log err "Master options misconfigured."
+        exit $OCF_ERR_CONFIGURED
+    fi
+
+    return $OCF_SUCCESS
+}
+
+if [ $# -ne 1 ]; then
+    echo "Incorrect parameter count."
+    sg_persist_action_usage
+    exit $OCF_ERR_ARGS
+fi
+
+ACTION=$1
+case $ACTION in
+    meta-data)    
+        meta_data
+        ;;
+        
+    validate-all)    
+        sg_persist_init
+        sg_persist_action_validate_all
+        ;;
+
+    start|promote|monitor|stop|demote)
+        ocf_log debug "$RESOURCE: starting action \"$ACTION\""
+        sg_persist_init
+        sg_persist_action_$ACTION
+        exit $?
+        ;;
+        
+    notify)    
+        sg_persist_action_notify
+        exit $?
+        ;;
+
+    usage|help)    
+        sg_persist_action_usage
+        exit $OCF_SUCCESS
+        ;;
+
+    *)  
+        sg_persist_action_usage
+        exit $OCF_ERR_ARGS
+        ;;
+
+    esac
diff --git a/heartbeat/slapd b/heartbeat/slapd
index ffb40e8..c26b16f 100755
--- a/heartbeat/slapd
+++ b/heartbeat/slapd
@@ -268,7 +268,7 @@ slapd_pid()
       return $OCF_SUCCESS
     fi
 
-    ocf_log err "slapd pid file '$pid_file' empty."
+    ocf_exit_reason "slapd pid file '$pid_file' empty."
     return $OCF_ERR_GENERIC
   fi
 
@@ -316,7 +316,7 @@ slapd_start()
   elif [ -f "$config" ]; then
     options="$options -f $config"
   else
-    ocf_log err "slapd configuration '$config' does not exist."
+    ocf_exit_reason "slapd configuration '$config' does not exist."
     return $OCF_ERR_INSTALLED
   fi
 
@@ -331,7 +331,7 @@ slapd_start()
   fi
 
   if [ $rc -ne 0 ]; then
-    ocf_log err "slapd returned error."
+    ocf_exit_reason "slapd returned error."
 
     return $OCF_ERR_GENERIC
   fi
@@ -366,7 +366,7 @@ slapd_stop()
 
   terminate $pid TERM $OCF_RESKEY_stop_escalate; rc=$?
   if [ $rc -ne 0  ]; then
-    ocf_log err "slapd failed to stop. Escalating to KILL."
+    ocf_exit_reason "slapd failed to stop. Escalating to KILL."
     terminate $pid KILL; rc=$?
   fi
 
@@ -391,12 +391,12 @@ slapd_monitor()
   if [ $state -eq $OCF_NOT_RUNNING ]; then
     if [ -z "$1" ];then
       if ! ocf_is_probe; then
-        ocf_log err "slapd process not found."
+        ocf_exit_reason "slapd process not found."
       fi
     fi
     return $state
   elif [ $state -ne $OCF_SUCCESS ]; then
-    ocf_log err "slapd returned error."
+    ocf_exit_reason "slapd returned error."
     return $state
   fi
 
@@ -427,7 +427,7 @@ slapd_monitor()
     if ocf_is_probe; then
       ocf_log info "slapd configuration '$config' does not exist during probe."
     else
-      ocf_log err "slapd configuration '$config' does not exist."
+      ocf_exit_reason "slapd configuration '$config' does not exist."
       return $OCF_ERR_INSTALLED
     fi
   fi
@@ -447,12 +447,12 @@ slapd_monitor()
         ocf_log debug "slapd database with suffix '$suffix' reachable"
         ;;
       "49")
-        ocf_log err "slapd database with suffix '$suffix' unreachable. Invalid credentials."
+        ocf_exit_reason "slapd database with suffix '$suffix' unreachable. Invalid credentials."
         return $OCF_ERR_CONFIGURED
         ;;
       *)
         if [ -z "$1" ] || [ -n "$1" -a $rc -ne 1 ]; then
-          ocf_log err "slapd database with suffix '$suffix' unreachable. exit code ($rc)"
+          ocf_exit_reason "slapd database with suffix '$suffix' unreachable. exit code ($rc)"
         fi
         state=$OCF_ERR_GENERIC
         ;;
@@ -480,7 +480,7 @@ slapd_validate_all()
       if ocf_is_probe; then
         ocf_log info "slapd configuration '$config' does not exist during probe."
       else
-        ocf_log err "slapd configuration '$config' does not exist."
+        ocf_exit_reason "slapd configuration '$config' does not exist."
         return $OCF_ERR_INSTALLED
       fi
     fi
@@ -489,14 +489,14 @@ slapd_validate_all()
   if [ -z "$user" ]; then
     user=`id -nu 2>/dev/null`
   elif ! id "$user" >/dev/null 2>&1; then
-    ocf_log err "slapd user '$user' does not exist"
+    ocf_exit_reason "slapd user '$user' does not exist"
     return $OCF_ERR_INSTALLED
   fi
 
   if [ -z "$group" ]; then
     group=`id -ng 2>/dev/null`
   elif ! grep "^$group:" /etc/group >/dev/null 2>&1; then
-    ocf_log err "slapd group '$group' does not exist"
+    ocf_exit_reason "slapd group '$group' does not exist"
     return $OCF_ERR_INSTALLED
   fi
 
diff --git a/heartbeat/symlink b/heartbeat/symlink
index 214092d..1e36a9c 100755
--- a/heartbeat/symlink
+++ b/heartbeat/symlink
@@ -117,7 +117,7 @@ symlink_monitor() {
         rc=$OCF_NOT_RUNNING
     elif [ ! -L  "$OCF_RESKEY_link" ]; then
         if [ -z "$OCF_RESKEY_backup_suffix" ]; then
-            ocf_log err "$OCF_RESKEY_link exists but is not a symbolic link!"
+            ocf_exit_reason "$OCF_RESKEY_link exists but is not a symbolic link!"
             exit $OCF_ERR_INSTALLED
         else
             ocf_log debug "$OCF_RESKEY_link exists but is not a symbolic link, will be moved to ${OCF_RESKEY_link}${OCF_RESKEY_backup_suffix} on start"
@@ -128,7 +128,7 @@ symlink_monitor() {
         rc=$OCF_SUCCESS
     else
         if [ -z "$OCF_RESKEY_backup_suffix" ]; then
-            ocf_log err "$OCF_RESKEY_link does not point to ${OCF_RESKEY_target}!"
+            ocf_exit_reason "$OCF_RESKEY_link does not point to ${OCF_RESKEY_target}!"
             exit $OCF_ERR_INSTALLED
         else
             ocf_log debug "$OCF_RESKEY_link does not point to ${OCF_RESKEY_target}, will be moved to ${OCF_RESKEY_link}${OCF_RESKEY_backup_suffix} on start"
@@ -146,7 +146,7 @@ symlink_start() {
                 # have errored out. But there is a chance that
                 # something else put that file there after
                 # symlink_monitor ran.
-                ocf_log err "$OCF_RESKEY_link exists and no backup_suffix is set, won't overwrite."
+                ocf_exit_reason "$OCF_RESKEY_link exists and no backup_suffix is set, won't overwrite."
                 exit $OCF_ERR_GENERIC
             else
                 ocf_log debug "Found $OCF_RESKEY_link, moving to ${OCF_RESKEY_link}${OCF_RESKEY_backup_suffix}"
@@ -174,7 +174,7 @@ symlink_stop() {
             fi
             return $OCF_SUCCESS
         else
-            ocf_log err "Removing $OCF_RESKEY_link failed."
+            ocf_exit_reason "Removing $OCF_RESKEY_link failed."
             return $OCF_ERR_GENERIC
         fi
     else
@@ -184,11 +184,11 @@ symlink_stop() {
 
 symlink_validate_all() {
     if [ "x${OCF_RESKEY_link}" = "x" ]; then
-        ocf_log err "Mandatory parameter link is unset"
+        ocf_exit_reason "Mandatory parameter link is unset"
         exit $OCF_ERR_CONFIGURED
     fi
     if [ "x${OCF_RESKEY_target}" = "x" ]; then
-        ocf_log err "Mandatory parameter target is unset"
+        ocf_exit_reason "Mandatory parameter target is unset"
         exit $OCF_ERR_CONFIGURED
     fi
 
diff --git a/heartbeat/tomcat b/heartbeat/tomcat
index 23a7e2f..8b7fe31 100755
--- a/heartbeat/tomcat
+++ b/heartbeat/tomcat
@@ -157,7 +157,7 @@ rotate_catalina_out()
 	su - -s /bin/sh $RESOURCE_TOMCAT_USER \
 		-c "touch \"$CATALINA_BASE/logs/catalina_$CURRENT_ROTATELOG_SUFFIX.log\"" > /dev/null 2>&1
 	if [ $? -ne 0 ]; then
-		ocf_log err "$CATALINA_BASE/logs/catalina_$CURRENT_ROTATELOG_SUFFIX.log is not writable."
+		ocf_exit_reason "$CATALINA_BASE/logs/catalina_$CURRENT_ROTATELOG_SUFFIX.log is not writable."
 		return $OCF_ERR_GENERIC
 	fi
 
@@ -235,7 +235,7 @@ start_tomcat()
 		if [ $? -eq 0 ]; then
 			ocf_log debug "Rotate catalina.out succeeded."
 		else
-			ocf_log err "Rotate catalina.out failed. Avoid starting tomcat without catalina.out rotation."
+			ocf_exit_reason "Rotate catalina.out failed. Avoid starting tomcat without catalina.out rotation."
 			return $OCF_ERR_GENERIC
 		fi
 	fi
@@ -534,12 +534,12 @@ validate_all_tomcat()
 	check_binary $WGET
 
 	if [ -z "${TOMCAT_START_SCRIPT}" ]; then
-		ocf_log err "No default tomcat start script detected. Please specify start script location using the 'tomcat_start_script' option"
+		ocf_exit_reason "No default tomcat start script detected. Please specify start script location using the 'tomcat_start_script' option"
 		rc=$OCF_ERR_CONFIGURED
 	fi
 
 	if [ -n "$MAX_STOP_TIME" ] && [ "$MAX_STOP_TIME" -lt 0 ]; then
-		ocf_log err "max_stop_time must be set to a value greater than 0."
+		ocf_exit_reason "max_stop_time must be set to a value greater than 0."
 		rc=$OCF_ERR_CONFIGURED
 	fi
 
@@ -550,14 +550,14 @@ validate_all_tomcat()
 		ocf_log debug "grep port=\"$port\" $CATALINA_BASE/conf/server.xml"
 		grep "port=\"$port\"" $CATALINA_BASE/conf/server.xml > /dev/null 2>&1
 		if [ $? -ne 0 ]; then
-			ocf_log err "Your configured status URL specifies a port ($port), but the server does not have a connector listening to that port in $CATALINA_BASE/conf/server.xml"
+			ocf_exit_reason "Your configured status URL specifies a port ($port), but the server does not have a connector listening to that port in $CATALINA_BASE/conf/server.xml"
 			rc=$OCF_ERR_INSTALLED
 		fi
 	fi
 
 	if ocf_is_true ${CATALINA_ROTATE_LOG}; then
 		if [ ! -x "$ROTATELOGS" ]; then
-			ocf_log err "rotatelogs command does not exist."
+			ocf_exit_reason "rotatelogs command does not exist."
 			rc=$OCF_ERR_INSTALLED
 		fi
 	fi
@@ -610,6 +610,18 @@ JAVA_ENDORSED_DIRS="${OCF_RESKEY_java_endorsed_dirs}"
 LOGGING_CONFIG="${OCF_RESKEY_logging_config}"
 LOGGING_MANAGER="${OCF_RESKEY_logging_manager}"
 
+if [ -z "${TOMCAT_START_SCRIPT}" ]; then
+	if [ -e "$CATALINA_HOME/bin/catalina.sh" ]; then
+		TOMCAT_START_SCRIPT="$CATALINA_HOME/bin/catalina.sh"
+	elif [ -e "/usr/sbin/tomcat" ]; then
+		REDIRECT_DEFAULT_CONFIG=1
+		TOMCAT_START_SCRIPT="/usr/sbin/tomcat"
+	elif [ -e "/usr/sbin/tomcat6" ]; then
+		REDIRECT_DEFAULT_CONFIG=1
+		TOMCAT_START_SCRIPT="/usr/sbin/tomcat6"
+	fi
+fi
+
 LSB_STATUS_STOPPED=3
 if [ $# -ne 1 ]; then
 	usage
@@ -620,22 +632,13 @@ case "$COMMAND" in
 	help|usage) usage; exit $OCF_SUCCESS;;
 esac
 
-if [ -z "${TOMCAT_START_SCRIPT}" ]; then
-	if [ -e "$CATALINA_HOME/bin/catalina.sh" ]; then
-		TOMCAT_START_SCRIPT="$CATALINA_HOME/bin/catalina.sh"
-	elif [ -e "/usr/sbin/tomcat" ]; then
-		REDIRECT_DEFAULT_CONFIG=1
-		TOMCAT_START_SCRIPT="/usr/sbin/tomcat"
-	fi
-fi
-
 if [ ! -d "$JAVA_HOME" -o ! -d "$CATALINA_HOME" -o ! -d "$CATALINA_BASE" ]; then
 	case $COMMAND in
 		stop)		exit	$OCF_SUCCESS;;
 		monitor)	exit	$OCF_NOT_RUNNING;;
 		status)		exit	$LSB_STATUS_STOPPED;;
 	esac
-	ocf_log err "JAVA_HOME or CATALINA_HOME or CATALINA_BASE does not exist."
+	ocf_exit_reason "JAVA_HOME or CATALINA_HOME or CATALINA_BASE does not exist."
 	exit $OCF_ERR_INSTALLED
 fi
 
@@ -649,7 +652,7 @@ if [ ! -x "$JAVA" ]; then
 		monitor)	exit	$OCF_NOT_RUNNING;;
 		status)		exit	$LSB_STATUS_STOPPED;;
 	esac
-	ocf_log err "java command does not exist."
+	ocf_exit_reason "java command does not exist."
 	exit $OCF_ERR_INSTALLED
 fi
 
diff --git a/heartbeat/vsftpd b/heartbeat/vsftpd
new file mode 100755
index 0000000..ce9dfab
--- /dev/null
+++ b/heartbeat/vsftpd
@@ -0,0 +1,253 @@
+#!/bin/bash
+#
+# Resource script for vsftpd
+#
+# Description:  Manages vsftpd as an OCF resource in 
+#		an Active-Passive High Availability setup.
+#
+# Author:	Michel Rode <rode at b1-systems.de> : vsftpd script 
+# License:      GNU General Public License (GPLv2) 
+#
+#
+#	usage: $0 {start|stop|status|monitor|validate-all|meta-data}
+#
+#	The "start" arg starts vsftpd.
+#
+#	The "stop" arg stops it.
+#
+# OCF parameters:
+#  OCF_RESKEY_binpath
+#  OCF_RESKEY_conffile
+#  OCF_RESKEY_pidfile
+#
+##########################################################################
+# Initialization:
+
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+
+: ${OCF_RESKEY_binpath="/usr/sbin/vsftpd"}
+: ${OCF_RESKEY_conffile="/etc/vsftpd/vsftpd.conf"}
+: ${OCF_RESKEY_pidfile="/var/run/vsftpd.pid"}
+
+USAGE="Usage: $0 {start|stop|status|monitor|validate-all|meta-data}";
+
+##########################################################################
+
+usage() 
+{
+	echo $USAGE >&2
+}
+
+meta_data() 
+{
+cat <<END
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="vsftpd">
+<version>1.0</version>
+<longdesc lang="en">
+This script manages vsftpd
+</longdesc>
+<shortdesc lang="en">Manages an vsftpd</shortdesc>
+
+<parameters>
+
+<parameter name="binpath">
+<longdesc lang="en">
+The vsftpd binary path.
+For example, "/usr/sbin/vsftpd"
+</longdesc>
+<shortdesc lang="en">Full path to the vsftpd binary</shortdesc>
+<content type="string" default="/usr/sbin/vsftpd"/>
+</parameter>
+
+<parameter name="conffile">
+<longdesc lang="en">
+The vsftpd configuration file name with full path. 
+For example, "/etc/vsftpd/vsftpd.conf"
+</longdesc>
+<shortdesc lang="en">Configuration file name with full path</shortdesc>
+<content type="string" default="/etc/vsftpd/vsftpd.conf" />
+</parameter>
+
+<parameter name="pidfile">
+<longdesc lang="en">
+The vsftpd pidfile with full path. 
+For example, "/var/run/vsftpd.pid"
+</longdesc>
+<shortdesc lang="en">PID file with full path</shortdesc>
+<content type="string" default="/var/run/vsftpd.pid" />
+</parameter>
+
+</parameters>
+
+<actions>
+<action name="start" timeout="20s"/>
+<action name="stop" timeout="20s"/>
+<action name="monitor" depth="0" timeout="20s" interval="60s" />
+<action name="validate-all" timeout="20s"/>
+<action name="meta-data"  timeout="5s"/>
+</actions>
+</resource-agent>
+END
+exit $OCF_SUCCESS
+}
+
+get_pidfile()
+{
+	PIDFILE=$OCF_RESKEY_pidfile
+}
+
+vsftpd_status()
+{
+	if [ -n "$PIDFILE" -a -f $PIDFILE ]; then
+		# vsftpd is probably running
+		PID=`cat $PIDFILE`
+		if [ -n "$PID" ]; then
+			if ps -p $PID | grep vsftpd >/dev/null ; then
+				ocf_log info "vsftpd daemon running"
+				return $OCF_SUCCESS
+			else
+				ocf_log info "vsftpd daemon is not running but pid file exists"
+				return $OCF_ERR_GENERIC
+			fi
+		else
+			ocf_log err "PID file empty!"
+			return $OCF_ERR_GENERIC
+		fi
+	fi
+		
+	# vsftpd is not running
+	ocf_log info "vsftpd daemon is not running"
+	return $OCF_NOT_RUNNING
+}
+
+
+vsftpd_start()
+{
+	# if vsftpd is running return success
+	vsftpd_status
+	retVal=$?
+	if [ $retVal -eq $OCF_SUCCESS ]; then
+		exit $OCF_SUCCESS
+	elif [ $retVal -ne $OCF_NOT_RUNNING ]; then
+		ocf_log err "Error. Unknown status."
+		exit $OCF_ERR_GENERIC
+	fi
+
+	if [ -n "$OCF_RESKEY_binpath" ]; then
+		COMMAND="$OCF_RESKEY_binpath"
+	fi
+	if [ -n "$OCF_RESKEY_conffile" ]; then
+		COMMAND="$COMMAND $OCF_RESKEY_conffile"
+	fi
+
+	$COMMAND;
+ 	if [ $? -ne 0 ]; then
+		ocf_log err "Error. vsftpd returned error $?."
+		exit $OCF_ERR_GENERIC
+	fi
+
+	PID=$( pgrep $OCF_RESKEY_binpath )
+	case $? in
+		0)
+			ocf_log info "PID file (pid:${PID} at $PIDFILE) created for vsftpd."
+			ocf_log info "Started vsftpd."
+			echo $PID > $PIDFILE
+			exit $OCF_SUCCESS
+			;;
+		1)
+			rm -f "$PIDFILE" > /dev/null 2>&1
+			ocf_log info "$Error getting pid."
+			exit $OCF_ERR_GENERIC
+			;;
+		*)
+			rm -f "$PIDFILE" > /dev/null 2>&1
+			ocf_exit_reason "Error encountered detecting pid of vsftpd."
+			exit OCF_ERR_GENERIC
+			;;
+	esac
+
+}
+
+
+vsftpd_stop()
+{
+	if vsftpd_status ; then
+		PID=`cat $PIDFILE`
+		if [ -n "$PID" ] ; then
+			kill $PID
+			if [ $? -ne 0 ]; then
+				kill -s KILL $PID
+				if [ $? -ne 0 ]; then
+					ocf_log err "Error. Could not stop vsftpd daemon."
+					return $OCF_ERR_GENERIC
+				fi
+			fi
+			rm $PIDFILE 2>/dev/null
+		fi
+	fi
+	ocf_log info "Stopped vsftpd daemon."
+	exit $OCF_SUCCESS
+}
+
+vsftpd_monitor()
+{
+	vsftpd_status
+}
+
+vsftpd_validate_all()
+{
+	check_binary $OCF_RESKEY_binpath
+
+	if [ -n "$OCF_RESKEY_conffile" -a ! -f "$OCF_RESKEY_conffile" ]; then
+		ocf_log err "Config file $OCF_RESKEY_conffile does not exist."
+		exit $OCF_ERR_ARGS
+	fi
+	
+	return $OCF_SUCCESS
+}
+
+
+#
+# Main
+#
+ 
+if [ $# -ne 1 ]; then
+	usage
+	exit $OCF_ERR_ARGS
+fi
+
+case $1 in
+	start)	get_pidfile
+		vsftpd_start
+		;;
+	
+	stop)	get_pidfile
+		vsftpd_stop
+		;;
+
+	status)	get_pidfile
+		vsftpd_status
+		;;
+
+	monitor)get_pidfile
+		vsftpd_monitor
+		;;
+
+	validate-all)	vsftpd_validate_all
+			;;
+
+	meta-data)	meta_data
+			;;
+
+	usage)	usage
+		exit $OCF_SUCCESS
+		;;
+
+	*)	usage
+		exit $OCF_ERR_UNIMPLEMENTED
+		;;
+esac
+
diff --git a/include/agent_config.h.in b/include/agent_config.h.in
index acc6c18..aea63fd 100644
--- a/include/agent_config.h.in
+++ b/include/agent_config.h.in
@@ -1,2 +1,3 @@
 /* Where Resouce agents keep state files */
 #undef HA_RSCTMPDIR
+
diff --git a/ldirectord/Makefile.am b/ldirectord/Makefile.am
index 71461f9..5641e07 100644
--- a/ldirectord/Makefile.am
+++ b/ldirectord/Makefile.am
@@ -22,7 +22,7 @@ CLEANFILES	       	= ldirectord.8
 
 EXTRA_DIST              = ldirectord ldirectord.cf
 
-SUBDIRS			= logrotate.d init.d OCF
+SUBDIRS			= logrotate.d init.d OCF systemd
 
 ldirectord.8: ldirectord
 	$(POD2MAN) --section=8 $< > $@
diff --git a/ldirectord/ldirectord.cf b/ldirectord/ldirectord.cf
index 6948f55..d1e8426 100644
--- a/ldirectord/ldirectord.cf
+++ b/ldirectord/ldirectord.cf
@@ -295,9 +295,9 @@ virtual=192.168.6.240:80
 # Note that using checktype=connect and protocol=udp
 # will also effect ping checks
 #virtual=192.168.6.240:1812
-#	real=192.168.6.2::1812 gate
-#	real=192.168.6.3::1812 gate
-#	real=192.168.6.6::1812 gate
+#	real=192.168.6.2:1812 gate
+#	real=192.168.6.3:1812 gate
+#	real=192.168.6.6:1812 gate
 #	fallback=127.0.0.1:1812 gate
 #	scheduler=rr
 #	#persistent=600
@@ -305,7 +305,7 @@ virtual=192.168.6.240:80
 #	protocol=udp
 #	checktype=negotiate
 #	service=radius
-#	password="readuser"
+#	login="readuser"
 #	passwd="genericpassword"
 #	secret="somesecret"
 #	checktimeout=1
diff --git a/ldirectord/ldirectord.in b/ldirectord/ldirectord.in
index b871509..5fca81f 100644
--- a/ldirectord/ldirectord.in
+++ b/ldirectord/ldirectord.in
@@ -130,10 +130,10 @@ Timeout in seconds for negotiate checks.
 
 If defined in a virtual server section then the global value is overridden.
 
-If undefined then the value of connecttimeout is used.  connecttimeout is
+If undefined then the value of checktimeout is used.  checktimeout is
 also a global value that may be overridden by a per-virtual setting.
 
-If both negotiatetimeout and connecttimeout are unset, the default is used.
+If both negotiatetimeout and checktimeout are unset, the default is used.
 
 Default: 30 seconds
 
@@ -597,7 +597,7 @@ B<login = ">I<username>B<">
 For FTP, IMAP, LDAP, MySQL, Oracle, POP and PostgreSQL, the username
 used to log in.
 
-For Radius the passwd is used for the attribute User-Name.
+For Radius the username is used for the attribute User-Name.
 
 For SIP, the username is used as both the to and from address for an
 OPTIONS query.
@@ -700,6 +700,13 @@ The log format is:
 
 Default: no separate logging of service checks.
 
+B<ops = >B<yes> | B<no>
+
+Specify that a virtual service uses one-packet scheduling. This option
+can be used only for UDP services. If this option is specified, all connections
+are created only to schedule one packet. Option is useful to schedule
+UDP packets from same client port to different real servers.
+
 =head1 IPv6
 
 Directives for IPv6 are virtual6, real6, fallback6.
@@ -711,7 +718,9 @@ Following checktype and service are supported.
 
 B<checktype: >B<connect> | B<external> | B<external-perl> | B<negotiate> | B<off> | B<on> | B<checktimeout>I<N>
 
-B<service: >B<dns> | B<nntp> | B<none> | B<simpletcp> | B<sip>
+B<service: >B<dns> | B<http> | B<https> | B<nntp> | B<none> | B<simpletcp> | B<sip>
+
+Note: When using a service type with http or https, you need to install perl module perl-Net-INET6Glue.
 
 
 =head1 FILES
@@ -835,7 +844,12 @@ use Pod::Usage;
 #use English;
 #use Time::HiRes qw( gettimeofday tv_interval );
 use Socket;
-use Socket6 qw(NI_NUMERICHOST NI_NUMERICSERV NI_NAMEREQD getaddrinfo getnameinfo);
+use Socket6 qw(NI_NUMERICHOST NI_NUMERICSERV NI_NAMEREQD getaddrinfo getnameinfo inet_pton inet_ntop);
+# Workaround warnning messages : Three "_in6" symbols redefined.
+eval "use Socket6 qw(pack_sockaddr_in6)" unless defined &pack_sockaddr_in6;
+eval "use Socket6 qw(sockaddr_in6)" unless defined &sockaddr_in6;
+eval "use Socket6 qw(unpack_sockaddr_in6)" unless defined &unpack_sockaddr_in6;
+
 use Sys::Hostname;
 use POSIX qw(setsid :sys_wait_h);
 use Sys::Syslog qw(:DEFAULT setlogsock);
@@ -1362,6 +1376,7 @@ sub read_config
 			$vsrv{num_connects} = 0;
 			$vsrv{httpmethod} = "GET";
 			$vsrv{secret} = "";
+			$vsrv{ops} = "no";
 			push(@VIRTUAL, \%vsrv);
 			while(<CFGFILE>) {
 				$line++;
@@ -1495,6 +1510,12 @@ sub read_config
 					} else {
 						&config_error($line, "invalid protocol");
 					}
+				} elsif ($rcmd  =~ /^ops\s*=\s*(.*)/) {
+					if ($1 eq "yes" || $1 eq "no") {
+						$vsrv{ops} = $1;
+					} else {
+						&config_error($line, "ops must be 'yes' or 'no'");
+					}
 				} elsif ($rcmd =~ /^service\s*=\s*(.*)/) {
 					$1 =~ /(\w+)/ && ($1 eq "dns"	||
 							  $1 eq "ftp"	||
@@ -2237,6 +2258,9 @@ sub ld_setup
 			$$v{proto} = "-f";
 		}
 		$$v{flags} = "$$v{proto} " .  &get_virtual_option($v) . " ";
+		if ($$v{protocol} eq "udp" && $$v{ops} eq "yes") {
+			$$v{flags} .= "-o ";
+		}
 		$$v{flags} .= "-s $$v{scheduler} " if defined ($$v{scheduler});
 		if (defined $$v{persistent}) {
 			$$v{flags} .= "-p $$v{persistent} ";
@@ -2826,15 +2850,21 @@ sub check_http
 	}
 	my ($v, $r) = @_;
 
-	$$r{url} =~ /(http|https):\/\/([^:\/]+)(:([^\/]+))?(\/.*)/;
-	my $host = $2;
-	#my $port = $3;
-	my $uri = $4;
+	my $host = $$r{server};
 	my $virtualhost = (defined $$v{virtualhost} ? $$v{virtualhost} : $host);
 
 	&ld_debug(2, "check_http: url=\"$$r{url}\" "
 		. "virtualhost=\"$virtualhost\"");
 
+	if (inet_pton(AF_INET6,&ld_strip_brackets($host))) {
+		no warnings 'once';
+		require Net::INET6Glue::INET_is_INET6;
+		# Workaround for Net-HTTP IPv6 Address URLs Broken
+		@LWP::Protocol::http::EXTRA_SOCK_OPTS = (PeerAddr => $host,
+							 PeerHost => &ld_strip_brackets($host),
+							 Host => &ld_strip_brackets($host));
+	}
+
 	my $ua = new LWP::UserAgent(ssl_opts => { verify_hostname => 0 });
 
 	my $h = undef;
@@ -4439,13 +4469,14 @@ sub ld_emailalert_net_smtp
 	my $smtp = Net::SMTP->new($smtp_server);
 
 	if ($smtp) {
-		$smtp->mail("$ENV{USER}\@$hostname");
+		my $myusername = getpwuid( $< );
+		$smtp->mail("$myusername\@$hostname");
 		$smtp->to($to_addr);
 		$smtp->data();
 		if($EMAILALERTFROM) {
 			$smtp->datasend("From: $EMAILALERTFROM\n");
 		} else {
-			$smtp->datasend("From: $ENV{USER}\@$hostname\n");
+			$smtp->datasend("From: $myusername\@$hostname\n");
 		}
 		$smtp->datasend("To: $to_addr\n");
 		$smtp->datasend("Subject: $subject\n\n");
diff --git a/ldirectord/Makefile.am b/ldirectord/systemd/Makefile.am
similarity index 61%
copy from ldirectord/Makefile.am
copy to ldirectord/systemd/Makefile.am
index 71461f9..d349184 100644
--- a/ldirectord/Makefile.am
+++ b/ldirectord/systemd/Makefile.am
@@ -18,28 +18,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 MAINTAINERCLEANFILES    = Makefile.in
-CLEANFILES	       	= ldirectord.8
 
-EXTRA_DIST              = ldirectord ldirectord.cf
-
-SUBDIRS			= logrotate.d init.d OCF
-
-ldirectord.8: ldirectord
-	$(POD2MAN) --section=8 $< > $@
-
-sbin_SCRIPTS		= ldirectord
-
-if BUILD_POD_DOC
-man_MANS	        = ldirectord.8
+if HAVE_SYSTEMD
+systemdsystemunit_DATA = ldirectord.service
 endif
-
-harddir			= $(sysconfdir)/ha.d/resource.d
-
-.PHONY: install-exec-hook
-install-exec-hook:
-	$(INSTALL) -d $(DESTDIR)$(harddir)
-	cd $(DESTDIR)$(harddir) && ln -s -f $(sbindir)/ldirectord .
-
-.PHONY: uninstall-hook
-uninstall-hook:
-	rm -f $(DESTDIR)$(harddir)/ldirectord
diff --git a/ldirectord/systemd/ldirectord.service.in b/ldirectord/systemd/ldirectord.service.in
new file mode 100644
index 0000000..85a988b
--- /dev/null
+++ b/ldirectord/systemd/ldirectord.service.in
@@ -0,0 +1,15 @@
+[Unit]
+Description=Monitor and administer real servers in a LVS cluster of load balanced virtual servers
+
+[Service]
+ExecStart=@sbindir@/ldirectord start
+ExecStartPost=/usr/bin/touch /var/lock/subsys/ldirectord
+ExecStop=@sbindir@/ldirectord stop
+ExecStopPost=/usr/bin/rm -f /var/lock/subsys/ldirectord
+ExecReload=@sbindir@/ldirectord reload
+PIDFile=/var/run/ldirectord.ldirectord.pid
+Type=forking
+KillMode=none
+
+[Install]
+WantedBy=multi-user.target
diff --git a/resource-agents.spec.in b/resource-agents.spec.in
index 64557db..0211cae 100644
--- a/resource-agents.spec.in
+++ b/resource-agents.spec.in
@@ -30,6 +30,12 @@
 %bcond_ at rgmanager@ rgmanager
 %bcond_ at linux-ha@ linuxha
 
+# build with HA_BIN compatibility for the existing Heartbeat stack
+%bcond_ at compat-habindir@ compat_habindir
+%if %{with compat_habindir}
+%global _libexecdir %{_libdir}
+%endif
+
 Name:		resource-agents
 Summary:	Open Source HA Reusable Cluster Resource Scripts
 Version:	@version@
@@ -126,6 +132,10 @@ Requires:	perl-Net-IMAP-Simple-SSL
 Requires(post):	/sbin/chkconfig
 Requires(preun):/sbin/chkconfig
 %endif
+%if %{defined systemd_requires}
+BuildRequires:  systemd
+%{?systemd_requires}
+%endif
 
 %description -n ldirectord
 The Linux Director Daemon (ldirectord) was written by Jacob Rief.
@@ -174,6 +184,9 @@ export CFLAGS
 %configure \
 	%{?conf_opt_rsctmpdir:%conf_opt_rsctmpdir} \
 	%{conf_opt_fatal} \
+%if %{defined _unitdir}
+    --with-systemdsystemunitdir=%{_unitdir} \
+%endif
 	--with-pkg-name=%{name} \
 	--with-ras-set=%{rasset}
 
@@ -198,7 +211,11 @@ rm -rf %{buildroot}/usr/share/doc/resource-agents
 test -d %{buildroot}/sbin || mkdir %{buildroot}/sbin
 (
   cd %{buildroot}/sbin
-  ln -sf /%{_sysconfdir}/init.d/ldirectord rcldirectord 
+%if %{defined _unitdir}
+  ln -s /usr/sbin/service rcldirectord
+%else
+  ln -sf /%{_sysconfdir}/init.d/ldirectord rcldirectord
+%endif
 ) || true
 %endif
 %endif
@@ -270,10 +287,26 @@ ccs_update_schema > /dev/null 2>&1 ||:
 
 %if 0%{?suse_version}
 %preun -n ldirectord
+%if %{defined _unitdir}
+%service_del_preun ldirectord.service
+%else
 %stop_on_removal ldirectord
+%endif
 %postun -n ldirectord
+%if %{defined _unitdir}
+%service_del_postun ldirectord.service
+%else
 %insserv_cleanup
 %endif
+%post -n ldirectord
+%if %{defined _unitdir}
+%service_add_post ldirectord.service
+%endif
+%pre -n ldirectord
+%if %{defined _unitdir}
+%service_add_pre ldirectord.service
+%endif
+%endif
 
 %if 0%{?fedora}
 %preun -n ldirectord
@@ -292,7 +325,12 @@ ccs_update_schema > /dev/null 2>&1 ||:
 %dir %{_sysconfdir}/ha.d
 %dir %{_sysconfdir}/ha.d/resource.d
 %{_sysconfdir}/ha.d/resource.d/ldirectord
+%if %{defined _unitdir}
+%{_unitdir}/ldirectord.service
+%exclude %{_sysconfdir}/init.d/ldirectord
+%else
 %{_sysconfdir}/init.d/ldirectord
+%endif
 %if 0%{?suse_version}
 /sbin/rcldirectord
 %endif
diff --git a/rgmanager/src/resources/Makefile.am b/rgmanager/src/resources/Makefile.am
index 7719f81..862bdf5 100644
--- a/rgmanager/src/resources/Makefile.am
+++ b/rgmanager/src/resources/Makefile.am
@@ -26,9 +26,10 @@ RESOURCES		= service.sh ip.sh nfsclient.sh nfsexport.sh \
 			  script.sh netfs.sh clusterfs.sh smb.sh \
 			  apache.sh openldap.sh samba.sh mysql.sh \
 			  postgres-8.sh tomcat-5.sh lvm.sh \
-			  vm.sh SAPInstance SAPDatabase named.sh \
+			  vm.sh SAPInstance SAPDatabase named.sh db2.sh \
 			  ASEHAagent.sh drbd.sh nfsserver.sh \
-			  tomcat-6.sh orainstance.sh oralistener.sh oracledb.sh
+			  tomcat-6.sh orainstance.sh oralistener.sh oracledb.sh \
+			  bind-mount.sh
 
 METADATA		= apache.metadata openldap.metadata samba.metadata \
 			  mysql.metadata postgres-8.metadata \
diff --git a/rgmanager/src/resources/apache.metadata b/rgmanager/src/resources/apache.metadata
index 93d5fd8..e747d16 100644
--- a/rgmanager/src/resources/apache.metadata
+++ b/rgmanager/src/resources/apache.metadata
@@ -21,6 +21,16 @@
 	    <content type="string"/>
         </parameter>
 
+        <parameter name="httpd" >
+	    <longdesc lang="en">
+	        Define the full path to the httpd binary.
+	    </longdesc>
+            <shortdesc lang="en">
+                httpd binary
+            </shortdesc>
+	    <content type="string" />
+        </parameter>
+
         <parameter name="server_root">
             <longdesc lang="en">
                 Define an alternate initial ServerRoot
diff --git a/rgmanager/src/resources/apache.sh b/rgmanager/src/resources/apache.sh
index a428780..2230ca8 100755
--- a/rgmanager/src/resources/apache.sh
+++ b/rgmanager/src/resources/apache.sh
@@ -28,10 +28,13 @@ export PATH=/bin:/sbin:/usr/bin:/usr/sbin
 . $(dirname $0)/utils/messages.sh
 . $(dirname $0)/utils/ra-skelet.sh
 
-if [ -x /usr/sbin/httpd ]; then
-	declare APACHE_HTTPD=/usr/sbin/httpd
-elif [ -x /usr/sbin/apache2 ]; then
-	declare APACHE_HTTPD=/usr/sbin/apache2
+APACHE_HTTPD=$OCF_RESKEY_httpd
+if [ -z "$APACHE_HTTPD" ]; then
+	if [ -x /usr/sbin/httpd ]; then
+		declare APACHE_HTTPD=/usr/sbin/httpd
+	elif [ -x /usr/sbin/apache2 ]; then
+		declare APACHE_HTTPD=/usr/sbin/apache2
+	fi
 fi
 declare APACHE_serverConfigFile
 declare APACHE_pid_file="`generate_name_for_pid_file`"
@@ -60,6 +63,11 @@ verify_all()
 		return $OCF_ERR_ARGS
 	fi
 
+	if [ -n "$OCF_RESKEY_httpd" ] && ! [ -e $OCF_RESKEY_httpd ]; then
+		clog_service_verify $CLOG_FAILED "Invalid httpd binary, $OCF_RESKEY_http does not exist"
+		return $OCF_ERR_ARGS
+	fi 
+
 	if [ -z "$OCF_RESKEY_service_name" ]; then
 		clog_service_verify $CLOG_FAILED_NOT_CHILD
 		return $OCF_ERR_ARGS
diff --git a/rgmanager/src/resources/bind-mount.sh b/rgmanager/src/resources/bind-mount.sh
new file mode 100755
index 0000000..077e0a3
--- /dev/null
+++ b/rgmanager/src/resources/bind-mount.sh
@@ -0,0 +1,167 @@
+#!/bin/bash
+#
+#  Copyright Red Hat Inc., 2014
+#
+#  This program is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU General Public License as published by the
+#  Free Software Foundation; either version 2, or (at your option) any
+#  later version.
+#
+#  This program is distributed in the hope that it will be useful, but
+#  WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; see the file COPYING.  If not, write to the
+#  Free Software Foundation, Inc.,  675 Mass Ave, Cambridge, 
+#  MA 02139, USA.
+#
+
+#
+# Bind mount script - mounts parent file system -o bind in another
+# location
+#
+
+LC_ALL=C
+LANG=C
+PATH=/bin:/sbin:/usr/bin:/usr/sbin
+export LC_ALL LANG PATH
+
+. $(dirname $0)/ocf-shellfuncs
+. $(dirname $0)/utils/fs-lib.sh
+
+export IS_BIND_MOUNT=1
+export OCF_RESKEY_use_findmnt=0
+export OCF_RESKEY_options="bind"
+export OCF_RESKEY_device="$OCF_RESKEY_source"
+rv=0
+
+do_metadata()
+{
+	cat <<EOT
+<?xml version="1.0" ?>
+<resource-agent name="bind-mount" version="rgmanager 2.0">
+	<version>1.0</version>
+
+	<longdesc lang="en">
+		Defines a bind mount.
+	</longdesc>
+
+	<shortdesc lang="en">
+		Defines a bind mount.
+	</shortdesc>
+
+	<parameters>
+
+		<parameter name="name" primary="1" unique="1">
+			<longdesc lang="en">
+			Symbolic name for this bind mount.
+			</longdesc>
+			<shortdesc lang="en">
+			Bind Mount Name
+			</shortdesc>
+		<content type="string"/>
+		</parameter>
+
+		<parameter name="mountpoint" unique="1" required="1">
+			<longdesc lang="en">
+			Target of this bind mount
+			</longdesc>
+			<shortdesc lang="en">
+			Target mountpoint
+			</shortdesc>
+		<content type="string"/>
+		</parameter>
+
+		<parameter name="source" required="1">
+			<longdesc lang="en">
+			Source of the bind mount
+			</longdesc>
+			<shortdesc lang="en">
+			Source of the bind mount
+			</shortdesc>
+		<content type="string"/>
+		</parameter>
+
+		<parameter name="force_unmount">
+			<longdesc lang="en">
+				If set, the cluster will kill all processes using 
+				this file system when the resource group is 
+				stopped.  Otherwise, the unmount will fail, and
+				the resource group will be restarted.
+			</longdesc>
+			<shortdesc lang="en">
+				Force Unmount
+			</shortdesc>
+		<content type="boolean"/>
+		</parameter>
+	</parameters>
+
+	<actions>
+		<action name="start" timeout="5"/>
+		<action name="stop" timeout="5"/>
+		<action name="recover" timeout="5"/>
+
+		<action name="status" timeout="5" interval="1h"/>
+		<action name="monitor" timeout="5" interval="1h"/>
+
+		<action name="meta-data" timeout="5"/>
+		<action name="verify-all" timeout="30"/>
+	</actions>
+
+	<special tag="rgmanager">
+		<child type="nfsexport" forbid="1"/>
+		<child type="nfsclient"/>
+	</special>
+
+</resource-agent>
+EOT
+}
+
+verify_source()
+{
+	if [ -z "$OCF_RESKEY_source" ]; then
+		ocf_log err "No source specified."
+		return $OCF_ERR_ARGS
+	fi
+
+	[ -d "$OCF_RESKEY_source" ] && return 0
+
+	ocf_log err "$OCF_RESKEY_source is not a directory"
+	
+	return $OCF_ERR_ARGS
+}
+
+verify_mountpoint()
+{
+	if [ -z "$OCF_RESKEY_mountpoint" ]; then
+		ocf_log err "No target path specified."
+		return $OCF_ERR_ARGS
+	fi
+
+	[ -d "$OCF_RESKEY_mountpoint" ] && return 0
+
+	mkdir -p $OCF_RESKEY_mountpoint && return 0
+
+	ocf_log err "$OCF_RESKEY_mountpoint is not a directory and could not be created"
+	
+	return $OCF_ERR_ARGS
+}
+
+do_validate()
+{
+	declare -i ret=0
+
+	verify_source || ret=$OCF_ERR_ARGS
+	verify_mountpoint || ret=$OCF_ERR_ARGS
+
+	return $ret
+}
+
+do_pre_mount()
+{
+	do_validate || exit $OCF_ERR_ARGS
+}
+
+main $*
diff --git a/rgmanager/src/resources/db2.sh b/rgmanager/src/resources/db2.sh
new file mode 100755
index 0000000..f396ff6
--- /dev/null
+++ b/rgmanager/src/resources/db2.sh
@@ -0,0 +1,133 @@
+#!/bin/bash
+#
+# Copyright (c) 2011 Holger Teutsch <holger.teutsch at web.de>
+# Copyright (c) 2014 David Vossel <dvossel at redhat.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+
+# NOTE:
+#
+# This agent is a wrapper around the heartbeat/db2 agent which limits the heartbeat
+# db2 agent to Standard role support.  This allows cluster managers such as rgmanager
+# which do not have multi-state resource support to manage db2 instances with
+# a limited feature set.
+#
+
+export LC_ALL=C
+export LANG=C
+export PATH=/bin:/sbin:/usr/bin:/usr/sbin
+. $(dirname $0)/ocf-shellfuncs
+
+meta_data() {
+cat <<END
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="db2.sh">
+<version>1.0</version>
+<longdesc lang="en">
+Resource Agent that manages an IBM DB2 LUW databases in Standard role. Multiple partitions are supported.
+
+When partitions are in use, each partition must be configured as a separate primitive resource.
+
+</longdesc>
+<shortdesc lang="en">Resource Agent that manages an IBM DB2 LUW databases in Standard role with multiple partition support.</shortdesc>
+
+<parameters>
+<parameter name="instance" unique="1" required="1">
+<longdesc lang="en">
+The instance of the database(s).
+</longdesc>
+<shortdesc lang="en">instance</shortdesc>
+<content type="string" default="" />
+</parameter>
+<parameter name="dblist" unique="0" required="0">
+<longdesc lang="en">
+List of databases to be managed, e.g "db1 db2".
+Defaults to all databases in the instance.
+</longdesc>
+<shortdesc lang="en">List of databases to be managed</shortdesc>
+<content type="string"/>
+</parameter>
+<parameter name="dbpartitionnum" unique="0" required="0">
+<longdesc lang="en">
+The number of the partion (DBPARTITIONNUM) to be managed.
+</longdesc>
+<shortdesc lang="en">database partition number (DBPARTITIONNUM)</shortdesc>
+<content type="string" default="0" />
+</parameter>
+</parameters>
+
+<actions>
+<action name="start" timeout="120"/>
+<action name="stop" timeout="120"/>
+<action name="monitor" depth="0" timeout="60" interval="20"/>
+<action name="monitor" depth="0" timeout="60" role="Master" interval="22"/>
+<action name="validate-all" timeout="5"/>
+<action name="meta-data" timeout="5"/>
+</actions>
+</resource-agent>
+END
+}
+
+heartbeat_db2_wrapper()
+{
+	# default heartbeat agent ocf root.
+	export OCF_ROOT=/usr/lib/ocf
+	heartbeat_db2="${OCF_ROOT}/resource.d/heartbeat/db2"
+
+	if ! [ -a $heartbeat_db2 ]; then
+		echo "heartbeat db2 agent not found at '${heartbeat_db2}'"
+		exit $OCF_ERR_INSTALLED 
+	fi
+
+	$heartbeat_db2 $1
+}
+
+case $1 in
+	meta-data)
+		meta_data
+		exit 0
+		;;
+	validate-all)
+		heartbeat_db2_wrapper $1	
+		exit $?
+		;;
+	start)
+		heartbeat_db2_wrapper $1	
+		exit $?
+		;;
+	stop)
+		heartbeat_db2_wrapper $1	
+		exit $?
+		;;
+	status|monitor)
+		heartbeat_db2_wrapper "monitor"
+		exit $?
+		;;
+	restart)
+		heartbeat_db2_wrapper "stop"
+		rc=$?
+		if [ $rc -ne 0 ]; then
+			exit $rc
+		fi 
+		heartbeat_db2_wrapper "start"
+		exit $?
+		;;
+	*)
+		echo "Usage: db2.sh {start|stop|monitor|validate-all|meta-data}"
+		exit $OCF_ERR_UNIMPLEMENTED
+		;;
+esac
diff --git a/rgmanager/src/resources/fs.sh.in b/rgmanager/src/resources/fs.sh.in
index 5ece354..c041fb7 100644
--- a/rgmanager/src/resources/fs.sh.in
+++ b/rgmanager/src/resources/fs.sh.in
@@ -293,7 +293,7 @@ verify_options()
 
 			if [ "$OCF_RESKEY_fstype" = "ext3" ] ||
 			   [ "$OCF_RESKEY_fstype" = "ext4" ]; then
-				case $0 in
+				case $o in
 				noload|data=*)
 					continue
 					;;
diff --git a/rgmanager/src/resources/lvm_by_vg.sh b/rgmanager/src/resources/lvm_by_vg.sh
index d8a3e31..ab60970 100755
--- a/rgmanager/src/resources/lvm_by_vg.sh
+++ b/rgmanager/src/resources/lvm_by_vg.sh
@@ -509,7 +509,7 @@ function vg_stop_single
 
 	#  Make sure we are the owner before we strip the tags
 	vg_owner
-	if [ $? -ne 0 ]; then
+	if [ $? -eq 1 ]; then
 		strip_tags
 	fi
 
diff --git a/rgmanager/src/resources/mysql.sh b/rgmanager/src/resources/mysql.sh
index c7a7723..736a91a 100755
--- a/rgmanager/src/resources/mysql.sh
+++ b/rgmanager/src/resources/mysql.sh
@@ -68,6 +68,8 @@ verify_all()
 
 start()
 {
+	declare username=""
+
         if status; then
                 ocf_log info "Starting Service $OCF_RESOURCE_INSTANCE > Already running"
                 return $OCF_SUCCESS
@@ -75,7 +77,13 @@ start()
 
 	clog_service_start $CLOG_INIT
 
-	create_pid_directory
+	# Pull out the user name from the options argument if it is set.
+	# We need this to properly set the pidfile permissions
+	if [ -n "$OCF_RESKEY_mysqld_options" ]; then
+		username=$(echo "$OCF_RESKEY_mysqld_options" | sed -n -e 's/^.*--user=\(\S*\)[[:space:]]*.*$/\1/p;s/^.*-u[[:space:]]*\(\S*\)[[:space:]]*.*$/\1/p')
+	fi
+
+	create_pid_directory "$username"
 	check_pid_file "$MYSQL_pid_file"
 
 	if [ $? -ne 0 ]; then
diff --git a/rgmanager/src/resources/named.sh b/rgmanager/src/resources/named.sh
index f73bf89..e0d8ebe 100755
--- a/rgmanager/src/resources/named.sh
+++ b/rgmanager/src/resources/named.sh
@@ -106,10 +106,17 @@ generate_config_file()
 start()
 {
 	declare ip_list;
+	declare username=""
 	
 	clog_service_start $CLOG_INIT
 
-	create_pid_directory
+	# Pull out the user name from the options argument if it is set.
+	# We need this to properly set the pidfile permissions
+	if [ -n "$OCF_RESKEY_named_options" ]; then
+		username=$(echo "$OCF_RESKEY_named_options" | sed -n -e 's/^.*-u[[:space:]]*\(\S*\)[[:space:]]*.*$/\1/p')
+	fi
+
+	create_pid_directory "$username"
 	create_conf_directory "$NAMED_conf_dir"
 	check_pid_file "$NAMED_pid_file"
 
@@ -137,6 +144,8 @@ start()
 		return $OCF_ERR_GENERIC
 	fi
 
+	[ -x /sbin/portrelease ] && /sbin/portrelease named &>/dev/null
+
 	generate_config_file "$OCF_RESKEY_config_file" "$NAMED_gen_config_file" "$ip_list"
 
 	$NAMED_NAMED -c "$NAMED_gen_config_file" $OCF_RESKEY_named_options
diff --git a/rgmanager/src/resources/nfsserver.sh b/rgmanager/src/resources/nfsserver.sh
old mode 100755
new mode 100644
index e2161dc..4d98230
--- a/rgmanager/src/resources/nfsserver.sh
+++ b/rgmanager/src/resources/nfsserver.sh
@@ -23,6 +23,11 @@ if [ $SELINUX_ENABLED ]; then
 	export SELINUX_LABEL="$(ls -ldZ /var/lib/nfs/statd | cut -f4 -d' ')"
 fi
 
+# strip trailing / off so pattern matching will work consistently.
+while [ "${OCF_RESKEY_path#${OCF_RESKEY_path%?}}" = "/" ]
+do
+	OCF_RESKEY_path="${OCF_RESKEY_path%/}"
+done
 
 log_do()
 {
@@ -97,6 +102,21 @@ meta_data()
 	    <content type="string" default=".clumanager/nfs"/>
         </parameter>
 
+        <parameter name="statdport">
+            <longdesc lang="en">
+		Specifies the port number used for RPC listener sockets. If
+		this option is not specified, rpc.statd chooses a random
+		ephemeral port for each listener socket. This option can be
+		used to fix the port value of its listeners when SM_NOTIFY
+		requests must traverse a firewall between
+		clients and servers.
+            </longdesc>
+            <shortdesc lang="en">
+                This is the port where rpc.statd should listen on.
+            </shortdesc>
+	    <content type="integer" default=""/>
+        </parameter>
+
     </parameters>
 
     <actions>
@@ -153,10 +173,26 @@ verify_nfspath()
 }
 
 
+verify_statdport()
+{
+	if [ -z "$OCF_RESKEY_statdport" ]; then
+		# this is fine, statdport is optional
+		return 0
+	fi
+
+	[ $OCF_RESKEY_statdport -gt 0 && $OCF_RESKEY_statdport -le 65535 ] && return 0
+
+	ocf_log err "$OCF_RESKEY_statdport is not a valid port number"
+
+	return $OCF_ERR_ARGS
+}
+
+
 verify_all()
 {
 	verify_path || return 1
 	verify_nfspath || return 1
+	verify_statdport || return 1
 
 	return 0
 }
@@ -339,6 +375,7 @@ cleanup_tree()
 start_locking()
 {
 	declare ret
+	declare statdport=""
 	[ -x /sbin/rpc.statd ] || return 1
 	
 	#
@@ -352,6 +389,10 @@ start_locking()
 		return 0
 	fi
 
+	if [ -n "$OCF_RESKEY_statdport" ]; then
+		statdport="-p $OCF_RESKEY_statdport"
+	fi
+
 	#
 	# Set this resrouce script as the callout program.  We are evil.
 	# In cases where we want to preserve lock information, this is needed
@@ -359,7 +400,7 @@ start_locking()
 	#
 	ocf_log info "Starting rpc.statd"
 	rm -f /var/run/sm-notify.pid
-	rpc.statd -H $0 -d
+	rpc.statd -H $0 -d $statdport
 	ret=$?
 	if [ $ret -ne 0 ]; then
 		ocf_log err "Failed to start rpc.statd"
@@ -401,22 +442,41 @@ killkill()
 	done
 }
 
-
-stop_locking()
+stop_process()
 {
-	declare ret 
+	declare process=$1
 
-	ocf_log info "Stopping rpc.statd"
-	if terminate rpc.statd; then
-		ocf_log debug "rpc.statd is stopped"
+	ocf_log info "Stopping $process"
+	if terminate $process; then
+		ocf_log debug "$process is stopped"
 	else
-		if killkill rpc.statd; then
-			ocf_log debug "rpc.statd is stopped"
+		if killkill $process; then
+			ocf_log debug "$process is stopped"
 		else
-			ocf_log debug "Failed to stop rpc.statd"
+			ocf_log debug "Failed to stop $process"
 			return 1
 		fi
 	fi
+	return 0
+}
+
+stop_locking()
+{
+	ret=0
+
+	# sm-notify can prevent umount of /var/lib/nfs/statd if
+	# it is still trying to notify unresponsive clients.
+	stop_process sm-notify
+	if [ $? -ne 0 ]; then
+		ret=1
+	fi
+
+	stop_process rpc.statd
+	if [ $? -ne 0 ]; then
+		ret=1
+	fi
+
+	return $ret
 }
 
 
diff --git a/rgmanager/src/resources/oracledb.sh b/rgmanager/src/resources/oracledb.sh
old mode 100644
new mode 100755
index 3ccc47a..a27a55f
--- a/rgmanager/src/resources/oracledb.sh
+++ b/rgmanager/src/resources/oracledb.sh
@@ -361,7 +361,7 @@ force_cleanup()
 	declare pid
 
 	# Patch from Shane Bradley to fix 471266
-	pids=`ps ax | grep $ORACLE_HOME | grep "ora_.*_${ORACLE_SID}" | grep -v grep | awk '{print $1}'`
+	pids=`ps ax | grep "ora_.*_${ORACLE_SID}" | grep -v grep | awk '{print $1}'`
 
 	ocf_log error "Not all Oracle processes for $ORACLE_SID exited cleanly, killing"
 
@@ -388,7 +388,8 @@ exit_idle()
 	declare -i n=0
 
 	ocf_log debug "Waiting for Oracle processes for $ORACLE_SID to terminate..."
-	while ps ax | grep $ORACLE_HOME | grep "ora_.*_${ORACLE_SID}" | grep -v grep | awk '{print $1}'; do
+	# grep -q "." keeps this loop going if the previous commands produce any stdout
+	while ps ax | grep "ora_.*_${ORACLE_SID}" | grep -v grep | awk '{print $1}' | grep -q "."; do
 		if [ $n -ge 90 ]; then
 			ocf_log debug "Timed out while waiting for Oracle processes for $ORACLE_SID to terminate"
 			force_cleanup
diff --git a/rgmanager/src/resources/orainstance.sh b/rgmanager/src/resources/orainstance.sh
index 9254744..b083224 100755
--- a/rgmanager/src/resources/orainstance.sh
+++ b/rgmanager/src/resources/orainstance.sh
@@ -207,7 +207,7 @@ exit_idle() {
 	declare -i n=0
 	
 	ocf_log debug "Waiting for Oracle processes for $ORACLE_SID to terminate..."
-	while ps ax | grep $ORACLE_SID | grep -q -v $LSNR_PROCNAME | grep -q -v grep; do
+	while ps ax | grep $ORACLE_SID | grep -v grep | grep -q -v $LSNR_PROCNAME; do
 		if [ $n -ge 90 ]; then
 			ocf_log debug "Timed out while waiting for Oracle processes for $ORACLE_SID to terminate"
 			force_cleanup
diff --git a/rgmanager/src/resources/oralistener.sh b/rgmanager/src/resources/oralistener.sh
index 90808ed..94698a9 100755
--- a/rgmanager/src/resources/oralistener.sh
+++ b/rgmanager/src/resources/oralistener.sh
@@ -107,6 +107,12 @@ start() {
 stop() {
 	ocf_log info "Stopping listener $LISTENER"
 
+	monitor $OCF_CHECK_LEVEL
+	if [ $? -ne 0 ]; then
+		ocf_log info "Listener $LISTENER already stopped"
+		return 0
+	fi
+
 	lsnrctl_stdout=$(lsnrctl stop "$LISTENER")
 	if [ $? -ne 0 ]; then
 		ocf_log debug "stop listener $LISTENER failed $lsnrctl_stdout"
diff --git a/rgmanager/src/resources/postgres-8.metadata b/rgmanager/src/resources/postgres-8.metadata
index 7b05d31..e7ae14f 100644
--- a/rgmanager/src/resources/postgres-8.metadata
+++ b/rgmanager/src/resources/postgres-8.metadata
@@ -63,6 +63,16 @@
 		<content type="integer" />
 	</parameter>
 
+        <parameter name="startup_wait">
+                <longdesc lang="en">
+                        Wait X seconds for correct end of service startup
+                </longdesc>
+                <shortdesc lang="en">
+			Wait X seconds for correct end of service startup.
+                </shortdesc>
+                <content type="number" default="10"/>
+        </parameter>
+
         <parameter name="service_name" inherit="service%name">
             <longdesc lang="en">
 	    	Inherit the service name.  We need to know
diff --git a/rgmanager/src/resources/postgres-8.sh b/rgmanager/src/resources/postgres-8.sh
index 023a5e5..76c22dd 100755
--- a/rgmanager/src/resources/postgres-8.sh
+++ b/rgmanager/src/resources/postgres-8.sh
@@ -34,7 +34,9 @@ declare PSQL_conf_dir="`generate_name_for_conf_dir`"
 declare PSQL_gen_config_file="$PSQL_conf_dir/postgresql.conf"
 declare PSQL_kill_timeout="5"
 declare PSQL_stop_timeout="15"
-declare PSQL_wait_after_start="2"
+if [ -z "$OCF_RESKEY_startup_wait" ]; then
+	OCF_RESKEY_startup_wait=10
+fi
 
 verify_all()
 {
@@ -117,6 +119,7 @@ generate_config_file()
 start()
 {
 	declare pguser_group
+	declare count=0
 	clog_service_start $CLOG_INIT
 
 	create_pid_directory
@@ -134,7 +137,7 @@ start()
 	# change it to be owned by the postgres user so that
 	# postmaster doesn't complain.
 	#
-	pguser_group=`groups $OCF_RESKEY_postmaster_user | cut -f1 -d ' '`
+	pguser_group=`groups $OCF_RESKEY_postmaster_user | cut -f3 -d ' '`
 	touch $PSQL_pid_file
 	chown $OCF_RESKEY_postmaster_user.$pguser_group $PSQL_pid_file
 
@@ -155,12 +158,17 @@ start()
 	su - "$OCF_RESKEY_postmaster_user" -c "$PSQL_POSTMASTER -c config_file=\"$PSQL_gen_config_file\" \
 		$OCF_RESKEY_postmaster_options" &> /dev/null &
 
-	# We need to sleep for a second to allow pg_ctl to detect that we've started.
-	sleep $PSQL_wait_after_start
+	# We need to sleep briefly to allow pg_ctl to detect that we've started.
 	# We need to fetch "-D /path/to/pgsql/data" from $OCF_RESKEY_postmaster_options
-	su - "$OCF_RESKEY_postmaster_user" -c "$PSQL_CTL status $OCF_RESKEY_postmaster_options" &> /dev/null
+	until [ "$count" -gt "$OCF_RESKEY_startup_wait" ] || 
+		[ `su - "$OCF_RESKEY_postmaster_user" -c \
+			"$PSQL_CTL status $OCF_RESKEY_postmaster_options" &> /dev/null; echo $?` = '0' ]
+	do
+		sleep 1
+		let count=$count+1
+	done
 
-	if [ $? -ne 0 ]; then
+	if [ "$count" -gt "$OCF_RESKEY_startup_wait" ]; then
 		clog_service_start $CLOG_FAILED
 		return $OCF_ERR_GENERIC
 	fi
diff --git a/rgmanager/src/resources/utils/config-utils.sh b/rgmanager/src/resources/utils/config-utils.sh
index 68ac859..95265d5 100644
--- a/rgmanager/src/resources/utils/config-utils.sh
+++ b/rgmanager/src/resources/utils/config-utils.sh
@@ -220,22 +220,12 @@ generate_name_for_conf_dir()
 	return 0;
 }
 
-#
-# Usage: create_pid_directory [username]
-#
-create_pid_directory()
+set_pid_directory_permissions()
 {
-	declare program_name="$(basename $0 | sed 's/^\(.*\)\..*/\1/')"
-	declare dirname="$RA_COMMON_pid_dir/$program_name"
-	declare username="$1"
+	declare program_name="$1"
+	declare dirname="$2"
+	declare username="$3"
 
-	if [ -d "$dirname" ]; then
-		return 0;
-	fi
-	
-	chmod 711 "$RA_COMMON_pid_dir"
-	mkdir -p "$dirname"
-	
 	if [ "$program_name" = "mysql" ]; then
 		if [ -n "$username" ]; then
 			chown "${username}.root" "$dirname"
@@ -248,8 +238,32 @@ create_pid_directory()
 		else
 			chown tomcat.root "$dirname"
 		fi
+	elif [ "$program_name" = "named" ]; then
+		if [ -n "$username" ]; then
+			chown "${username}.root" "$dirname"
+		fi
 	fi
+}
+
+#
+# Usage: create_pid_directory [username]
+#
+create_pid_directory()
+{
+	declare program_name="$(basename $0 | sed 's/^\(.*\)\..*/\1/')"
+	declare dirname="$RA_COMMON_pid_dir/$program_name"
+	declare username="$1"
+
+	if [ -d "$dirname" ]; then
+		# make sure the permissions are correct even if directory exists
+		set_pid_directory_permissions "$program_name" "$dirname" "$username"
+		return 0;
+	fi
+	
+	chmod 711 "$RA_COMMON_pid_dir"
+	mkdir -p "$dirname"
 
+	set_pid_directory_permissions "$program_name" "$dirname" "$username"
 	return 0;
 }
 
diff --git a/rgmanager/src/resources/utils/fs-lib.sh b/rgmanager/src/resources/utils/fs-lib.sh
index 2697ed2..6288c09 100644
--- a/rgmanager/src/resources/utils/fs-lib.sh
+++ b/rgmanager/src/resources/utils/fs-lib.sh
@@ -27,6 +27,10 @@ LANG=C
 PATH=/bin:/sbin:/usr/bin:/usr/sbin
 export LC_ALL LANG PATH
 
+# Define this value to 0 by default, bind-mount.sh or any other agent
+# that uses this value will alter it after sourcing fs-lib.sh
+export IS_BIND_MOUNT=0
+
 # Private return codes
 FAIL=2
 NO=1
@@ -130,6 +134,10 @@ real_device()
 	declare dev="$1"
 	declare realdev
 
+	if [ $IS_BIND_MOUNT -eq 1 ]; then
+		REAL_DEVICE="$dev"
+		return $OCF_SUCCESS
+	fi
 	REAL_DEVICE=""
 
 	[ -z "$dev" ] && return $OCF_ERR_ARGS
@@ -192,6 +200,15 @@ verify_device()
 	return $OCF_ERR_ARGS
 }
 
+list_mounts()
+{
+	if [ $IS_BIND_MOUNT -eq 1 ]; then
+		cat /etc/mtab
+	else
+		cat /proc/mounts
+	fi
+}
+
 ##
 # Tries to use findmnt util to return list
 # of mountpoints for a device
@@ -217,11 +234,11 @@ try_findmnt()
 
 	which findmnt > /dev/null 2>&1
 	if [ $? -eq 0 ]; then
-		FINDMNT_OUTPUT=$(findmnt -o TARGET --noheadings $1)
+		FINDMNT_OUTPUT="$(findmnt -o TARGET --noheadings $1)"
 		if [ $? -ne 0 ]; then
 			# workaround mount helpers inconsistency that still
 			# add / on the device entry in /proc/mounts
-			FINDMNT_OUTPUT=$(findmnt -o TARGET --noheadings $1/)
+			FINDMNT_OUTPUT="$(findmnt -o TARGET --noheadings $1/)"
 			if [ $? -ne 0 ]; then
 				return 1
 			else
@@ -260,6 +277,7 @@ strip_trailing_slashes()
 kill_procs_using_mount () {
 	declare mp
 	declare procs
+	declare mmap_procs
 
 	if [ $# -lt 1 -o -z "$1" ]; then
 		ocf_log err "Usage: kill_procs_using_mount mount_point [signal]"
@@ -277,6 +295,10 @@ kill_procs_using_mount () {
 	# anything held open in mount point after the slash
 	procs=$(find /proc/[0-9]*/ -type l -lname "${mp}/*" -or -lname "${mp}" 2>/dev/null | awk -F/ '{print $3}' | uniq)
 
+	# anything with memory mapping to something in the mountpoint
+	mmap_procs=$(grep " ${mp}" /proc/[0-9]*/maps | awk -F/ '{print $3}' | uniq)
+	procs=$(echo -e "${procs}\n${mmap_procs}" | sort | uniq)
+
 	for pid in $procs; do
 		if [ -n "$2" ]; then
 			kill -s $2 $pid
@@ -325,7 +347,7 @@ mount_in_use () {
 				if [ "$tmp_mp" = "$mp" ]; then
 					return $YES
 				fi
-			done < <(echo $FINDMNT_OUTPUT)
+			done < <(echo "$FINDMNT_OUTPUT")
 			;;
 		*)
 			return $YES
@@ -368,7 +390,7 @@ mount_in_use () {
 		if [ -n "$tmp_mp" -a "$tmp_mp" = "$mp" ]; then
 			return $YES
 		fi
-	done < <(cat /proc/mounts)
+	done < <(list_mounts)
 
 	return $NO
 }
@@ -403,7 +425,7 @@ real_mountpoint()
 				found=0
 				break
 			fi
-		done < <(echo $FINDMNT_OUTPUT)
+		done < <(echo "$FINDMNT_OUTPUT")
 		;;
 	1)
 		# findmnt found no mount points for the device
@@ -450,7 +472,7 @@ real_mountpoint()
 					fi
 				fi
 			fi
-		done < <(cat /proc/mounts)
+		done < <(list_mounts)
 	esac
 
 	if [ $found -ne 0 ]; then
@@ -561,7 +583,7 @@ is_alive()
 	done
 	if [ $rw -eq $YES ]; then
 		file=$(mktemp "$mount_point/.check_writable.$(hostname).XXXXXX")
-		if [ ! -e $file ]; then
+		if [ ! -e "$file" ]; then
 			ocf_log err "${OCF_RESOURCE_INSTANCE}: is_alive: failed write test on [$mount_point]. Return code: $errcode"
 			return $NO
 		fi
diff --git a/rgmanager/src/resources/vm.sh b/rgmanager/src/resources/vm.sh
index 7c8ba9a..d1494d5 100755
--- a/rgmanager/src/resources/vm.sh
+++ b/rgmanager/src/resources/vm.sh
@@ -20,6 +20,13 @@
 
 PATH=/bin:/sbin:/usr/bin:/usr/sbin
 
+# Only allow pid status checks during monitor operations.
+# Otherwise we want proceed with failing if virsh is not available.
+ALLOW_PID_STATUS_CHECK=0
+if [ "$1" = "monitor" ] || [ "$1" = "status" ]; then
+	ALLOW_PID_STATUS_CHECK=1
+fi
+
 export PATH
 
 . $(dirname $0)/ocf-shellfuncs || exit 1
@@ -28,6 +35,8 @@ export PATH
 # Virtual Machine start/stop script (requires the virsh command)
 #
 
+EMULATOR_STATE="/var/run/vm-${OCF_RESKEY_name}-emu.state"
+
 # Indeterminate state: xend/libvirtd is down.
 export OCF_APP_ERR_INDETERMINATE=150
 
@@ -287,6 +296,17 @@ meta_data()
 	    <content type="string" default="auto" />
 	</parameter>
 
+	<parameter name="no_kill">
+            <longdesc lang="en">
+		Do not force kill vm during stop, instead
+		fail after the timeout expires.
+            </longdesc>
+            <shortdesc lang="en">
+		Don't force kill vm on stop.
+            </shortdesc >
+	    <content type="boolean" default="false" />
+	</parameter>
+
     </parameters>
 
     <actions>
@@ -404,6 +424,34 @@ get_timeout()
 	return 0
 }
 
+get_emulator()
+{
+	local emulator=""
+
+	emulator=$(virsh $VIRSH_OPTIONS dumpxml $OCF_RESKEY_name 2>/dev/null | sed -n -e 's/^.*<emulator>\(.*\)<\/emulator>.*$/\1/p')
+	if [ -z "$emulator" ] && [ -a "$EMULATOR_STATE" ]; then
+		emulator=$(cat $EMULATOR_STATE)
+	fi
+	if [ -z "$emulator" ]; then
+		emulator=$(cat ${OCF_RESKEY_config} | sed -n -e 's/^.*<emulator>\(.*\)<\/emulator>.*$/\1/p')
+	fi
+
+	if [ -n "$emulator" ]; then
+		basename $emulator
+	else 
+		ocf_log error "Unable to determine emulator for $OCF_RESKEY_name" 
+	fi
+}
+
+update_emulator_cache()
+{
+	local emulator
+
+	emulator=$(get_emulator)
+	if [ -n "$emulator" ]; then
+		echo $emulator > $EMULATOR_STATE
+	fi
+}
 
 #
 # Start a virtual machine given the parameters from
@@ -413,6 +461,7 @@ do_virsh_start()
 {
 	declare cmdline
 	declare snapshotimage
+	declare rc
 
 	echo -n "Virtual machine $OCF_RESKEY_name is "
 	do_status && return 0
@@ -438,9 +487,12 @@ do_virsh_start()
 	ocf_log debug "$cmdline"
 
 	$cmdline
-	return $?
-}
+	rc=$?
+
+	update_emulator_cache
 
+	return $rc
+}
 
 do_xm_stop()
 {
@@ -510,6 +562,7 @@ do_virsh_stop()
 		done
 	done
 
+	ocf_log err "Stop operation timed out for vm '$OCF_RESKEY_name'"
 	return 1
 }
 
@@ -580,6 +633,41 @@ xm_status()
 	return $OCF_NOT_RUNNING
 }
 
+# attempt to check domain status outside of libvirt using the emulator process
+vm_pid_status()
+{
+	local rc=$OCF_ERR_GENERIC
+	local emulator
+
+	if [ $ALLOW_PID_STATUS_CHECK -eq 0 ]; then
+		echo "indeterminate"
+		return $OCF_APP_ERR_INDETERMINATE
+	fi
+
+	emulator=$(get_emulator)
+	case "$emulator" in
+		qemu-kvm|qemu-system-*)
+			rc=$OCF_NOT_RUNNING
+			ps awx | grep -E "[q]emu-(kvm|system).*-name $OCF_RESKEY_name " > /dev/null 2>&1
+			if [ $? -eq 0 ]; then
+				rc=$OCF_SUCCESS
+			fi
+			;;
+		# This can be expanded to check for additional emulators
+		*)
+			echo "indeterminate"
+			return $OCF_APP_ERR_INDETERMINATE
+			;;
+	esac
+
+	if [ $rc -eq $OCF_SUCCESS ]; then
+		echo "running"
+	elif [ $rc -eq $OCF_NOT_RUNNING ]; then
+		echo "shut off"
+	fi
+
+	return $rc
+}
 
 virsh_status()
 {
@@ -599,9 +687,10 @@ virsh_status()
 	# libvirtd is required for migration.
 	#
 	pid=$(pidof libvirtd)
-	if [ -z "$pid" ]; then 
-		echo indeterminate
-		return $OCF_APP_ERR_INDETERMINATE
+	if [ -z "$pid" ]; then
+		# attempt to determine if vm is running from pid file
+		vm_pid_status
+		return $?
 	fi
 
 	state=$(virsh domstate $OCF_RESKEY_name)
@@ -610,6 +699,7 @@ virsh_status()
 
 	if [ "$state" = "running" ] || [ "$state" = "paused" ] || [ "$state" = "no state" ] || 
 	   [ "$state" = "idle" ]; then
+		update_emulator_cache
 		return 0
 	fi
 
@@ -729,7 +819,20 @@ choose_management_tool()
 	return 0
 }
 
-
+get_hypervisor()
+{
+	local hypervisor="`virsh version | grep \"Running hypervisor:\" | awk '{print $3}' | tr A-Z a-z`"
+	# if virsh gives us nothing (likely because libvirt is down), we can attempt
+	# to determine auto detect the hypervisor is qemu if a qemu emulator is used
+	# for this domain.
+	if [ -z "$hypervisor" ]; then
+		get_emulator | grep "qemu" > /dev/null 2>&1
+		if [ $? -eq 0 ]; then
+			hypervisor="qemu"
+		fi
+	fi
+	echo $hypervisor
+}
 
 validate_all()
 {
@@ -744,7 +847,7 @@ validate_all()
 	#
 	if [ -z "$OCF_RESKEY_hypervisor" ] ||
 	   [ "$OCF_RESKEY_hypervisor" = "auto" ]; then
-		export OCF_RESKEY_hypervisor="`virsh version | grep \"Running hypervisor:\" | awk '{print $3}' | tr A-Z a-z`"
+		export OCF_RESKEY_hypervisor="`get_hypervisor`"
 		if [ -z "$OCF_RESKEY_hypervisor" ]; then
 			ocf_log err "Could not determine Hypervisor"
 			return $OCF_ERR_ARGS
@@ -858,6 +961,11 @@ validate_all()
 		export migrateuriopt="tcp:%s"
 	fi
 
+	case "$OCF_RESKEY_no_kill" in
+		yes|true|1|YES|TRUE|on|ON)
+			OCF_RESKEY_no_kill=1
+		;;
+	esac
 	#virsh list --all | awk '{print $2}' | grep -q "^$OCF_RESKEY_name\$"
 	return $?
 }
@@ -1036,7 +1144,11 @@ case $1 in
 		;;
 	stop)
 		validate_all || exit $OCF_ERR_ARGS
-		do_stop shutdown destroy
+		if [ $OCF_RESKEY_no_kill -eq 1 ]; then
+			do_stop shutdown
+		else
+			do_stop shutdown destroy
+		fi
 		exit $?
 		;;
 	kill)
diff --git a/tools/ocft/Filesystem b/tools/ocft/Filesystem
index 75203d7..0b4d781 100644
--- a/tools/ocft/Filesystem
+++ b/tools/ocft/Filesystem
@@ -1,6 +1,9 @@
 # Filesystem
 # by dejan at suse.de on 
 # Tue Feb 15 18:50:04 CET 2011
+#
+# NB: Replace /var/run below with /run if your system mounts /run
+# as tmpfs!
 
 CONFIG
 	Agent Filesystem
diff --git a/tools/ocft/Makefile.am b/tools/ocft/Makefile.am
index 2374531..6218bc3 100644
--- a/tools/ocft/Makefile.am
+++ b/tools/ocft/Makefile.am
@@ -40,13 +40,17 @@ ocftcfgs_DATA      =  apache  	\
 		       	 oracle		\
 		       	 drbd.linbit		\
 		       	 exportfs	\
+		       	 exportfs-multidir	\
 		       	 nfsserver	\
 		       	 portblock	\
 		       	 iscsi	\
 		       	 named	\
 		       	 postfix	\
+			 sg_persist \
 			 tomcat	\
 		       	 Xinetd	\
+		       	 Xen	\
+		       	 VirtualDomain	\
 			 SendArp
 
 ocftdir			= $(datadir)/$(PACKAGE_NAME)/ocft
diff --git a/tools/ocft/VirtualDomain b/tools/ocft/VirtualDomain
new file mode 100644
index 0000000..10302ce
--- /dev/null
+++ b/tools/ocft/VirtualDomain
@@ -0,0 +1,70 @@
+# VirtualDomain
+# by dejan at suse.de on 
+# Tue Jul  8 12:48:03 CEST 2014
+
+CONFIG
+	Agent VirtualDomain
+	AgentRoot /usr/lib/ocf/resource.d/heartbeat
+	HangTimeout 20
+
+# set OCFT_config to the libvirt configuration file
+# the guest is going to be stopped and started
+VARIABLE
+	OCFT_config=/etc/libvirt/qemu/sle11-sp3.xml
+
+CASE-BLOCK required_args
+	Env OCF_RESKEY_config=$OCFT_config
+
+CASE-BLOCK unset_utilization
+	Env OCF_RESKEY_autoset_utilization_hv_memory=false
+	Env OCF_RESKEY_autoset_utilization_cpu=false
+
+CASE-BLOCK default_status
+	AgentRun stop
+
+CASE-BLOCK prepare
+	Include required_args
+	Include unset_utilization
+	Include default_status
+
+CASE "check base env"
+	Include prepare
+	AgentRun start OCF_SUCCESS
+
+CASE "check base env: invalid 'OCF_RESKEY_config'"
+	Include prepare
+	Env OCF_RESKEY_config=/no_such_file
+	AgentRun start OCF_ERR_INSTALLED
+
+CASE "check base env: unset 'OCF_RESKEY_config'"
+	Include prepare
+	Unenv OCF_RESKEY_config
+	AgentRun start OCF_ERR_CONFIGURED
+
+CASE "normal stop"
+	Include prepare
+	AgentRun start
+	AgentRun stop OCF_SUCCESS
+
+CASE "double start"
+	Include prepare
+	AgentRun start
+	AgentRun start OCF_SUCCESS
+
+CASE "double stop"
+	Include prepare
+	AgentRun stop OCF_SUCCESS
+
+CASE "monitor when running"
+	Include prepare
+	AgentRun start
+	AgentRun monitor OCF_SUCCESS
+
+CASE "monitor when not running"
+	Include prepare
+	AgentRun monitor OCF_NOT_RUNNING
+
+CASE "unimplemented command"
+	Include prepare
+	AgentRun no_cmd OCF_ERR_UNIMPLEMENTED
+
diff --git a/tools/ocft/apache b/tools/ocft/Xen
similarity index 55%
copy from tools/ocft/apache
copy to tools/ocft/Xen
index e939044..731564b 100644
--- a/tools/ocft/apache
+++ b/tools/ocft/Xen
@@ -1,39 +1,40 @@
-# apache
-# make sure that your apache configuration loads mod_status
+# Xen
+# by dejan at suse.de on 
+# Tue Jul  8 12:20:23 CEST 2014
 
 CONFIG
-	Agent apache
+	Agent Xen
 	AgentRoot /usr/lib/ocf/resource.d/heartbeat
-	InstallPackage apache2
-        HangTimeout 20
+	HangTimeout 20
 
-SETUP-AGENT
-	/etc/init.d/apache2 start
-	/etc/init.d/apache2 stop
+# set OCFT_xmfile to the xen-xm format file
+# the guest is going to be stopped and started
+VARIABLE
+	OCFT_xmfile=/etc/xen/vm/xen-f
+
+CASE-BLOCK required_args
+	Env OCF_RESKEY_xmfile=$OCFT_xmfile
 
 CASE-BLOCK default_status
 	AgentRun stop
 
 CASE-BLOCK prepare
+	Include required_args
 	Include default_status
 
 CASE "check base env"
 	Include prepare
 	AgentRun start OCF_SUCCESS
 
-CASE "check base env: set non-existing OCF_RESKEY_statusurl"
-	Include prepare
-	Env OCF_RESKEY_statusurl="yoyoyoyo"
-	AgentRun start OCF_ERR_GENERIC
-
-CASE "check base env: set non-existing OCF_RESKEY_configfile"
+CASE "check base env: invalid 'OCF_RESKEY_xmfile'"
 	Include prepare
-	Env OCF_RESKEY_configfile="/yoyoyoyo/nosuchfile"
+	Env OCF_RESKEY_xmfile=/no_such_file
 	AgentRun start OCF_ERR_INSTALLED
 
-CASE "normal start"
+CASE "check base env: unset 'OCF_RESKEY_xmfile'"
 	Include prepare
-	AgentRun start OCF_SUCCESS
+	Unenv OCF_RESKEY_xmfile
+	AgentRun start OCF_ERR_INSTALLED
 
 CASE "normal stop"
 	Include prepare
@@ -49,15 +50,16 @@ CASE "double stop"
 	Include prepare
 	AgentRun stop OCF_SUCCESS
 
-CASE "running monitor"
+CASE "monitor when running"
 	Include prepare
 	AgentRun start
 	AgentRun monitor OCF_SUCCESS
 
-CASE "not running monitor"
+CASE "monitor when not running"
 	Include prepare
 	AgentRun monitor OCF_NOT_RUNNING
 
 CASE "unimplemented command"
 	Include prepare
 	AgentRun no_cmd OCF_ERR_UNIMPLEMENTED
+
diff --git a/tools/ocft/apache b/tools/ocft/apache
index e939044..b2219ec 100644
--- a/tools/ocft/apache
+++ b/tools/ocft/apache
@@ -8,8 +8,8 @@ CONFIG
         HangTimeout 20
 
 SETUP-AGENT
-	/etc/init.d/apache2 start
-	/etc/init.d/apache2 stop
+	rcapache2 start
+	rcapache2 stop
 
 CASE-BLOCK default_status
 	AgentRun stop
diff --git a/tools/ocft/caselib.in b/tools/ocft/caselib.in
index 8c8a45a..01b108f 100644
--- a/tools/ocft/caselib.in
+++ b/tools/ocft/caselib.in
@@ -89,6 +89,14 @@ agent_install()
   done
 }
 
+set_ocf_env()
+{
+  export OCF_RA_VERSION_MAJOR=1
+  export OCF_RA_VERSION_MINOR=0
+  export OCF_RESOURCE_TYPE=$1
+  export OCF_RESOURCE_INSTANCE=${OCF_RESOURCE_INSTANCE:-"ocft"}
+}
+
 agent_run()
 {
   local agent cmd timeout pid i ret aroot
@@ -96,6 +104,8 @@ agent_run()
   cmd="$2"
   timeout="$3"
 
+  set_ocf_env $agent
+
   aroot=${__OCFT__MYROOT:-$__OCFT__AGENT_ROOT}
 
   setsid $aroot/$agent $cmd >/tmp/.ocft_runlog 2>&1 &
@@ -292,3 +302,5 @@ while read __OCFT__line; do
   fi
 done <<<"$(sed 's/#.*//' $OCF_LIB/ocf-returncodes)"
 
+
+# vim:ts=2:sw=2:et:
diff --git a/tools/ocft/mysql b/tools/ocft/exportfs-multidir
similarity index 53%
copy from tools/ocft/mysql
copy to tools/ocft/exportfs-multidir
index 27fcb58..72551c6 100644
--- a/tools/ocft/mysql
+++ b/tools/ocft/exportfs-multidir
@@ -1,34 +1,51 @@
-# mysql
+# exportfs
+#
+#
 
 CONFIG
-	Agent mysql
+	Agent exportfs
 	AgentRoot /usr/lib/ocf/resource.d/heartbeat
-	InstallPackage mysql
-        HangTimeout 20
+        HangTimeout 40
 
 SETUP-AGENT
-	/etc/init.d/mysql start
-	/etc/init.d/mysql stop
+        # nothing
 
-CASE-BLOCK crm_setting
-	Env OCF_RESKEY_CRM_meta_timeout=15000
+CASE-BLOCK set_testenv
+        Env OCF_RESKEY_directory="/usr /var"
+        Env OCF_RESKEY_fsid=105
+        Env OCF_RESKEY_clientspec="*"
+	Env OCF_RESKEY_CRM_meta_timeout=30000
 
 CASE-BLOCK default_status
 	AgentRun stop
 
 CASE-BLOCK prepare
-	Include crm_setting
+        Include set_testenv
 	Include default_status
 
 CASE "check base env"
 	Include prepare
 	AgentRun start OCF_SUCCESS
 
-CASE "check base env: invalid 'OCF_RESKEY_binary'"
+CASE "check base env: no 'OCF_RESKEY_fsid'"
 	Include prepare
-	Env OCF_RESKEY_binary=no_such
+	Env OCF_RESKEY_fsid=
+	AgentRun start OCF_ERR_CONFIGURED
+
+CASE "check base env: invalid 'OCF_RESKEY_directory'"
+	Include prepare
+	Env OCF_RESKEY_directory=/no_such
 	AgentRun start OCF_ERR_INSTALLED
 
+CASE "check base env: invalid 'OCF_RESKEY_fsid'"
+	Include prepare
+	Env OCF_RESKEY_fsid=root
+	AgentRun start OCF_ERR_CONFIGURED
+
+CASE "unimplemented command"
+	Include prepare
+	AgentRun no_cmd OCF_ERR_UNIMPLEMENTED
+
 CASE "normal start"
 	Include prepare
 	AgentRun start OCF_SUCCESS
@@ -47,31 +64,16 @@ CASE "double stop"
 	Include prepare
 	AgentRun stop OCF_SUCCESS
 
-CASE "running monitor"
+CASE "stop with no env"
+	Include prepare
+	Env OCF_RESKEY_directory="/usr /no_such"
+	AgentRun stop OCF_SUCCESS
+
+CASE "started: monitor"
 	Include prepare
 	AgentRun start
 	AgentRun monitor OCF_SUCCESS
 
-CASE "not running monitor"
+CASE "not started: monitor"
 	Include prepare
 	AgentRun monitor OCF_NOT_RUNNING
-
-CASE "check lib file"
-	Include prepare
-	Bash chmod u-w /var/lib/mysql
-	BashAtExit chmod u+w /var/lib/mysql
-	AgentRun start OCF_ERR_PERM
-
-CASE "unimplemented command"
-	Include prepare
-	AgentRun no_cmd OCF_ERR_UNIMPLEMENTED
-
-CASE "non-existent user"
-	Include prepare
-	Env OCF_RESKEY_user=no_user
-	AgentRun start OCF_ERR_INSTALLED
-
-CASE "invalid user"
-	Include prepare
-	Env OCF_RESKEY_user=nobody
-	AgentRun start OCF_ERR_PERM
diff --git a/tools/ocft/iscsi b/tools/ocft/iscsi
index c1325a1..7b0452a 100644
--- a/tools/ocft/iscsi
+++ b/tools/ocft/iscsi
@@ -16,9 +16,9 @@ SETUP-AGENT
 	dd if=/dev/zero of=$OCFT_disk bs=1024k count=1 2>/dev/null
 	echo Target $OCFT_target >> /etc/ietd.conf
 	echo "        Lun 0 Path=$OCFT_disk,Type=fileio" >> /etc/ietd.conf
-	/etc/init.d/iscsitarget start
-	/etc/init.d/open-iscsi start
-	/etc/init.d/iscsitarget restart
+	rciscsitarget start
+	rcopen-iscsi start
+	rciscsitarget restart
 
 CLEANUP-AGENT
 	rm -f $OCFT_disk
diff --git a/tools/ocft/mysql b/tools/ocft/mysql
index 27fcb58..bae0dc7 100644
--- a/tools/ocft/mysql
+++ b/tools/ocft/mysql
@@ -7,8 +7,8 @@ CONFIG
         HangTimeout 20
 
 SETUP-AGENT
-	/etc/init.d/mysql start
-	/etc/init.d/mysql stop
+	rcmysql start
+	rcmysql stop
 
 CASE-BLOCK crm_setting
 	Env OCF_RESKEY_CRM_meta_timeout=15000
diff --git a/tools/ocft/named b/tools/ocft/named
index 90a4351..15024a8 100644
--- a/tools/ocft/named
+++ b/tools/ocft/named
@@ -10,8 +10,8 @@ CONFIG
 	InstallPackage bind-utils
 
 SETUP-AGENT
-	/etc/init.d/named start
-	/etc/init.d/named stop
+	rcnamed start
+	rcnamed stop
 
 CASE-BLOCK crm_setting
         Env OCF_RESKEY_CRM_meta_timeout=15000
diff --git a/tools/ocft/ocft.in b/tools/ocft/ocft.in
index 957a8fc..bfa9dcb 100644
--- a/tools/ocft/ocft.in
+++ b/tools/ocft/ocft.in
@@ -263,12 +263,12 @@ parse_cfg()
           stat=2
           continue
           ;;
-	SETUP-AGENT)
+        SETUP-AGENT)
           case_parse_finish
           stat=3
           continue
           ;;
-	CLEANUP-AGENT)
+        CLEANUP-AGENT)
           case_parse_finish
           stat=4
           continue
@@ -635,6 +635,8 @@ EOF
 start_test()
 {
   local sh shs testsh agents line ret
+  local rc=0
+  local varlib
 
   if ! cd $CASES_DIR >/dev/null 2>&1; then
     die "cases directory not found."
@@ -663,8 +665,17 @@ start_test()
               cleanup_${shs}.sh"
     fi
 
+    if [ -n "$opt_trace_ra" ]; then
+      varlib=${HA_VARLIB:="/var/lib/heartbeat"}
+      export OCF_RESKEY_trace_ra=1
+      echo "RA trace on, output in $varlib/trace_ra"
+    fi
+
     for sh in $testsh; do
       if [ -r "$sh" ]; then
+        if [ -n "$opt_trace_ra" ]; then
+          export OCF_RESOURCE_INSTANCE="`echo $sh | sed 's/_.*//'`"
+        fi
         ./$sh
         ret=$?
 
@@ -674,6 +685,7 @@ start_test()
                 warn "SETUP failed, break all tests of '$shs'."
                 break
               fi
+              rc=$((rc|ret))
               ;;
           cleanup*)
               if [ $ret -ne 0 ]; then
@@ -687,6 +699,7 @@ start_test()
                 1) touch ${sh%.*}.retest ;;
                 0) rm -f ${sh%.*}.retest ;;
               esac
+              rc=$((rc|ret))
               ;;
         esac
       fi
@@ -696,6 +709,7 @@ start_test()
       sed -r 's/\^\[\[[0-9]+m|\^I|.$//g' >>logs/$shs.log
     done
   done
+  return $rc
 }
 
 agent_clean()
@@ -731,13 +745,14 @@ $0 ACTION [OPTION] [agent1 [agent2] [...]]
 ACTIONs include:
      make [-d dir]   Generate the testing shell scripts.
                        -d  The directory that contains 
-		           configuration of cases.
-     test [-v|-i]    Execute the testing shell scripts.
+           configuration of cases.
+     test [-v|-i|-X]  Execute the testing shell scripts.
                        -v  Verbose output mode.
                        -i  Incremental mode, skip case 
                            which succeeded. If cleaning 
                            the status of incremental mode
                            is needed, try to '$0 clean RA_NAME'.
+                       -X  Trace the RA
      clean           Delete the testing shell scripts.
      help [-v]       Show this help and exit.
                        -v  Show HOWTO and exit.
@@ -832,16 +847,23 @@ case "$command" in
     parse_cfg "$@"
     ;;
   test)
-    for v in 1 2; do
+    for v in 1 2 3; do
       case "$1" in
         -v)
           opt_verbose=1
           shift
           ;;
+        -X)
+          opt_trace_ra=1
+          shift
+          ;;
         -i)
           opt_incremental=1
           shift
           ;;
+        -*)
+          die "bad option $1"
+          ;;
       esac
     done
     start_test "$@"
@@ -863,3 +885,5 @@ case "$command" in
     exit 1
     ;;
 esac
+
+# vim:ts=2:sw=2:et:
diff --git a/tools/ocft/pgsql b/tools/ocft/pgsql
index 9944b09..abab33f 100644
--- a/tools/ocft/pgsql
+++ b/tools/ocft/pgsql
@@ -7,8 +7,8 @@ CONFIG
         HangTimeout 20
 
 SETUP-AGENT
-	/etc/init.d/postgresql start
-	/etc/init.d/postgresql stop
+	rcpostgresql start
+	rcpostgresql stop
 
 CASE-BLOCK crm_setting
         Env OCF_RESKEY_CRM_meta_timeout=15000
diff --git a/tools/ocft/sg_persist b/tools/ocft/sg_persist
new file mode 100644
index 0000000..f11d36d
--- /dev/null
+++ b/tools/ocft/sg_persist
@@ -0,0 +1,225 @@
+# sg_persist
+# Before use sg_persist ocft test case you have to prepare to make pacemaker 
+# can be started successfully because sg_persist RA needs to use the result of
+# `crm_node -i` as sg_persist register key.
+
+CONFIG
+	Agent sg_persist
+	AgentRoot /usr/lib/ocf/resource.d/heartbeat
+	InstallPackage sg3_utils
+	HangTimeout 20
+
+VARIABLE
+	
+	# Please set the disk you want to test
+	#OCFT_DEVS="/dev/disk/by-id/scsi-14945540000000000844965720e6e555176b19461345d68d5"
+	OCFT_DEVS=
+
+	# please set the IP addrees of the other server.
+	IP_2=
+	
+SETUP-AGENT
+	# start pacemaker to get node_id through crm_node -i
+	systemctl start pacemaker
+
+CLEANUP-AGENT
+
+CASE-BLOCK required_args
+	Env HA_SBIN_DIR=""
+	Env OCF_RESKEY_devs="$OCFT_DEVS"
+	Env OCF_RESKEY_reservation_type=1
+	Env OCF_RESKEY_CRM_meta_notify=true
+	Env OCF_RESKEY_CRM_meta_master_max=1
+
+CASE-BLOCK required_args_error
+	Env HA_SBIN_DIR=""
+	Env OCF_RESKEY_devs="$OCFT_DEVS"
+	Env OCF_RESKEY_reservation_type=1
+	Env OCF_RESKEY_CRM_meta_notify=true
+	Env OCF_RESKEY_CRM_meta_master_max=2
+
+CASE-BLOCK required_args_type_5
+	Env HA_SBIN_DIR=""
+	Env OCF_RESKEY_devs="$OCFT_DEVS"
+	Env OCF_RESKEY_reservation_type=5
+	Env OCF_RESKEY_CRM_meta_notify=true
+	Env OCF_RESKEY_CRM_meta_master_max=1
+
+CASE-BLOCK required_args_type_7
+	Env HA_SBIN_DIR=""
+	Env OCF_RESKEY_devs="$OCFT_DEVS"
+	Env OCF_RESKEY_reservation_type=7
+	Env OCF_RESKEY_CRM_meta_notify=true
+	Env OCF_RESKEY_CRM_meta_master_max=2
+
+CASE-BLOCK default_status
+	AgentRun stop
+
+CASE-BLOCK prepare
+	Include required_args
+	Include default_status
+
+CASE-BLOCK prepare_type_5
+	Include required_args_type_5
+	Include default_status
+
+CASE-BLOCK prepare_type_7
+	Include required_args_type_7
+	Include default_status
+
+CASE "check base env"
+	Include prepare
+	AgentRun start OCF_SUCCESS
+
+CASE "check base env: unset 'OCF_RESKEY_sg_persist_devs'"
+	Include prepare
+	Unenv OCF_RESKEY_devs
+	AgentRun start OCF_ERR_INSTALLED
+
+CASE "validate all"
+	Include required_args
+	AgentRun validate-all OCF_SUCCESS
+
+CASE "validate all--type:5"
+	Include required_args_type_5
+	AgentRun validate-all OCF_SUCCESS
+
+CASE "validate all--type:7"
+	Include required_args_type_7
+	AgentRun validate-all OCF_SUCCESS
+
+CASE "validate all--error configured"
+	Include required_args_error
+	AgentRun validate-all OCF_ERR_CONFIGURED
+
+CASE "normal start"
+	Include prepare
+	AgentRun start OCF_SUCCESS
+
+CASE "normal stop"
+	Include prepare
+	AgentRun start
+	AgentRun stop OCF_SUCCESS
+
+CASE "double start"
+	Include prepare
+	AgentRun start
+	AgentRun start OCF_SUCCESS
+
+CASE "double stop"
+	Include prepare
+	AgentRun stop
+	AgentRun stop OCF_SUCCESS
+
+CASE "monitor when running"
+	Include prepare
+	AgentRun start
+	AgentRun monitor OCF_SUCCESS
+
+CASE "monitor when not running"
+	Include prepare
+	AgentRun stop
+	AgentRun monitor OCF_NOT_RUNNING
+
+CASE "Primary/Secondary monitor"
+	Include prepare
+	AgentRun start
+	AgentRun promote
+	AgentRun monitor OCF_RUNNING_MASTER
+	AgentRun demote
+	AgentRun monitor OCF_SUCCESS
+	AgentRun stop
+
+CASE "promote/demote test in single-primary mode"
+	Include prepare
+	Include@$IP_2 prepare
+
+    # start 
+	AgentRun start
+	AgentRun@$IP_2 start
+
+	# promote local first
+	AgentRun promote OCF_SUCCESS
+
+    # demote local prepare for remote promote
+	AgentRun demote
+    
+    # remote promote
+	AgentRun@$IP_2 promote OCF_SUCCESS
+	AgentRun@$IP_2 monitor OCF_RUNNING_MASTER
+	AgentRun monitor OCF_SUCCESS
+	
+     # promote local then promote(by preempt) remote, which will cause local OCF_NOT_RUNNING
+	AgentRun promote OCF_SUCCESS
+	AgentRun monitor OCF_RUNNING_MASTER
+	AgentRun@$IP_2 monitor OCF_NOT_RUNNING
+
+    # remove all reservation registration
+	AgentRun demote
+	AgentRun stop 
+	AgentRun@$IP_2 stop
+
+
+CASE "normal start--type 5"
+	Include prepare_type_5
+	Include@$IP_2 prepare_type_5
+	AgentRun start OCF_SUCCESS
+	AgentRun@$IP_2 start OCF_SUCCESS
+
+CASE "promote/demote test in single-primary mode"
+	Include prepare_type_5
+	Include@$IP_2 prepare_type_5
+
+    # start 
+	AgentRun start
+	AgentRun@$IP_2 start
+
+    # promote local first
+	AgentRun promote OCF_SUCCESS
+	AgentRun monitor OCF_RUNNING_MASTER
+
+    # demote local prepare for remote promote
+	AgentRun demote
+	# remote promote
+	AgentRun@$IP_2 promote OCF_SUCCESS
+	AgentRun@$IP_2 monitor OCF_RUNNING_MASTER
+	AgentRun monitor OCF_SUCCESS
+	
+     # promote local then promote(by preempt) remote, which will cause local OCF_NOT_RUNNING
+	AgentRun promote OCF_SUCCESS
+	AgentRun monitor OCF_RUNNING_MASTER
+	AgentRun@$IP_2 monitor OCF_NOT_RUNNING
+
+    # remove all reservation registration
+	AgentRun demote
+	AgentRun stop 
+	AgentRun@$IP_2 stop
+
+CASE "normal start--type 7: 2 masters"
+	Include prepare_type_7
+	Include@$IP_2 prepare_type_7
+	AgentRun start OCF_SUCCESS
+	AgentRun@$IP_2 start OCF_SUCCESS
+
+
+CASE "Primary/Primary monitor--type 7"
+	Include prepare_type_7
+	Include@$IP_2 prepare_type_7
+	
+	AgentRun start OCF_SUCCESS
+	AgentRun@$IP_2 start OCF_SUCCESS
+
+	AgentRun promote OCF_SUCCESS
+	AgentRun monitor OCF_RUNNING_MASTER
+
+	AgentRun@$IP_2 promote OCF_SUCCESS
+	AgentRun monitor OCF_RUNNING_MASTER
+	AgentRun@$IP_2 monitor OCF_RUNNING_MASTER
+	
+	AgentRun demote OCF_SUCCESS
+	AgentRun monitor OCF_NOT_RUNNING
+	AgentRun@$IP_2 demote OCF_SUCCESS
+	AgentRun@$IP_2 monitor OCF_NOT_RUNNING
+	
+	AgentRun stop 
+	AgentRun@$IP_2 stop

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-ha/resource-agents.git



More information about the Debian-HA-Commits mailing list